From 4686391a6ce004eef529536186c038888d1c8a0f Mon Sep 17 00:00:00 2001 From: yangxudong Date: Fri, 20 Dec 2024 14:52:39 +0800 Subject: [PATCH 01/51] fix bug of autoint model demo config --- easy_rec/python/inference/predictor.py | 2 +- examples/configs/autoint_on_movielens.config | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easy_rec/python/inference/predictor.py b/easy_rec/python/inference/predictor.py index 93220f047..fa0853ea5 100644 --- a/easy_rec/python/inference/predictor.py +++ b/easy_rec/python/inference/predictor.py @@ -534,7 +534,7 @@ def _parse_value(all_vals): ] for k in self._reserved_cols: if k in all_vals and all_vals[k].dtype == np.object: - all_vals[k] = [val.decode('utf-8') for val in all_vals[k]] + all_vals[k] = [val.decode('utf-8', errors='ignore') for val in all_vals[k]] ts2 = time.time() reserve_vals = self._get_reserve_vals(self._reserved_cols, diff --git a/examples/configs/autoint_on_movielens.config b/examples/configs/autoint_on_movielens.config index f74918899..35f0009ae 100644 --- a/examples/configs/autoint_on_movielens.config +++ b/examples/configs/autoint_on_movielens.config @@ -3,7 +3,7 @@ eval_input_path: "examples/data/movielens_1m/movies_test_data" model_dir: "examples/ckpt/autoint_on_movieslen_ckpt" train_config { - log_step_count_steps: 100 + log_step_count_steps: 1000 optimizer_config: { adam_optimizer: { learning_rate: { @@ -157,5 +157,5 @@ model_config: { embedding_regularization: 1e-4 } export_config { - multi_placeholder: false + multi_placeholder: true } From a23298b375e4a65235ddc187f316bb80c27b873f Mon Sep 17 00:00:00 2001 From: yangxudong Date: Wed, 25 Dec 2024 13:14:19 +0800 Subject: [PATCH 02/51] fix bug of wrong demo model configs --- examples/configs/autoint_on_movielens.config | 4 ++-- examples/configs/dcn_backbone_on_movielens.config | 5 +---- examples/configs/dcn_on_movielens.config | 3 --- examples/configs/deepfm_backbone_on_movielens.config | 3 --- examples/configs/deepfm_on_movielens.config | 3 --- examples/configs/dssm_on_books.config | 3 --- examples/configs/dssm_on_books_negative_sample.config | 3 --- examples/configs/dssm_senet_on_taobao.config | 3 --- examples/configs/fibinet_on_movielens.config | 3 --- examples/configs/masknet_on_movielens.config | 3 --- examples/configs/mind_on_books.config | 3 --- examples/configs/mind_on_books_negative_sample.config | 3 --- 12 files changed, 3 insertions(+), 36 deletions(-) diff --git a/examples/configs/autoint_on_movielens.config b/examples/configs/autoint_on_movielens.config index 35f0009ae..cbf43729f 100644 --- a/examples/configs/autoint_on_movielens.config +++ b/examples/configs/autoint_on_movielens.config @@ -3,7 +3,7 @@ eval_input_path: "examples/data/movielens_1m/movies_test_data" model_dir: "examples/ckpt/autoint_on_movieslen_ckpt" train_config { - log_step_count_steps: 1000 + log_step_count_steps: 100 optimizer_config: { adam_optimizer: { learning_rate: { @@ -17,7 +17,7 @@ train_config { } use_moving_average: false } - save_checkpoints_steps: 100 + save_checkpoints_steps: 1000 sync_replicas: True num_steps: 2500 } diff --git a/examples/configs/dcn_backbone_on_movielens.config b/examples/configs/dcn_backbone_on_movielens.config index dbffb76b7..7be038dbf 100644 --- a/examples/configs/dcn_backbone_on_movielens.config +++ b/examples/configs/dcn_backbone_on_movielens.config @@ -1,6 +1,6 @@ train_input_path: "examples/data/movielens_1m/movies_train_data" eval_input_path: "examples/data/movielens_1m/movies_test_data" -model_dir: "examples/ckpt/dcn_on_movieslen" +model_dir: "examples/ckpt/dcn_backbone_on_movieslen" train_config { log_step_count_steps: 100 @@ -199,6 +199,3 @@ model_config: { } embedding_regularization: 1e-4 } -export_config { - multi_placeholder: false -} diff --git a/examples/configs/dcn_on_movielens.config b/examples/configs/dcn_on_movielens.config index 4556fef88..09110c81d 100644 --- a/examples/configs/dcn_on_movielens.config +++ b/examples/configs/dcn_on_movielens.config @@ -180,6 +180,3 @@ model_config: { } embedding_regularization: 1e-4 } -export_config { - multi_placeholder: false -} diff --git a/examples/configs/deepfm_backbone_on_movielens.config b/examples/configs/deepfm_backbone_on_movielens.config index 56f210b10..5e6ea9b8d 100644 --- a/examples/configs/deepfm_backbone_on_movielens.config +++ b/examples/configs/deepfm_backbone_on_movielens.config @@ -241,6 +241,3 @@ model_config: { } embedding_regularization: 1e-4 } -export_config { - multi_placeholder: false -} diff --git a/examples/configs/deepfm_on_movielens.config b/examples/configs/deepfm_on_movielens.config index f87b7c894..a49a1988c 100644 --- a/examples/configs/deepfm_on_movielens.config +++ b/examples/configs/deepfm_on_movielens.config @@ -182,6 +182,3 @@ model_config: { } embedding_regularization: 1e-4 } -export_config { - multi_placeholder: false -} diff --git a/examples/configs/dssm_on_books.config b/examples/configs/dssm_on_books.config index 326e7432f..eebcdb295 100644 --- a/examples/configs/dssm_on_books.config +++ b/examples/configs/dssm_on_books.config @@ -112,6 +112,3 @@ model_config:{ } embedding_regularization: 5e-5 } - -export_config { -} diff --git a/examples/configs/dssm_on_books_negative_sample.config b/examples/configs/dssm_on_books_negative_sample.config index 069ebb4a5..8e3fe87f1 100644 --- a/examples/configs/dssm_on_books_negative_sample.config +++ b/examples/configs/dssm_on_books_negative_sample.config @@ -127,6 +127,3 @@ model_config:{ loss_type: SOFTMAX_CROSS_ENTROPY embedding_regularization: 5e-6 } - -export_config { -} diff --git a/examples/configs/dssm_senet_on_taobao.config b/examples/configs/dssm_senet_on_taobao.config index f8f415d1a..7b8e0da1c 100644 --- a/examples/configs/dssm_senet_on_taobao.config +++ b/examples/configs/dssm_senet_on_taobao.config @@ -281,6 +281,3 @@ model_config:{ } embedding_regularization: 5e-5 } - -export_config { -} diff --git a/examples/configs/fibinet_on_movielens.config b/examples/configs/fibinet_on_movielens.config index 5b0feb072..b4ecaf613 100644 --- a/examples/configs/fibinet_on_movielens.config +++ b/examples/configs/fibinet_on_movielens.config @@ -200,6 +200,3 @@ model_config: { } embedding_regularization: 1e-4 } -export_config { - multi_placeholder: false -} diff --git a/examples/configs/masknet_on_movielens.config b/examples/configs/masknet_on_movielens.config index 4dd54c914..04205ddd5 100644 --- a/examples/configs/masknet_on_movielens.config +++ b/examples/configs/masknet_on_movielens.config @@ -196,6 +196,3 @@ model_config: { } embedding_regularization: 1e-4 } -export_config { - multi_placeholder: false -} diff --git a/examples/configs/mind_on_books.config b/examples/configs/mind_on_books.config index 4b50d04b4..d19eb5d96 100644 --- a/examples/configs/mind_on_books.config +++ b/examples/configs/mind_on_books.config @@ -111,6 +111,3 @@ model_config:{ } embedding_regularization: 5e-5 } - -export_config { -} diff --git a/examples/configs/mind_on_books_negative_sample.config b/examples/configs/mind_on_books_negative_sample.config index 3816e9f7e..6058e6a2f 100644 --- a/examples/configs/mind_on_books_negative_sample.config +++ b/examples/configs/mind_on_books_negative_sample.config @@ -128,6 +128,3 @@ model_config:{ embedding_regularization: 5e-5 loss_type: SOFTMAX_CROSS_ENTROPY } - -export_config { -} From 7f5ddad1707c04c9925e0547c8ad347086efdf77 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Tue, 31 Dec 2024 16:19:31 +0800 Subject: [PATCH 03/51] fix bug of undefined flags of easyrec tools run with DeepRec --- .../python/tools/add_boundaries_to_config.py | 3 +++ .../tools/add_feature_info_to_config.py | 3 +++ easy_rec/python/tools/faiss_index_pai.py | 3 +++ easy_rec/python/tools/feature_selection.py | 3 +++ easy_rec/python/tools/hit_rate_ds.py | 3 +++ easy_rec/python/tools/hit_rate_pai.py | 3 +++ easy_rec/python/tools/pre_check.py | 3 +++ easy_rec/python/tools/split_model_pai.py | 3 +++ easy_rec/python/tools/split_pdn_model_pai.py | 3 +++ easy_rec/python/utils/io_util.py | 21 +++++++++++++++++++ 10 files changed, 48 insertions(+) diff --git a/easy_rec/python/tools/add_boundaries_to_config.py b/easy_rec/python/tools/add_boundaries_to_config.py index 09d2d9a1d..18d5f6037 100644 --- a/easy_rec/python/tools/add_boundaries_to_config.py +++ b/easy_rec/python/tools/add_boundaries_to_config.py @@ -3,11 +3,13 @@ import json import logging import os +import sys import common_io import tensorflow as tf from easy_rec.python.utils import config_util +from easy_rec.python.utils import io_util if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -61,4 +63,5 @@ def main(argv): if __name__ == '__main__': + sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv) tf.app.run() diff --git a/easy_rec/python/tools/add_feature_info_to_config.py b/easy_rec/python/tools/add_feature_info_to_config.py index b11cfc0a7..7594d038b 100644 --- a/easy_rec/python/tools/add_feature_info_to_config.py +++ b/easy_rec/python/tools/add_feature_info_to_config.py @@ -3,10 +3,12 @@ import json import logging import os +import sys import tensorflow as tf from easy_rec.python.utils import config_util +from easy_rec.python.utils import io_util from easy_rec.python.utils.hive_utils import HiveUtils if tf.__version__ >= '2.0': @@ -139,4 +141,5 @@ def main(argv): if __name__ == '__main__': + sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv) tf.app.run() diff --git a/easy_rec/python/tools/faiss_index_pai.py b/easy_rec/python/tools/faiss_index_pai.py index 718382733..b7eb66bc0 100644 --- a/easy_rec/python/tools/faiss_index_pai.py +++ b/easy_rec/python/tools/faiss_index_pai.py @@ -4,10 +4,12 @@ import logging import os +import sys import faiss import numpy as np import tensorflow as tf +from easy_rec.python.utils import io_util logging.basicConfig( level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') @@ -109,4 +111,5 @@ def main(argv): if __name__ == '__main__': + sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv) tf.app.run() diff --git a/easy_rec/python/tools/feature_selection.py b/easy_rec/python/tools/feature_selection.py index 6d9f59911..f50a00fac 100644 --- a/easy_rec/python/tools/feature_selection.py +++ b/easy_rec/python/tools/feature_selection.py @@ -3,6 +3,7 @@ import json import os +import sys from collections import OrderedDict import numpy as np @@ -11,6 +12,7 @@ from tensorflow.python.framework.meta_graph import read_meta_graph_file from easy_rec.python.utils import config_util +from easy_rec.python.utils import io_util if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -299,6 +301,7 @@ def _visualize_feature_importance(self, feature_importance, group_name): if __name__ == '__main__': + sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv) if FLAGS.model_type == 'variational_dropout': fs = VariationalDropoutFS( FLAGS.config_path, diff --git a/easy_rec/python/tools/hit_rate_ds.py b/easy_rec/python/tools/hit_rate_ds.py index 552b96aad..5528e0aa2 100644 --- a/easy_rec/python/tools/hit_rate_ds.py +++ b/easy_rec/python/tools/hit_rate_ds.py @@ -20,12 +20,14 @@ import json import logging import os +import sys import graphlearn as gl import tensorflow as tf from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils import config_util +from easy_rec.python.utils import io_util from easy_rec.python.utils.config_util import process_multi_file_input_path from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch from easy_rec.python.utils.hit_rate_utils import load_graph @@ -217,4 +219,5 @@ def main(): if __name__ == '__main__': + sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv) main() diff --git a/easy_rec/python/tools/hit_rate_pai.py b/easy_rec/python/tools/hit_rate_pai.py index 73c8a2095..5f97b3429 100644 --- a/easy_rec/python/tools/hit_rate_pai.py +++ b/easy_rec/python/tools/hit_rate_pai.py @@ -17,8 +17,10 @@ from __future__ import division from __future__ import print_function +import sys import tensorflow as tf +from easy_rec.python.utils import io_util from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch from easy_rec.python.utils.hit_rate_utils import load_graph from easy_rec.python.utils.hit_rate_utils import reduce_hitrate @@ -131,4 +133,5 @@ def main(): if __name__ == '__main__': + sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv) main() diff --git a/easy_rec/python/tools/pre_check.py b/easy_rec/python/tools/pre_check.py index 8fcaa2caf..da7f1923b 100644 --- a/easy_rec/python/tools/pre_check.py +++ b/easy_rec/python/tools/pre_check.py @@ -3,12 +3,14 @@ import json import logging import os +import sys import tensorflow as tf from easy_rec.python.input.input import Input from easy_rec.python.utils import config_util from easy_rec.python.utils import fg_util +from easy_rec.python.utils import io_util from easy_rec.python.utils.check_utils import check_env_and_input_path from easy_rec.python.utils.check_utils import check_sequence @@ -114,4 +116,5 @@ def main(argv): if __name__ == '__main__': + sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv) tf.app.run() diff --git a/easy_rec/python/tools/split_model_pai.py b/easy_rec/python/tools/split_model_pai.py index bdb2087de..cf1657deb 100644 --- a/easy_rec/python/tools/split_model_pai.py +++ b/easy_rec/python/tools/split_model_pai.py @@ -2,6 +2,7 @@ import copy import logging import os +import sys import tensorflow as tf from tensorflow.core.framework import graph_pb2 @@ -11,6 +12,7 @@ from tensorflow.python.saved_model import signature_constants from tensorflow.python.tools import saved_model_utils from tensorflow.python.training import saver as tf_saver +from easy_rec.python.utils import io_util if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -282,4 +284,5 @@ def main(argv): if __name__ == '__main__': + sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv) tf.app.run() diff --git a/easy_rec/python/tools/split_pdn_model_pai.py b/easy_rec/python/tools/split_pdn_model_pai.py index e2341d57d..849250b37 100644 --- a/easy_rec/python/tools/split_pdn_model_pai.py +++ b/easy_rec/python/tools/split_pdn_model_pai.py @@ -2,6 +2,7 @@ import copy import logging import os +import sys import tensorflow as tf from tensorflow.core.framework import graph_pb2 @@ -12,6 +13,7 @@ from tensorflow.python.saved_model.utils_impl import get_variables_path from tensorflow.python.tools import saved_model_utils from tensorflow.python.training import saver as tf_saver +from easy_rec.python.utils import io_util FLAGS = tf.app.flags.FLAGS tf.app.flags.DEFINE_string('model_dir', '', '') @@ -265,4 +267,5 @@ def main(argv): if __name__ == '__main__': + sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv) tf.app.run() diff --git a/easy_rec/python/utils/io_util.py b/easy_rec/python/utils/io_util.py index 091e10e07..431394e5e 100644 --- a/easy_rec/python/utils/io_util.py +++ b/easy_rec/python/utils/io_util.py @@ -185,3 +185,24 @@ def read_data_from_json_path(json_path): else: logging.info('json_path not exists, return None') return None + +def filter_unknown_args(flags, args): + """Filter unknown args.""" + defined_flags = set(flag.name for flag in flags._flags().values()) + logging.info('defined arguments: %s', ', '.join(defined_flags)) + logging.info('actual arguments: %s', ', '.join(args[1:])) + known_args = [args[0]] + unknown = False + for arg in args[1:]: + if arg.startswith('--'): + flag_name = arg.split('=')[0][2:] + if flag_name in defined_flags: + known_args.append(arg) + unknown = False + else: + unknown = True + logging.warning('Ignore unknown arg: %s' % arg) + elif not unknown: + known_args.append(arg) + logging.info('keep arguments: %s', ', '.join(known_args[1:])) + return known_args From 74b1edcc600e273e9446e33a8e7b9938134bb220 Mon Sep 17 00:00:00 2001 From: "weisu.yxd" Date: Fri, 3 Jan 2025 12:53:23 +0800 Subject: [PATCH 04/51] fix bug --- easy_rec/python/utils/io_util.py | 76 +++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 16 deletions(-) diff --git a/easy_rec/python/utils/io_util.py b/easy_rec/python/utils/io_util.py index 431394e5e..cfe20d4ac 100644 --- a/easy_rec/python/utils/io_util.py +++ b/easy_rec/python/utils/io_util.py @@ -186,23 +186,67 @@ def read_data_from_json_path(json_path): logging.info('json_path not exists, return None') return None + +def convert_tf_flags_to_argparse(flags): + """Convert tf.app.flags.FLAGS to argparse.ArgumentParser. + + Args: + flags: tf.app.flags.FLAGS + Returns: + argparse.ArgumentParser: configurate ArgumentParser object + """ + import argparse + import ast + parser = argparse.ArgumentParser() + + args = set() + for flag in flags._flags().values(): + flag_name = flag.name + if flag_name in args: + continue + args.add(flag_name) + default = flag.value + flag_type = type(default) + help_str = flag.help or '' + if flag_type == bool: + parser.add_argument( + '--' + flag_name, + dest=flag_name, + action='store_true' if default else 'store_false', + help=help_str) + elif flag_type == str: + if hasattr(flag, 'choices') and flag.choices: + parser.add_argument( + '--' + flag_name, + type=str, + choices=flag.choices, + default=default, + help=help_str) + else: + parser.add_argument( + '--' + flag_name, type=str, default=default, help=help_str) + elif flag_type in (list, dict): + parser.add_argument( + '--' + flag_name, + type=lambda s: ast.literal_eval(s), + default=default, + help=help_str) + else: + parser.add_argument( + '--' + flag_name, type=flag_type, default=default, help=help_str) + return parser + + def filter_unknown_args(flags, args): """Filter unknown args.""" - defined_flags = set(flag.name for flag in flags._flags().values()) - logging.info('defined arguments: %s', ', '.join(defined_flags)) - logging.info('actual arguments: %s', ', '.join(args[1:])) known_args = [args[0]] - unknown = False - for arg in args[1:]: - if arg.startswith('--'): - flag_name = arg.split('=')[0][2:] - if flag_name in defined_flags: - known_args.append(arg) - unknown = False - else: - unknown = True - logging.warning('Ignore unknown arg: %s' % arg) - elif not unknown: - known_args.append(arg) - logging.info('keep arguments: %s', ', '.join(known_args[1:])) + parser = convert_tf_flags_to_argparse(flags) + args, unknown = parser.parse_known_args(args) + if len(unknown) > 1: + logging.info('undefined arguments: %s', ', '.join(unknown[1:])) + for key, value in vars(args).items(): + if type(value) != bool and not value: + continue + known_args.append('--' + key + '=' + str(value)) + logging.info('defined arguments: %s', ', '.join(known_args[1:])) return known_args From 8ffbc29ab78a77b25ce5be5d9f0fbb9756195cad Mon Sep 17 00:00:00 2001 From: "weisu.yxd" Date: Fri, 3 Jan 2025 16:59:00 +0800 Subject: [PATCH 05/51] fix bug --- easy_rec/python/utils/io_util.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/easy_rec/python/utils/io_util.py b/easy_rec/python/utils/io_util.py index cfe20d4ac..d5a949630 100644 --- a/easy_rec/python/utils/io_util.py +++ b/easy_rec/python/utils/io_util.py @@ -199,15 +199,21 @@ def convert_tf_flags_to_argparse(flags): import ast parser = argparse.ArgumentParser() - args = set() + args = {} for flag in flags._flags().values(): flag_name = flag.name if flag_name in args: + args[flag_name][0] = True continue - args.add(flag_name) default = flag.value flag_type = type(default) help_str = flag.help or '' + args[flag_name] = [ + False, flag_type, default, help_str, + flag.choices if hasattr(flag, 'choices') else None + ] + + for flag_name, (multi, flag_type, default, help_str, choices) in args.items(): if flag_type == bool: parser.add_argument( '--' + flag_name, @@ -215,13 +221,16 @@ def convert_tf_flags_to_argparse(flags): action='store_true' if default else 'store_false', help=help_str) elif flag_type == str: - if hasattr(flag, 'choices') and flag.choices: + if choices: parser.add_argument( '--' + flag_name, type=str, - choices=flag.choices, + choices=choices, default=default, help=help_str) + elif multi: + parser.add_argument( + '--' + flag_name, type=str, default=default, help=help_str) else: parser.add_argument( '--' + flag_name, type=str, default=default, help=help_str) @@ -231,9 +240,12 @@ def convert_tf_flags_to_argparse(flags): type=lambda s: ast.literal_eval(s), default=default, help=help_str) - else: + elif flag_type in (int, float): parser.add_argument( '--' + flag_name, type=flag_type, default=default, help=help_str) + else: + parser.add_argument( + '--' + flag_name, type=str, default=default, help=help_str) return parser @@ -245,7 +257,7 @@ def filter_unknown_args(flags, args): if len(unknown) > 1: logging.info('undefined arguments: %s', ', '.join(unknown[1:])) for key, value in vars(args).items(): - if type(value) != bool and not value: + if type(value) in (list, dict) and not value: continue known_args.append('--' + key + '=' + str(value)) logging.info('defined arguments: %s', ', '.join(known_args[1:])) From 3f97db9aa40323e3b21fa6b9b6045af5ec80d703 Mon Sep 17 00:00:00 2001 From: "weisu.yxd" Date: Fri, 3 Jan 2025 17:03:34 +0800 Subject: [PATCH 06/51] fix bug --- easy_rec/python/utils/io_util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/easy_rec/python/utils/io_util.py b/easy_rec/python/utils/io_util.py index d5a949630..5cef1d9ef 100644 --- a/easy_rec/python/utils/io_util.py +++ b/easy_rec/python/utils/io_util.py @@ -230,7 +230,11 @@ def convert_tf_flags_to_argparse(flags): help=help_str) elif multi: parser.add_argument( - '--' + flag_name, type=str, default=default, help=help_str) + '--' + flag_name, + type=str, + action='append', + default=default, + help=help_str) else: parser.add_argument( '--' + flag_name, type=str, default=default, help=help_str) From c7d7a0db51370b2b3223bf007e745e5184108dbd Mon Sep 17 00:00:00 2001 From: "weisu.yxd" Date: Fri, 3 Jan 2025 20:00:32 +0800 Subject: [PATCH 07/51] fix bug --- easy_rec/python/utils/io_util.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/easy_rec/python/utils/io_util.py b/easy_rec/python/utils/io_util.py index 5cef1d9ef..7b4666391 100644 --- a/easy_rec/python/utils/io_util.py +++ b/easy_rec/python/utils/io_util.py @@ -213,12 +213,24 @@ def convert_tf_flags_to_argparse(flags): flag.choices if hasattr(flag, 'choices') else None ] + def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + for flag_name, (multi, flag_type, default, help_str, choices) in args.items(): if flag_type == bool: parser.add_argument( '--' + flag_name, - dest=flag_name, - action='store_true' if default else 'store_false', + type=str2bool, + nargs='?', + const=True, + default=False, help=help_str) elif flag_type == str: if choices: From eed82839481ae0d89f2201ca06fd9fb757144ba5 Mon Sep 17 00:00:00 2001 From: "weisu.yxd" Date: Mon, 6 Jan 2025 09:56:40 +0800 Subject: [PATCH 08/51] fix bug --- easy_rec/python/utils/io_util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easy_rec/python/utils/io_util.py b/easy_rec/python/utils/io_util.py index 7b4666391..92c9c8a1f 100644 --- a/easy_rec/python/utils/io_util.py +++ b/easy_rec/python/utils/io_util.py @@ -273,6 +273,8 @@ def filter_unknown_args(flags, args): if len(unknown) > 1: logging.info('undefined arguments: %s', ', '.join(unknown[1:])) for key, value in vars(args).items(): + if value is None: + continue if type(value) in (list, dict) and not value: continue known_args.append('--' + key + '=' + str(value)) From d7a1860f9efcd6fcb3957bcfc8c027d1d6b78e95 Mon Sep 17 00:00:00 2001 From: yanzhen1233 Date: Thu, 16 Jan 2025 18:37:38 +0800 Subject: [PATCH 09/51] Build docker (#515) * Updated Docker image --- docker/Dockerfile_tf112 | 74 +++++++++++++++++++++++++++++++++++ docker/Dockerfile_tf115 | 36 +++++++++++++++++ docker/Dockerfile_tf212 | 36 +++++++++++++++++ examples/readme.md | 18 ++++----- requirements/runtime.txt | 3 +- scripts/build_docker_tf112.sh | 21 ++++++++++ scripts/build_docker_tf115.sh | 21 ++++++++++ scripts/build_docker_tf212.sh | 21 ++++++++++ 8 files changed, 220 insertions(+), 10 deletions(-) create mode 100644 docker/Dockerfile_tf112 create mode 100644 docker/Dockerfile_tf115 create mode 100644 docker/Dockerfile_tf212 create mode 100644 scripts/build_docker_tf112.sh create mode 100644 scripts/build_docker_tf115.sh create mode 100644 scripts/build_docker_tf212.sh diff --git a/docker/Dockerfile_tf112 b/docker/Dockerfile_tf112 new file mode 100644 index 000000000..9077e59e2 --- /dev/null +++ b/docker/Dockerfile_tf112 @@ -0,0 +1,74 @@ +#FROM tensorflow/tensorflow:1.12.0 +FROM my_tensorflow_base:1.12.0-py2 + +COPY docker/sources_18.04.list /etc/apt/sources.list + +# necessary for later commands to take effect +RUN md5sum /etc/apt/sources.list \ + && apt-get update \ + && apt-get install apt-utils inetutils-ping wget curl telnet vim strace libpq-dev curl libsasl2-dev gcc g++ unzip openjdk-8-jdk -y \ + && apt-get install build-essential cython -y \ + && pip install cython \ + && pip install setuptools_scm +# 检查 Cython 是否安装成功 +RUN python -c "import Cython" +RUN pip --version + +RUN mkdir /EasyRec +COPY requirements /EasyRec/requirements +COPY requirements.txt /EasyRec/ +COPY easy_rec /EasyRec/easy_rec/ +COPY setup.cfg /EasyRec/ +COPY setup.py /EasyRec/ +COPY MANIFEST.in /EasyRec/ +COPY README.md /EasyRec/ +COPY scripts /EasyRec/scripts + +RUN curl "http://easyrec.oss-cn-beijing.aliyuncs.com/tools/odpscmd_public_0.45.0.zip" -o /EasyRec/odpscmd_public.zip +RUN mkdir /usr/local/odps_clt/ && cd /usr/local/odps_clt/ && unzip /EasyRec/odpscmd_public.zip +RUN ln -s /usr/local/odps_clt/bin/odpscmd /usr/local/bin/odpscmd + +RUN pip install pystack-debugger idna kafka-python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +# 升级pip +RUN pip install --upgrade pip setuptools wheel + +# 安装 setuptools-rust 和 rustc +RUN pip install setuptools-rust +RUN pip install tensorflow_probability==0.5.0 +RUN apt-get update && apt-get install -y rustc +RUN apt-get update && \ + apt-get install -y rustc && \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ + . $HOME/.cargo/env +# 安装 cryptography +RUN pip install cryptography +# 安装基础工具链与依赖项 +RUN apt-get update && \ + apt-get install -y build-essential libssl-dev libffi-dev python-dev && \ + apt-get install -y rustc cargo cmake curl + +# 设置国内的 Rust 镜像源 +RUN echo '[source.crates-io]\n' > $HOME/.cargo/config +RUN echo 'replace-with = "ustc"' >> $HOME/.cargo/config +RUN echo '[source.ustc]\n' >> $HOME/.cargo/config +RUN echo 'registry = "https://mirrors.ustc.edu.cn/crates.io-index"' >> $HOME/.cargo/config + +# 确保 curl 支持 HTTP2 +RUN curl -V + +# 显示安装好的工具链版本，确保已正确安装 +RUN rustc --version +RUN cargo --version +RUN cmake --version + +RUN pip install -r /EasyRec/requirements/runtime.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip install -r /EasyRec/requirements/extra.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com + +RUN pip install --user -U https://tfsmoke1.oss-cn-zhangjiakou.aliyuncs.com/tunnel_paiio/common_io/py2/common_io-0.1.0-cp27-cp27mu-linux_x86_64.whl +RUN pip install graphlearn + +RUN cd /EasyRec && python setup.py install +RUN rm -rf /EasyRec +RUN python -c "import easy_rec; import pyhive; import datahub; import kafka" + +COPY docker/hadoop_env.sh /opt/hadoop_env.sh \ No newline at end of file diff --git a/docker/Dockerfile_tf115 b/docker/Dockerfile_tf115 new file mode 100644 index 000000000..30d857726 --- /dev/null +++ b/docker/Dockerfile_tf115 @@ -0,0 +1,36 @@ +FROM datascience-registry.cn-beijing.cr.aliyuncs.com/tensorflow/tensorflow:1.15.5 + +COPY docker/sources_18.04.list /etc/apt/sources.list + +# necessary for later commands to take effect +RUN md5sum /etc/apt/sources.list + +RUN apt-get update +RUN apt-get install apt-utils inetutils-ping wget curl telnet vim strace libpq-dev curl libsasl2-dev gcc g++ unzip openjdk-8-jdk -y + +RUN mkdir /EasyRec +COPY requirements /EasyRec/requirements +COPY requirements.txt /EasyRec/ +COPY easy_rec /EasyRec/easy_rec/ +COPY setup.cfg /EasyRec/ +COPY setup.py /EasyRec/ +COPY MANIFEST.in /EasyRec/ +COPY README.md /EasyRec/ +COPY scripts /EasyRec/scripts + +RUN curl "http://easyrec.oss-cn-beijing.aliyuncs.com/tools/odpscmd_public_0.45.0.zip" -o /EasyRec/odpscmd_public.zip +RUN mkdir /usr/local/odps_clt/ && cd /usr/local/odps_clt/ && unzip /EasyRec/odpscmd_public.zip +RUN ln -s /usr/local/odps_clt/bin/odpscmd /usr/local/bin/odpscmd +RUN pip3 install --upgrade pip +RUN pip3 install pystack-debugger idna kafka-python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install -r /EasyRec/requirements/runtime.txt +RUN pip3 install -r /EasyRec/requirements/extra.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-1.1.0-cp36-cp36m-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/releases/pai_automl-0.0.1rc1-py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.3.0-cp36-cp36m-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install tensorflow_probability==0.8 +RUN cd /EasyRec && pip install . +RUN rm -rf /EasyRec +RUN python -c "import easy_rec; easy_rec.help(); import pyhive; import datahub; import kafka" + +COPY docker/hadoop_env.sh /opt/hadoop_env.sh \ No newline at end of file diff --git a/docker/Dockerfile_tf212 b/docker/Dockerfile_tf212 new file mode 100644 index 000000000..49977d60b --- /dev/null +++ b/docker/Dockerfile_tf212 @@ -0,0 +1,36 @@ +FROM tensorflow/tensorflow:2.12.0 +COPY docker/sources_20.04.list /etc/apt/sources.list + +# necessary for later commands to take effect +RUN md5sum /etc/apt/sources.list + +RUN apt-get update +RUN apt-get install apt-utils inetutils-ping wget curl telnet vim strace libpq-dev curl libsasl2-dev gcc g++ unzip openjdk-8-jdk -y + +RUN mkdir /EasyRec +COPY requirements /EasyRec/requirements +COPY requirements.txt /EasyRec/ +COPY easy_rec /EasyRec/easy_rec/ +COPY setup.cfg /EasyRec/ +COPY setup.py /EasyRec/ +COPY MANIFEST.in /EasyRec/ +COPY README.md /EasyRec/ +COPY scripts /EasyRec/scripts + +RUN curl "http://easyrec.oss-cn-beijing.aliyuncs.com/tools/odpscmd_public_0.45.0.zip" -o /EasyRec/odpscmd_public.zip +RUN mkdir /usr/local/odps_clt/ && cd /usr/local/odps_clt/ && unzip /EasyRec/odpscmd_public.zip +RUN ln -s /usr/local/odps_clt/bin/odpscmd /usr/local/bin/odpscmd + +RUN pip3 install pystack-debugger idna kafka-python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install -r /EasyRec/requirements/runtime.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install -r /EasyRec/requirements/extra.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-1.1.0-cp38-cp38-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +# RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/releases/pai_automl-0.0.1rc1-py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install tensorflow_probability==0.20.0 +#RUN pip3 install encodings +RUN pip3 install https://dlc-task.oss-cn-hangzhou.aliyuncs.com/whl/common_io-0.4.1%2Btunnel-py2.py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN cd /EasyRec && python setup.py install +RUN rm -rf /EasyRec +# RUN python -c "import easy_rec; easy_rec.help(); import pyhive; import datahub; import kafka" + +COPY docker/hadoop_env.sh /opt/hadoop_env.sh \ No newline at end of file diff --git a/examples/readme.md b/examples/readme.md index bf936cf21..a587cc991 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -36,14 +36,14 @@ cd EasyRec -- Docker环境可选 (1) `python=3.6.9` + `tenserflow=1.15.5` -docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 -docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.8.5 +docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.8.5 docker exec -it bash -(2) `python=3.8.10` + `tenserflow=2.10.0` -docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4 -docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4 +(2) `python=3.8.10` + `tenserflow=2.12.0` +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-0.8.5 +docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-0.8.5 docker exec -it bash ``` @@ -56,12 +56,12 @@ cd EasyRec -- Docker环境可选 (1) `python=3.6.9` + `tenserflow=1.15.5` -bash scripts/build_docker.sh +bash scripts/build_docker_tf115.sh sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15- -(2) `python=3.8.10` + `tenserflow=2.10.0` -bash scripts/build_docker_tf210.sh -sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10- +(2) `python=3.8.10` + `tenserflow=2.12.0` +bash scripts/build_docker_tf212.sh +sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12- sudo docker exec -it bash ``` diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 8e6fa5616..402e16b6e 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -1,4 +1,5 @@ -eas_prediction +eas_prediction == 0.24; python_version < '3.0' +eas_prediction; python_version >= '3.0' future matplotlib numpy <= 1.23 diff --git a/scripts/build_docker_tf112.sh b/scripts/build_docker_tf112.sh new file mode 100644 index 000000000..5bad07df3 --- /dev/null +++ b/scripts/build_docker_tf112.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +bash scripts/gen_proto.sh +if [ $? -ne 0 ] +then + echo "gen proto failed" + exit 1 +fi + +version=`grep "__version__" easy_rec/version.py | awk '{ if($1 == "__version__") print $NF}'` +# strip "'" +version=${version//\'/} +echo "EasyRec Version: $version" + +if [ -z "$version" ] +then + echo "Failed to get EasyRec version" + exit 1 +fi + +sudo docker build --network=host . -f docker/Dockerfile_tf112 -t mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py27-tf1.12-${version} \ No newline at end of file diff --git a/scripts/build_docker_tf115.sh b/scripts/build_docker_tf115.sh new file mode 100644 index 000000000..a52616944 --- /dev/null +++ b/scripts/build_docker_tf115.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +bash scripts/gen_proto.sh +if [ $? -ne 0 ] +then + echo "gen proto failed" + exit 1 +fi + +version=`grep "__version__" easy_rec/version.py | awk '{ if($1 == "__version__") print $NF}'` +# strip "'" +version=${version//\'/} +echo "EasyRec Version: $version" + +if [ -z "$version" ] +then + echo "Failed to get EasyRec version" + exit 1 +fi + +sudo docker build --network=host . -f docker/Dockerfile_tf115 -t mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-${version} \ No newline at end of file diff --git a/scripts/build_docker_tf212.sh b/scripts/build_docker_tf212.sh new file mode 100644 index 000000000..50baf5aa3 --- /dev/null +++ b/scripts/build_docker_tf212.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +bash scripts/gen_proto.sh +if [ $? -ne 0 ] +then + echo "gen proto failed" + exit 1 +fi + +version=`grep "__version__" easy_rec/version.py | awk '{ if($1 == "__version__") print $NF}'` +# strip "'" +version=${version//\'/} +echo "EasyRec Version: $version" + +if [ -z "$version" ] +then + echo "Failed to get EasyRec version" + exit 1 +fi + +sudo docker build --network=host . -f docker/Dockerfile_tf212 -t mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-${version} \ No newline at end of file From 179cfa37843775731bece8c5276196bc376bf025 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Fri, 17 Jan 2025 15:18:58 +0800 Subject: [PATCH 10/51] merge from master --- docker/Dockerfile_tf210 | 1 + docs/source/feature/feature.rst | 1 - docs/source/quick_start/local_tutorial.md | 34 +++++++++++++++++++++-- docs/source/vector_retrieve.md | 4 +-- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile_tf210 b/docker/Dockerfile_tf210 index 54c67b698..90b54adef 100644 --- a/docker/Dockerfile_tf210 +++ b/docker/Dockerfile_tf210 @@ -27,6 +27,7 @@ RUN pip3 install -r /EasyRec/requirements/runtime.txt -i http://mirrors.aliyun RUN pip3 install -r /EasyRec/requirements/extra.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com RUN pip3 install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-1.1.0-cp38-cp38-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com # RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/releases/pai_automl-0.0.1rc1-py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install tensorflow_probability==0.18.0 RUN pip3 install https://dlc-task.oss-cn-hangzhou.aliyuncs.com/whl/common_io-0.4.1%2Btunnel-py2.py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com RUN cd /EasyRec && python setup.py install RUN rm -rf /EasyRec diff --git a/docs/source/feature/feature.rst b/docs/source/feature/feature.rst index 901fe6673..0f70989dd 100644 --- a/docs/source/feature/feature.rst +++ b/docs/source/feature/feature.rst @@ -136,7 +136,6 @@ RawFeature：连续值特征 features { input_names: "ctr" feature_type: RawFeature - embedding_dim: 8 } } diff --git a/docs/source/quick_start/local_tutorial.md b/docs/source/quick_start/local_tutorial.md index 443312ce9..9b2b0f0d4 100644 --- a/docs/source/quick_start/local_tutorial.md +++ b/docs/source/quick_start/local_tutorial.md @@ -8,20 +8,39 @@ #### 本地Anaconda安装 +温馨提示：**在搭载Apple芯片的MacBook上必须使用TensorFlow 2.5或更高版本**。 + Demo实验中使用的环境为 `python=3.6.8` + `tenserflow=1.12.0` ```bash conda create -n py36_tf12 python=3.6.8 conda activate py36_tf12 pip install tensorflow==1.12.0 +pip install tensorflow_probability==0.5.0 ``` +注意：必须要安装`tensorflow_probability`包，需要根据tensorflow的版本安装对应版本的`tensorflow_robability`包。 + +常见版本对应关系： + +| TensorFlow版本 | TensorFlowProbability版本 | +|--------------|-------------------------| +| 1.12 | 0.5.0 | +| 1.15 | 0.8.0 | +| 2.5.0 | 0.13.0 | +| 2.6.0 | 0.14.0 | +| 2.7.0 | 0.15.0 | +| 2.8.0 | 0.16.0 | +| 2.10 | 0.18.0 | +| 2.12 | 0.20.0 | + +其他版本对应关系请查看链接：[Releases · tensorflow/probability](https://github.com/tensorflow/probability/releases)。 + ```bash git clone https://github.com/alibaba/EasyRec.git cd EasyRec bash scripts/init.sh python setup.py install - ``` #### Docker镜像启动 @@ -33,13 +52,22 @@ Docker的环境为`python=3.6.9` + `tenserflow=1.15.5` ```bash git clone https://github.com/alibaba/EasyRec.git cd EasyRec -docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 -docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.8.5 +docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.8.5 docker exec -it bash ``` ##### 方法二：自行构建Docker镜像 +我们提供四个版本的tensorflow镜像构建示例，对应的脚步路径如下： + +- scripts/build_docker_tf112.sh +- scripts/build_docker_tf115.sh +- scripts/build_docker_tf210.sh +- scripts/build_docker_tf212.sh + +默认使用`tensorflow 1.15`的版本，示例脚本如下，请根据需要替换脚本路径： + ```bash git clone https://github.com/alibaba/EasyRec.git cd EasyRec diff --git a/docs/source/vector_retrieve.md b/docs/source/vector_retrieve.md index fe02a9358..8d3f7b909 100644 --- a/docs/source/vector_retrieve.md +++ b/docs/source/vector_retrieve.md @@ -37,7 +37,7 @@ pai -name easy_rec_ext -project algo_public_dev ## 使用示例 -### 1. 创建查询表 +### 1. 创建索引表 ```sql create table doc_table(pk BIGINT,vector string) partitioned by (pt string); @@ -53,7 +53,7 @@ VALUES ; ``` -### 2. 创建索引表 +### 2. 创建查询表 ```sql create table query_table(pk BIGINT,vector string) partitioned by (pt string); From 1f81f2bf1a4c6eca977d5fef388ca6319de2cbf9 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Fri, 20 Dec 2024 14:52:39 +0800 Subject: [PATCH 11/51] fix bug of autoint model demo config --- examples/configs/autoint_on_movielens.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/configs/autoint_on_movielens.config b/examples/configs/autoint_on_movielens.config index cbf43729f..6435974e8 100644 --- a/examples/configs/autoint_on_movielens.config +++ b/examples/configs/autoint_on_movielens.config @@ -3,7 +3,7 @@ eval_input_path: "examples/data/movielens_1m/movies_test_data" model_dir: "examples/ckpt/autoint_on_movieslen_ckpt" train_config { - log_step_count_steps: 100 + log_step_count_steps: 1000 optimizer_config: { adam_optimizer: { learning_rate: { From c00d33ec46278fda903880a1abb328b475ec9af0 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Wed, 25 Dec 2024 13:14:19 +0800 Subject: [PATCH 12/51] fix bug of wrong demo model configs --- examples/configs/autoint_on_movielens.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/configs/autoint_on_movielens.config b/examples/configs/autoint_on_movielens.config index 6435974e8..cbf43729f 100644 --- a/examples/configs/autoint_on_movielens.config +++ b/examples/configs/autoint_on_movielens.config @@ -3,7 +3,7 @@ eval_input_path: "examples/data/movielens_1m/movies_test_data" model_dir: "examples/ckpt/autoint_on_movieslen_ckpt" train_config { - log_step_count_steps: 1000 + log_step_count_steps: 100 optimizer_config: { adam_optimizer: { learning_rate: { From 8317df780c7184a3e23c1fd2d3cbec038977636d Mon Sep 17 00:00:00 2001 From: yangxudong Date: Fri, 17 Jan 2025 15:36:41 +0800 Subject: [PATCH 13/51] merge from master --- docs/source/feature/feature.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/source/feature/feature.rst b/docs/source/feature/feature.rst index 0f70989dd..384b8dc13 100644 --- a/docs/source/feature/feature.rst +++ b/docs/source/feature/feature.rst @@ -139,6 +139,18 @@ RawFeature：连续值特征 } } +也可以为每个RawFeature添加一个Field Embedding(原始特征值乘上一个可学习embedding参数)，如下： + +.. code:: protobuf + + feature_config:{ + features { + input_names: "ctr" + feature_type: RawFeature + embedding_dim: 8 + } + } + 分箱组件使用方法见： `机器学习组件 `_ 也可以手动导入分箱信息。如下： From 9cd8ca18bb1dee62aea1e47d3a2a4748aa4b7325 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Fri, 17 Jan 2025 16:55:56 +0800 Subject: [PATCH 14/51] merge from master --- docs/source/quick_start/local_tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/quick_start/local_tutorial.md b/docs/source/quick_start/local_tutorial.md index 9b2b0f0d4..51a242e9a 100644 --- a/docs/source/quick_start/local_tutorial.md +++ b/docs/source/quick_start/local_tutorial.md @@ -59,7 +59,7 @@ docker exec -it bash ##### 方法二：自行构建Docker镜像 -我们提供四个版本的tensorflow镜像构建示例，对应的脚步路径如下： +我们提供四个版本的tensorflow镜像构建示例，对应的脚本路径如下： - scripts/build_docker_tf112.sh - scripts/build_docker_tf115.sh From cd9fdcc4543ceab523ac1a547c4c4c37cc103162 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Fri, 17 Jan 2025 16:58:13 +0800 Subject: [PATCH 15/51] merge from master --- docs/source/quick_start/local_tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/quick_start/local_tutorial.md b/docs/source/quick_start/local_tutorial.md index 51a242e9a..f0eb41256 100644 --- a/docs/source/quick_start/local_tutorial.md +++ b/docs/source/quick_start/local_tutorial.md @@ -8,7 +8,7 @@ #### 本地Anaconda安装 -温馨提示：**在搭载Apple芯片的MacBook上必须使用TensorFlow 2.5或更高版本**。 +温馨提示：**在搭载Apple芯片的MacBook上必须使用TensorFlow 2.5或更高版本**，安装方法请查看TF官方文档。 Demo实验中使用的环境为 `python=3.6.8` + `tenserflow=1.12.0` From 893577677802e42c2113fd33a55f1968672f295d Mon Sep 17 00:00:00 2001 From: yangxudong Date: Fri, 17 Jan 2025 18:11:08 +0800 Subject: [PATCH 16/51] merge from master --- docs/source/quick_start/local_tutorial.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/quick_start/local_tutorial.md b/docs/source/quick_start/local_tutorial.md index f0eb41256..b3a75fb00 100644 --- a/docs/source/quick_start/local_tutorial.md +++ b/docs/source/quick_start/local_tutorial.md @@ -8,7 +8,7 @@ #### 本地Anaconda安装 -温馨提示：**在搭载Apple芯片的MacBook上必须使用TensorFlow 2.5或更高版本**，安装方法请查看TF官方文档。 +温馨提示：**在搭载Apple M系列芯片的MacBook上必须使用TensorFlow 2.5或更高版本**，安装方法请查看TF官方文档。 Demo实验中使用的环境为 `python=3.6.8` + `tenserflow=1.12.0` @@ -19,7 +19,7 @@ pip install tensorflow==1.12.0 pip install tensorflow_probability==0.5.0 ``` -注意：必须要安装`tensorflow_probability`包，需要根据tensorflow的版本安装对应版本的`tensorflow_robability`包。 +注意：必须要安装`tensorflow_probability`包，需要根据tensorflow的版本安装对应版本的`tensorflow_probability`包。 常见版本对应关系： From fdffb3738365fdfaa78a60e62fe8e2cad6f445b8 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 23 Jan 2025 15:09:09 +0800 Subject: [PATCH 17/51] fix bug of SeqAugment layer --- easy_rec/python/layers/keras/data_augment.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/easy_rec/python/layers/keras/data_augment.py b/easy_rec/python/layers/keras/data_augment.py index 6f12f8666..a11f08120 100644 --- a/easy_rec/python/layers/keras/data_augment.py +++ b/easy_rec/python/layers/keras/data_augment.py @@ -75,12 +75,28 @@ def mask_fn(): def reorder_fn(): return item_reorder(seq, length, aug_param.reorder_rate) - method = tf.random.uniform([], minval=0, maxval=3, dtype=tf.int32) + trans_fn = [] + if aug_param.crop_rate < 1.0: + trans_fn.append(crop_fn) + if aug_param.mask_rate > 0: + trans_fn.append(mask_fn) + if aug_param.reorder_rate > 0: + trans_fn.append(reorder_fn) + + num_trans = len(trans_fn) + if num_trans == 0: + return seq, length + + if num_trans == 1: + return trans_fn[0]() + + method = tf.random.uniform([], minval=0, maxval=num_trans, dtype=tf.int32) + if num_trans == 2: + return tf.cond(tf.equal(method, 0), trans_fn[0], trans_fn[1]) aug_seq, aug_len = tf.cond( tf.equal(method, 0), crop_fn, lambda: tf.cond(tf.equal(method, 1), mask_fn, reorder_fn)) - return aug_seq, aug_len From a62c34bf7a4db3096c552f639792ef509127e974 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 23 Jan 2025 15:37:19 +0800 Subject: [PATCH 18/51] add compacity for windows users --- git-lfs/git_lfs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/git-lfs/git_lfs.py b/git-lfs/git_lfs.py index ff2f492dd..0c079a6af 100644 --- a/git-lfs/git_lfs.py +++ b/git-lfs/git_lfs.py @@ -224,6 +224,7 @@ def get_yes_no(msg): 'usage: python git_lfs.py [pull] [push] [add filename] [resolve_conflict]' ) sys.exit(1) + home_directory = os.path.expanduser("~") with open('.git_oss_config_pub', 'r') as fin: git_oss_data_dir = None host = None @@ -237,7 +238,7 @@ def get_yes_no(msg): continue if line_str.startswith('#'): continue - line_str = line_str.replace('~/', os.environ['HOME'] + '/') + line_str = line_str.replace('~/', home_directory + '/') line_str = line_str.replace('${TMPDIR}/', os.environ.get('TMPDIR', '/tmp/')) line_str = line_str.replace('${PROJECT_NAME}', get_proj_name()) @@ -251,7 +252,7 @@ def get_yes_no(msg): elif line_tok[0] == 'git_oss_private_config': git_oss_private_path = line_tok[1] if git_oss_private_path.startswith('~/'): - git_oss_private_path = os.path.join(os.environ['HOME'], + git_oss_private_path = os.path.join(home_directory, git_oss_private_path[2:]) elif line_tok[0] == 'git_oss_cache_dir': git_oss_cache_dir = line_tok[1] From 0385791ab0a8d22c46e9d91ae3ef08b8c077f6d4 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Tue, 18 Feb 2025 18:08:02 +0800 Subject: [PATCH 19/51] prevant to include wrong tensorflow_probability version --- .gitignore | 3 +++ pai_jobs/deploy_ext.sh | 33 +++++++++++++++++---------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 2d0fa8010..227b49663 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ pai_jobs/easy_rec*.tar.gz # unit test /data /UNIT_TEST_CASE_LIST + +.DS_Store +.python-version diff --git a/pai_jobs/deploy_ext.sh b/pai_jobs/deploy_ext.sh index 3c8383439..ff17f1760 100755 --- a/pai_jobs/deploy_ext.sh +++ b/pai_jobs/deploy_ext.sh @@ -143,26 +143,27 @@ then rm -rf faiss.tar.gz fi -if [ ! -d "tensorflow_probability" ] +if [ -d "tensorflow_probability" ] then - if [ $is_tf15 -gt 0 ]; then - tfp_version='0.8.0' - else - tfp_version='0.5.0' - fi - if [ ! -e "tensorflow_probability" ] + rm -rf tensorflow_probability +fi +if [ $is_tf15 -gt 0 ]; then + tfp_version='0.8.0' +else + tfp_version='0.5.0' +fi +if [ ! -e "tensorflow_probability" ] +then + wget http://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/probability-${tfp_version}.tar.gz + if [ $? -ne 0 ] then - wget http://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/probability-${tfp_version}.tar.gz - if [ $? -ne 0 ] - then - echo "tensorflow_probability download failed." - fi + echo "tensorflow_probability download failed." fi - tar -xzvf probability-${tfp_version}.tar.gz --strip-components=1 probability-${tfp_version}/tensorflow_probability - rm -rf tensorflow_probability/examples - rm -rf tensorflow_probability/g3doc - rm -rf probability-${tfp_version}.tar.gz fi +tar -xzvf probability-${tfp_version}.tar.gz --strip-components=1 probability-${tfp_version}/tensorflow_probability +rm -rf tensorflow_probability/examples +rm -rf tensorflow_probability/g3doc +rm -rf probability-${tfp_version}.tar.gz tar -cvzhf $RES_PATH easy_rec datahub lz4 cprotobuf kafka faiss tensorflow_probability run.py From 8f13ba47bcff44d8114c961f85138df6db9b829c Mon Sep 17 00:00:00 2001 From: yangxudong Date: Tue, 13 May 2025 16:23:06 +0800 Subject: [PATCH 20/51] add some examples for backbone components --- docs/source/component/backbone.md | 17 ++++++++++++++ docs/source/component/sequence.md | 37 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/docs/source/component/backbone.md b/docs/source/component/backbone.md index de77f85ec..0e600dbb8 100644 --- a/docs/source/component/backbone.md +++ b/docs/source/component/backbone.md @@ -1294,6 +1294,23 @@ message InputLayer { - `wide_output_dim` wide模型每个特征的参数权重维度，一般设定为1 - `concat_seq_feature` 是否需要把序列特征的embedding拼接在一起 +比如同一个group的所有特征 pooling 为一个特征的示例如下： + +- 前提条件：这个group内的所有特征的在embedding_dim都相同 + +```protobuf +blocks { + name: 'feat_pooling' + inputs { + feature_group_name: 'feat_group' + } + input_layer { + only_output_3d_tensor: true + } + extra_input_fn: 'lambda x: tf.reduce_sum(x, axis=1)' +} +``` + ## 3. Lambda组件块 `Lambda组件块`可以配置一个lambda函数，执行一些较简单的操作。示例如下： diff --git a/docs/source/component/sequence.md b/docs/source/component/sequence.md index ddcbace7d..c048ed74a 100644 --- a/docs/source/component/sequence.md +++ b/docs/source/component/sequence.md @@ -73,6 +73,43 @@ blocks { } ``` +## 对输入序列pooling的例子 + +```protobuf +model_config: { + model_name: 'SumPooling' + model_class: 'RankModel + ... + feature_groups: { + group_name: 'sequence' + feature_names: "tag_category_list" + feature_names: "tag_brand_list" + wide_deep: DEEP + } + backbone { + + blocks { + name: 'seq_input' + inputs { + feature_group_name: 'sequence' + } + input_layer { + output_seq_and_normal_feature: true + } + } + blocks { + name: 'sum_pooling' + inputs { + block_name: 'seq_input' + input_slice: '[0]' + } + extra_input_fn: 'lambda x: tf.reduce_sum(x, axis=1)' + } + ... + } +} +``` + ## 完整的例子 - [DIN](../models/din.md) From 55d6a87f91f2b13b0e74e621939f4f23174aa2e4 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Wed, 28 May 2025 17:28:35 +0800 Subject: [PATCH 21/51] fix bug of gfile compatibility --- easy_rec/python/core/metrics.py | 6 +++++- easy_rec/python/core/sampler.py | 8 +++++--- easy_rec/python/export.py | 6 ++++-- easy_rec/python/input/criteo_input.py | 5 ++++- easy_rec/python/input/datahub_input.py | 6 ++++-- easy_rec/python/input/kafka_input.py | 6 ++++-- easy_rec/python/input/odps_rtp_input_v2.py | 7 +++++-- easy_rec/python/input/parquet_input.py | 11 ++++------- easy_rec/python/main.py | 5 ++++- easy_rec/python/test/hpo_test.py | 2 +- easy_rec/python/test/train_eval_test.py | 6 ++++-- easy_rec/python/train_eval.py | 6 ++++-- 12 files changed, 48 insertions(+), 26 deletions(-) diff --git a/easy_rec/python/core/metrics.py b/easy_rec/python/core/metrics.py index bd7cb0976..769901199 100644 --- a/easy_rec/python/core/metrics.py +++ b/easy_rec/python/core/metrics.py @@ -7,6 +7,10 @@ import numpy as np import tensorflow as tf +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tf.io.gfile as gfile from sklearn import metrics as sklearn_metrics from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops @@ -184,7 +188,7 @@ def _distribute_separated_auc_impl(labels, cur_task_index, task_num = get_task_index_and_num() cur_work_device = 'job_' + cur_job_name + '__' + 'task_' + str(cur_task_index) eval_tmp_results_dir = os.environ['eval_tmp_results_dir'] - assert tf.gfile.IsDirectory( + assert gfile.IsDirectory( eval_tmp_results_dir), 'eval_tmp_results_dir not exists' def update_pyfunc(labels, predictions, keys): diff --git a/easy_rec/python/core/sampler.py b/easy_rec/python/core/sampler.py index bfa3fd542..e8cb49403 100644 --- a/easy_rec/python/core/sampler.py +++ b/easy_rec/python/core/sampler.py @@ -8,13 +8,15 @@ import math import os import sys -# import re import threading import numpy as np import six import tensorflow as tf - +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tf.io.gfile as gfile from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils import ds_util from easy_rec.python.utils.config_util import process_multi_file_input_path @@ -395,7 +397,7 @@ def _load_data(self, data_path, attr_delimiter): item_id_col = 0 fea_id_col = 2 print('NegativeSamplerInMemory: load sample feature from %s' % data_path) - with tf.gfile.GFile(data_path, 'r') as fin: + with gfile.GFile(data_path, 'r') as fin: for line_id, line_str in enumerate(fin): line_str = line_str.strip() cols = line_str.split('\t') diff --git a/easy_rec/python/export.py b/easy_rec/python/export.py index b5d42e65e..bc356b29b 100644 --- a/easy_rec/python/export.py +++ b/easy_rec/python/export.py @@ -5,8 +5,10 @@ import tensorflow as tf from tensorflow.python.lib.io import file_io -from tensorflow.python.platform import gfile - +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tf.io.gfile as gfile from easy_rec.python.main import export from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import config_util diff --git a/easy_rec/python/input/criteo_input.py b/easy_rec/python/input/criteo_input.py index 6b65615f2..d09c67f01 100644 --- a/easy_rec/python/input/criteo_input.py +++ b/easy_rec/python/input/criteo_input.py @@ -3,7 +3,10 @@ import logging import tensorflow as tf -from tensorflow.python.platform import gfile +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tf.io.gfile as gfile from easy_rec.python.input.criteo_binary_reader import BinaryDataset from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/datahub_input.py b/easy_rec/python/input/datahub_input.py index 786d48d83..6b9cf98df 100644 --- a/easy_rec/python/input/datahub_input.py +++ b/easy_rec/python/input/datahub_input.py @@ -5,9 +5,11 @@ import traceback import tensorflow as tf +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tf.io.gfile as gfile from tensorflow.python.framework import dtypes -from tensorflow.python.platform import gfile - from easy_rec.python.input.input import Input from easy_rec.python.utils import odps_util from easy_rec.python.utils.config_util import parse_time diff --git a/easy_rec/python/input/kafka_input.py b/easy_rec/python/input/kafka_input.py index 50f516756..e5b654da7 100644 --- a/easy_rec/python/input/kafka_input.py +++ b/easy_rec/python/input/kafka_input.py @@ -6,8 +6,10 @@ import six import tensorflow as tf -from tensorflow.python.platform import gfile - +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tf.io.gfile as gfile from easy_rec.python.input.input import Input from easy_rec.python.input.kafka_dataset import KafkaDataset from easy_rec.python.utils.config_util import parse_time diff --git a/easy_rec/python/input/odps_rtp_input_v2.py b/easy_rec/python/input/odps_rtp_input_v2.py index c3dde6657..d0fb6b1f0 100644 --- a/easy_rec/python/input/odps_rtp_input_v2.py +++ b/easy_rec/python/input/odps_rtp_input_v2.py @@ -6,7 +6,10 @@ import tensorflow as tf from easy_rec.python.input.odps_rtp_input import OdpsRTPInput - +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tf.io.gfile as gfile try: import pai import rtp_fg @@ -45,7 +48,7 @@ def __init__(self, logging.info('fg config path: {}'.format(self._fg_config_path)) if self._fg_config_path is None: raise ValueError('fg_json_path is not set') - with tf.gfile.GFile(self._fg_config_path, 'r') as f: + with gfile.GFile(self._fg_config_path, 'r') as f: self._fg_config = json.load(f) def _parse_table(self, *fields): diff --git a/easy_rec/python/input/parquet_input.py b/easy_rec/python/input/parquet_input.py index f98611284..1a418bab6 100644 --- a/easy_rec/python/input/parquet_input.py +++ b/easy_rec/python/input/parquet_input.py @@ -3,17 +3,14 @@ import logging import multiprocessing import queue -# import threading import time -# import numpy as np -# import pandas as pd import tensorflow as tf -# from tensorflow.python.framework import ops +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tf.io.gfile as gfile from tensorflow.python.ops import array_ops -# from tensorflow.python.ops import logging_ops -# from tensorflow.python.ops import math_ops -from tensorflow.python.platform import gfile from easy_rec.python.compat import queues from easy_rec.python.input import load_parquet diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index 01c088a54..f17b19a59 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -13,8 +13,11 @@ import six import tensorflow as tf +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tf.io.gfile as gfile from tensorflow.core.protobuf import saved_model_pb2 -from tensorflow.python.platform import gfile import easy_rec from easy_rec.python.builders import strategy_builder diff --git a/easy_rec/python/test/hpo_test.py b/easy_rec/python/test/hpo_test.py index 75752435f..794c9ef58 100644 --- a/easy_rec/python/test/hpo_test.py +++ b/easy_rec/python/test/hpo_test.py @@ -15,7 +15,7 @@ from easy_rec.python.utils import test_utils if tf.__version__ >= '2.0': - gfile = tf.compat.v1.gfile + gfile = tf.io.gfile from tensorflow.core.protobuf import config_pb2 ConfigProto = config_pb2.ConfigProto diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 72d1eaa5f..99bfc2f48 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -12,8 +12,10 @@ import six import tensorflow as tf from distutils.version import LooseVersion -from tensorflow.python.platform import gfile - +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tf.io.gfile as gfile from easy_rec.python.main import predict from easy_rec.python.utils import config_util from easy_rec.python.utils import constant diff --git a/easy_rec/python/train_eval.py b/easy_rec/python/train_eval.py index 085884d1c..0adf73ea0 100644 --- a/easy_rec/python/train_eval.py +++ b/easy_rec/python/train_eval.py @@ -6,8 +6,10 @@ import os import tensorflow as tf -from tensorflow.python.platform import gfile - +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tf.io.gfile as gfile from easy_rec.python.main import _train_and_evaluate_impl from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import config_util From a3efa8505b573fea6c24914caf8d30636b3bc4e1 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Wed, 28 May 2025 18:11:37 +0800 Subject: [PATCH 22/51] fix bug of gfile compatibility --- easy_rec/python/core/metrics.py | 2 +- easy_rec/python/core/sampler.py | 2 +- easy_rec/python/export.py | 2 +- easy_rec/python/input/criteo_input.py | 2 +- easy_rec/python/input/kafka_input.py | 2 +- easy_rec/python/input/odps_rtp_input_v2.py | 2 +- easy_rec/python/input/parquet_input.py | 2 +- easy_rec/python/main.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/easy_rec/python/core/metrics.py b/easy_rec/python/core/metrics.py index 769901199..9f0a9c756 100644 --- a/easy_rec/python/core/metrics.py +++ b/easy_rec/python/core/metrics.py @@ -10,7 +10,7 @@ if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: - import tf.io.gfile as gfile + import tensorflow.io.gfile as gfile from sklearn import metrics as sklearn_metrics from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops diff --git a/easy_rec/python/core/sampler.py b/easy_rec/python/core/sampler.py index e8cb49403..fe4a0c52f 100644 --- a/easy_rec/python/core/sampler.py +++ b/easy_rec/python/core/sampler.py @@ -16,7 +16,7 @@ if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: - import tf.io.gfile as gfile + import tensorflow.io.gfile as gfile from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils import ds_util from easy_rec.python.utils.config_util import process_multi_file_input_path diff --git a/easy_rec/python/export.py b/easy_rec/python/export.py index bc356b29b..081342e51 100644 --- a/easy_rec/python/export.py +++ b/easy_rec/python/export.py @@ -8,7 +8,7 @@ if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: - import tf.io.gfile as gfile + import tensorflow.io.gfile as gfile from easy_rec.python.main import export from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import config_util diff --git a/easy_rec/python/input/criteo_input.py b/easy_rec/python/input/criteo_input.py index d09c67f01..bb28bb6a7 100644 --- a/easy_rec/python/input/criteo_input.py +++ b/easy_rec/python/input/criteo_input.py @@ -6,7 +6,7 @@ if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: - import tf.io.gfile as gfile + import tensorflow.io.gfile as gfile from easy_rec.python.input.criteo_binary_reader import BinaryDataset from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/kafka_input.py b/easy_rec/python/input/kafka_input.py index e5b654da7..c7938efb4 100644 --- a/easy_rec/python/input/kafka_input.py +++ b/easy_rec/python/input/kafka_input.py @@ -9,7 +9,7 @@ if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: - import tf.io.gfile as gfile + import tensorflow.io.gfile as gfile from easy_rec.python.input.input import Input from easy_rec.python.input.kafka_dataset import KafkaDataset from easy_rec.python.utils.config_util import parse_time diff --git a/easy_rec/python/input/odps_rtp_input_v2.py b/easy_rec/python/input/odps_rtp_input_v2.py index d0fb6b1f0..a51ca95f2 100644 --- a/easy_rec/python/input/odps_rtp_input_v2.py +++ b/easy_rec/python/input/odps_rtp_input_v2.py @@ -9,7 +9,7 @@ if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: - import tf.io.gfile as gfile + import tensorflow.io.gfile as gfile try: import pai import rtp_fg diff --git a/easy_rec/python/input/parquet_input.py b/easy_rec/python/input/parquet_input.py index 1a418bab6..e94d4a1ab 100644 --- a/easy_rec/python/input/parquet_input.py +++ b/easy_rec/python/input/parquet_input.py @@ -9,7 +9,7 @@ if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: - import tf.io.gfile as gfile + import tensorflow.io.gfile as gfile from tensorflow.python.ops import array_ops from easy_rec.python.compat import queues diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index f17b19a59..2119f3e1c 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -16,7 +16,7 @@ if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: - import tf.io.gfile as gfile + import tensorflow.io.gfile as gfile from tensorflow.core.protobuf import saved_model_pb2 import easy_rec From 20a391be71e1f4225bda05b665e9b1276def4a1f Mon Sep 17 00:00:00 2001 From: yangxudong Date: Wed, 28 May 2025 18:14:36 +0800 Subject: [PATCH 23/51] fix bug of gfile compatibility --- easy_rec/python/input/datahub_input.py | 2 +- easy_rec/python/test/train_eval_test.py | 2 +- easy_rec/python/train_eval.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easy_rec/python/input/datahub_input.py b/easy_rec/python/input/datahub_input.py index 6b9cf98df..7fe0dea98 100644 --- a/easy_rec/python/input/datahub_input.py +++ b/easy_rec/python/input/datahub_input.py @@ -8,7 +8,7 @@ if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: - import tf.io.gfile as gfile + import tensorflow.io.gfile as gfile from tensorflow.python.framework import dtypes from easy_rec.python.input.input import Input from easy_rec.python.utils import odps_util diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 99bfc2f48..8fa3070fc 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -15,7 +15,7 @@ if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: - import tf.io.gfile as gfile + import tensorflow.io.gfile as gfile from easy_rec.python.main import predict from easy_rec.python.utils import config_util from easy_rec.python.utils import constant diff --git a/easy_rec/python/train_eval.py b/easy_rec/python/train_eval.py index 0adf73ea0..2bd5eb102 100644 --- a/easy_rec/python/train_eval.py +++ b/easy_rec/python/train_eval.py @@ -9,7 +9,7 @@ if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: - import tf.io.gfile as gfile + import tensorflow.io.gfile as gfile from easy_rec.python.main import _train_and_evaluate_impl from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import config_util From 38fa46b5fa4c286598f2c1c0699563caf7a5ea0d Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 29 May 2025 14:23:47 +0800 Subject: [PATCH 24/51] fix bug of gfile compatibility --- easy_rec/python/main.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index 2119f3e1c..dad813124 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -68,6 +68,16 @@ BestExporter = exporter.BestExporter +def is_directory(path): + if not gfile.exists(path): + return False + try: + gfile.listdir(path) + return True + except: + return False + + def _get_input_fn(data_config, feature_configs, data_path=None, @@ -243,7 +253,7 @@ def _metric_cmp_fn(best_eval_result, current_eval_result): def _check_model_dir(model_dir, continue_train): if not continue_train: - if not gfile.IsDirectory(model_dir): + if not is_directory(model_dir): gfile.MakeDirs(model_dir) else: assert len(gfile.Glob(model_dir + '/model.ckpt-*.meta')) == 0, \ @@ -252,18 +262,18 @@ def _check_model_dir(model_dir, continue_train): 'delete dir %s or specify --continue_train[internal use only])' % ( model_dir, model_dir) else: - if not gfile.IsDirectory(model_dir): + if not is_directory(model_dir): logging.info('%s does not exists, create it automatically' % model_dir) gfile.MakeDirs(model_dir) def _get_ckpt_path(pipeline_config, checkpoint_path): if checkpoint_path != '' and checkpoint_path is not None: - if gfile.IsDirectory(checkpoint_path): + if is_directory(checkpoint_path): ckpt_path = estimator_utils.latest_checkpoint(checkpoint_path) else: ckpt_path = checkpoint_path - elif gfile.IsDirectory(pipeline_config.model_dir): + elif is_directory(pipeline_config.model_dir): ckpt_path = estimator_utils.latest_checkpoint(pipeline_config.model_dir) logging.info('checkpoint_path is not specified, ' 'will use latest checkpoint %s from %s' % @@ -565,11 +575,10 @@ def distribute_evaluate(pipeline_config, return eval_result model_dir = get_model_dir_path(pipeline_config) eval_tmp_results_dir = os.path.join(model_dir, 'distribute_eval_tmp_results') - if not gfile.IsDirectory(eval_tmp_results_dir): + if not is_directory(eval_tmp_results_dir): logging.info('create eval tmp results dir {}'.format(eval_tmp_results_dir)) gfile.MakeDirs(eval_tmp_results_dir) - assert gfile.IsDirectory( - eval_tmp_results_dir), 'tmp results dir not create success.' + assert is_directory(eval_tmp_results_dir), 'tmp results dir not create success.' os.environ['eval_tmp_results_dir'] = eval_tmp_results_dir server_target = None From f7a62e2f2fa7fe8211a51ce1130acdc0fbbb1467 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 29 May 2025 15:36:32 +0800 Subject: [PATCH 25/51] fix bug of gfile compatibility --- easy_rec/python/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index dad813124..90b6cf24c 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -69,6 +69,8 @@ def is_directory(path): + if tf.__version__.startswith('1.'): + return gfile.IsDirectory(path) if not gfile.exists(path): return False try: From 8e8e3c7cd7668ff78ece6317082e6be4358f3ef1 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 29 May 2025 15:41:17 +0800 Subject: [PATCH 26/51] fix bug of gfile compatibility --- easy_rec/python/test/hpo_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easy_rec/python/test/hpo_test.py b/easy_rec/python/test/hpo_test.py index 794c9ef58..a570d87d0 100644 --- a/easy_rec/python/test/hpo_test.py +++ b/easy_rec/python/test/hpo_test.py @@ -15,13 +15,13 @@ from easy_rec.python.utils import test_utils if tf.__version__ >= '2.0': - gfile = tf.io.gfile + import tensorflow.io.gfile as gfile from tensorflow.core.protobuf import config_pb2 ConfigProto = config_pb2.ConfigProto GPUOptions = config_pb2.GPUOptions else: - gfile = tf.gfile + from tensorflow.python.platform import gfile GPUOptions = tf.GPUOptions ConfigProto = tf.ConfigProto From dcc0f4a2e60a390adadc9e07ddf7407287fe75bc Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 29 May 2025 16:46:50 +0800 Subject: [PATCH 27/51] fix bug of gfile compatibility --- easy_rec/python/core/metrics.py | 9 ++-- easy_rec/python/core/sampler.py | 10 ++-- easy_rec/python/export.py | 10 ++-- easy_rec/python/input/criteo_input.py | 7 +-- easy_rec/python/input/datahub_input.py | 10 ++-- easy_rec/python/input/kafka_input.py | 8 ++-- easy_rec/python/input/odps_rtp_input_v2.py | 1 + easy_rec/python/input/parquet_input.py | 9 ++-- easy_rec/python/main.py | 54 ++++++++-------------- easy_rec/python/test/train_eval_test.py | 10 ++-- easy_rec/python/train_eval.py | 10 ++-- 11 files changed, 70 insertions(+), 68 deletions(-) diff --git a/easy_rec/python/core/metrics.py b/easy_rec/python/core/metrics.py index 9f0a9c756..fa5ab022b 100644 --- a/easy_rec/python/core/metrics.py +++ b/easy_rec/python/core/metrics.py @@ -7,10 +7,6 @@ import numpy as np import tensorflow as tf -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile from sklearn import metrics as sklearn_metrics from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops @@ -22,6 +18,11 @@ from easy_rec.python.utils.io_util import save_data_to_json_path from easy_rec.python.utils.shape_utils import get_shape_list +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tensorflow.io.gfile as gfile + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/core/sampler.py b/easy_rec/python/core/sampler.py index fe4a0c52f..dcdcd44f0 100644 --- a/easy_rec/python/core/sampler.py +++ b/easy_rec/python/core/sampler.py @@ -13,15 +13,17 @@ import numpy as np import six import tensorflow as tf -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile + from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils import ds_util from easy_rec.python.utils.config_util import process_multi_file_input_path from easy_rec.python.utils.tf_utils import get_tf_type +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tensorflow.io.gfile as gfile + # patch graph-learn string_attrs for utf-8 @property diff --git a/easy_rec/python/export.py b/easy_rec/python/export.py index 081342e51..c1a8ce670 100644 --- a/easy_rec/python/export.py +++ b/easy_rec/python/export.py @@ -5,15 +5,17 @@ import tensorflow as tf from tensorflow.python.lib.io import file_io -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile + from easy_rec.python.main import export from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import config_util from easy_rec.python.utils import estimator_utils +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tensorflow.io.gfile as gfile + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/input/criteo_input.py b/easy_rec/python/input/criteo_input.py index bb28bb6a7..f43ceb5a2 100644 --- a/easy_rec/python/input/criteo_input.py +++ b/easy_rec/python/input/criteo_input.py @@ -3,14 +3,15 @@ import logging import tensorflow as tf + +from easy_rec.python.input.criteo_binary_reader import BinaryDataset +from easy_rec.python.input.input import Input + if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: import tensorflow.io.gfile as gfile -from easy_rec.python.input.criteo_binary_reader import BinaryDataset -from easy_rec.python.input.input import Input - if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/input/datahub_input.py b/easy_rec/python/input/datahub_input.py index 7fe0dea98..37e3292d4 100644 --- a/easy_rec/python/input/datahub_input.py +++ b/easy_rec/python/input/datahub_input.py @@ -5,15 +5,17 @@ import traceback import tensorflow as tf -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile from tensorflow.python.framework import dtypes + from easy_rec.python.input.input import Input from easy_rec.python.utils import odps_util from easy_rec.python.utils.config_util import parse_time +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tensorflow.io.gfile as gfile + try: import common_io except Exception: diff --git a/easy_rec/python/input/kafka_input.py b/easy_rec/python/input/kafka_input.py index c7938efb4..38eab121f 100644 --- a/easy_rec/python/input/kafka_input.py +++ b/easy_rec/python/input/kafka_input.py @@ -6,13 +6,15 @@ import six import tensorflow as tf + +from easy_rec.python.input.input import Input +from easy_rec.python.input.kafka_dataset import KafkaDataset +from easy_rec.python.utils.config_util import parse_time + if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: import tensorflow.io.gfile as gfile -from easy_rec.python.input.input import Input -from easy_rec.python.input.kafka_dataset import KafkaDataset -from easy_rec.python.utils.config_util import parse_time try: from kafka import KafkaConsumer, TopicPartition diff --git a/easy_rec/python/input/odps_rtp_input_v2.py b/easy_rec/python/input/odps_rtp_input_v2.py index a51ca95f2..77edb46c1 100644 --- a/easy_rec/python/input/odps_rtp_input_v2.py +++ b/easy_rec/python/input/odps_rtp_input_v2.py @@ -6,6 +6,7 @@ import tensorflow as tf from easy_rec.python.input.odps_rtp_input import OdpsRTPInput + if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: diff --git a/easy_rec/python/input/parquet_input.py b/easy_rec/python/input/parquet_input.py index e94d4a1ab..c1cb41190 100644 --- a/easy_rec/python/input/parquet_input.py +++ b/easy_rec/python/input/parquet_input.py @@ -6,16 +6,17 @@ import time import tensorflow as tf -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile from tensorflow.python.ops import array_ops from easy_rec.python.compat import queues from easy_rec.python.input import load_parquet from easy_rec.python.input.input import Input +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tensorflow.io.gfile as gfile + class ParquetInput(Input): diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index 90b6cf24c..1747155fe 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -13,10 +13,6 @@ import six import tensorflow as tf -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile from tensorflow.core.protobuf import saved_model_pb2 import easy_rec @@ -68,18 +64,6 @@ BestExporter = exporter.BestExporter -def is_directory(path): - if tf.__version__.startswith('1.'): - return gfile.IsDirectory(path) - if not gfile.exists(path): - return False - try: - gfile.listdir(path) - return True - except: - return False - - def _get_input_fn(data_config, feature_configs, data_path=None, @@ -255,27 +239,27 @@ def _metric_cmp_fn(best_eval_result, current_eval_result): def _check_model_dir(model_dir, continue_train): if not continue_train: - if not is_directory(model_dir): - gfile.MakeDirs(model_dir) + if not tf.gfile.IsDirectory(model_dir): + tf.gfile.MakeDirs(model_dir) else: - assert len(gfile.Glob(model_dir + '/model.ckpt-*.meta')) == 0, \ + assert len(tf.gfile.Glob(model_dir + '/model.ckpt-*.meta')) == 0, \ 'model_dir[=%s] already exists and not empty(if you ' \ 'want to continue train on current model_dir please ' \ 'delete dir %s or specify --continue_train[internal use only])' % ( model_dir, model_dir) else: - if not is_directory(model_dir): + if not tf.gfile.IsDirectory(model_dir): logging.info('%s does not exists, create it automatically' % model_dir) - gfile.MakeDirs(model_dir) + tf.gfile.MakeDirs(model_dir) def _get_ckpt_path(pipeline_config, checkpoint_path): if checkpoint_path != '' and checkpoint_path is not None: - if is_directory(checkpoint_path): + if tf.gfile.IsDirectory(checkpoint_path): ckpt_path = estimator_utils.latest_checkpoint(checkpoint_path) else: ckpt_path = checkpoint_path - elif is_directory(pipeline_config.model_dir): + elif tf.gfile.IsDirectory(pipeline_config.model_dir): ckpt_path = estimator_utils.latest_checkpoint(pipeline_config.model_dir) logging.info('checkpoint_path is not specified, ' 'will use latest checkpoint %s from %s' % @@ -299,7 +283,8 @@ def train_and_evaluate(pipeline_config_path, continue_train=False): Returns: None, the model will be saved into pipeline_config.model_dir """ - assert gfile.Exists(pipeline_config_path), 'pipeline_config_path not exists' + assert tf.gfile.Exists( + pipeline_config_path), 'pipeline_config_path not exists' pipeline_config = config_util.get_configs_from_pipeline_file( pipeline_config_path) @@ -338,7 +323,7 @@ def _train_and_evaluate_impl(pipeline_config, if estimator_utils.is_chief(): _check_model_dir(pipeline_config.model_dir, continue_train) config_util.save_pipeline_config(pipeline_config, pipeline_config.model_dir) - with gfile.GFile(version_file, 'w') as f: + with tf.gfile.GFile(version_file, 'w') as f: f.write(easy_rec.__version__ + '\n') train_steps = None @@ -524,7 +509,7 @@ def evaluate(pipeline_config, model_dir = pipeline_config.model_dir eval_result_file = os.path.join(model_dir, eval_result_filename) logging.info('save eval result to file %s' % eval_result_file) - with gfile.GFile(eval_result_file, 'w') as ofile: + with tf.gfile.GFile(eval_result_file, 'w') as ofile: result_to_write = {} for key in sorted(eval_result): # skip logging binary data @@ -577,10 +562,11 @@ def distribute_evaluate(pipeline_config, return eval_result model_dir = get_model_dir_path(pipeline_config) eval_tmp_results_dir = os.path.join(model_dir, 'distribute_eval_tmp_results') - if not is_directory(eval_tmp_results_dir): + if not tf.gfile.IsDirectory(eval_tmp_results_dir): logging.info('create eval tmp results dir {}'.format(eval_tmp_results_dir)) - gfile.MakeDirs(eval_tmp_results_dir) - assert is_directory(eval_tmp_results_dir), 'tmp results dir not create success.' + tf.gfile.MakeDirs(eval_tmp_results_dir) + assert tf.gfile.IsDirectory( + eval_tmp_results_dir), 'tmp results dir not create success.' os.environ['eval_tmp_results_dir'] = eval_tmp_results_dir server_target = None @@ -693,7 +679,7 @@ def distribute_evaluate(pipeline_config, if cur_job_name == 'master': print('eval_result = ', eval_result) logging.info('eval_result = {0}'.format(eval_result)) - with gfile.GFile(eval_result_file, 'w') as ofile: + with tf.gfile.GFile(eval_result_file, 'w') as ofile: result_to_write = {'eval_method': 'distribute'} for key in sorted(eval_result): # skip logging binary data @@ -780,8 +766,8 @@ def export(export_dir, AssertionError, if: * pipeline_config_path does not exist """ - if not gfile.Exists(export_dir): - gfile.MakeDirs(export_dir) + if not tf.gfile.Exists(export_dir): + tf.gfile.MakeDirs(export_dir) pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config) if pipeline_config.fg_json_path: @@ -844,10 +830,10 @@ def export(export_dir, ] export_ts = export_ts[-1] saved_pb_path = os.path.join(final_export_dir, 'saved_model.pb') - with gfile.GFile(saved_pb_path, 'rb') as fin: + with tf.gfile.GFile(saved_pb_path, 'rb') as fin: saved_model.ParseFromString(fin.read()) saved_model.meta_graphs[0].meta_info_def.meta_graph_version = export_ts - with gfile.GFile(saved_pb_path, 'wb') as fout: + with tf.gfile.GFile(saved_pb_path, 'wb') as fout: fout.write(saved_model.SerializeToString()) logging.info('model has been exported to %s successfully' % final_export_dir) diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 8fa3070fc..f1898b2b2 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -12,16 +12,18 @@ import six import tensorflow as tf from distutils.version import LooseVersion -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile + from easy_rec.python.main import predict from easy_rec.python.utils import config_util from easy_rec.python.utils import constant from easy_rec.python.utils import estimator_utils from easy_rec.python.utils import test_utils +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tensorflow.io.gfile as gfile + try: import graphlearn as gl except Exception: diff --git a/easy_rec/python/train_eval.py b/easy_rec/python/train_eval.py index 2bd5eb102..bafdf0c1a 100644 --- a/easy_rec/python/train_eval.py +++ b/easy_rec/python/train_eval.py @@ -6,10 +6,7 @@ import os import tensorflow as tf -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile + from easy_rec.python.main import _train_and_evaluate_impl from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import config_util @@ -21,6 +18,11 @@ from easy_rec.python.utils.config_util import set_eval_input_path from easy_rec.python.utils.config_util import set_train_input_path +if tf.__version__.startswith('1.'): + from tensorflow.python.platform import gfile +else: + import tensorflow.io.gfile as gfile + from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_train_worker_num_on_ds # NOQA if tf.__version__ >= '2.0': From cf6a4c58d5aa391033c38e945b7fcbe231170cf2 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 29 May 2025 18:47:50 +0800 Subject: [PATCH 28/51] fix bug of gfile compatibility --- easy_rec/python/test/train_eval_test.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index f1898b2b2..aca203efe 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -19,11 +19,6 @@ from easy_rec.python.utils import estimator_utils from easy_rec.python.utils import test_utils -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile - try: import graphlearn as gl except Exception: From d6bd2b6d42739124bc822f9383175719be01c838 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 29 May 2025 18:49:23 +0800 Subject: [PATCH 29/51] fix bug of gfile compatibility --- easy_rec/python/test/train_eval_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index aca203efe..72d1eaa5f 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -12,6 +12,7 @@ import six import tensorflow as tf from distutils.version import LooseVersion +from tensorflow.python.platform import gfile from easy_rec.python.main import predict from easy_rec.python.utils import config_util From cb37281dc6002bbe4ab15ca1cd28a27d59659dd4 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Tue, 3 Jun 2025 22:03:37 +0800 Subject: [PATCH 30/51] fix bug of gfile compatibility --- easy_rec/python/core/metrics.py | 5 ----- easy_rec/python/input/criteo_input.py | 11 +++-------- easy_rec/python/input/parquet_input.py | 8 +++----- easy_rec/python/input/parquet_input_v3.py | 7 ++++--- 4 files changed, 10 insertions(+), 21 deletions(-) diff --git a/easy_rec/python/core/metrics.py b/easy_rec/python/core/metrics.py index fa5ab022b..97496c333 100644 --- a/easy_rec/python/core/metrics.py +++ b/easy_rec/python/core/metrics.py @@ -18,11 +18,6 @@ from easy_rec.python.utils.io_util import save_data_to_json_path from easy_rec.python.utils.shape_utils import get_shape_list -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile - if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/input/criteo_input.py b/easy_rec/python/input/criteo_input.py index f43ceb5a2..0eb3ee595 100644 --- a/easy_rec/python/input/criteo_input.py +++ b/easy_rec/python/input/criteo_input.py @@ -7,11 +7,6 @@ from easy_rec.python.input.criteo_binary_reader import BinaryDataset from easy_rec.python.input.input import Input -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile - if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -42,9 +37,9 @@ def __init__(self, for label_path, dense_path, category_path in zip( input_path.label_path, input_path.dense_path, input_path.category_path): - label_paths = gfile.Glob(input_path.label_path) - dense_paths = gfile.Glob(input_path.dense_path) - category_paths = gfile.Glob(input_path.category_path) + label_paths = tf.gfile.Glob(input_path.label_path) + dense_paths = tf.gfile.Glob(input_path.dense_path) + category_paths = tf.gfile.Glob(input_path.category_path) assert len(label_paths) == len(dense_paths) and len(label_paths) == \ len(category_paths), 'label_path(%s) dense_path(%s) category_path(%s) ' + \ 'matched different number of files(%d %d %d)' % ( diff --git a/easy_rec/python/input/parquet_input.py b/easy_rec/python/input/parquet_input.py index c1cb41190..dcc6e867f 100644 --- a/easy_rec/python/input/parquet_input.py +++ b/easy_rec/python/input/parquet_input.py @@ -12,10 +12,8 @@ from easy_rec.python.input import load_parquet from easy_rec.python.input.input import Input -if tf.__version__.startswith('1.'): - from tensorflow.python.platform import gfile -else: - import tensorflow.io.gfile as gfile +if tf.__version__ >= '2.0': + tf = tf.compat.v1 class ParquetInput(Input): @@ -38,7 +36,7 @@ def __init__(self, self._input_files = [] for sub_path in input_path.strip().split(','): - self._input_files.extend(gfile.Glob(sub_path)) + self._input_files.extend(tf.gfile.Glob(sub_path)) logging.info('parquet input_path=%s file_num=%d' % (input_path, len(self._input_files))) mp_ctxt = multiprocessing.get_context('spawn') diff --git a/easy_rec/python/input/parquet_input_v3.py b/easy_rec/python/input/parquet_input_v3.py index 28f8741f0..866ec37ab 100644 --- a/easy_rec/python/input/parquet_input_v3.py +++ b/easy_rec/python/input/parquet_input_v3.py @@ -3,8 +3,6 @@ import logging import tensorflow as tf -from tensorflow.python.platform import gfile - from easy_rec.python.input.input import Input from easy_rec.python.utils.input_utils import get_type_defaults @@ -19,6 +17,9 @@ _has_deep_rec = False pass +if tf.__version__ >= '2.0': + tf = tf.compat.v1 + class ParquetInputV3(Input): @@ -114,7 +115,7 @@ def _parse_dataframe(self, df): def _build(self, mode, params): input_files = [] for sub_path in self._input_path.strip().split(','): - input_files.extend(gfile.Glob(sub_path)) + input_files.extend(tf.gfile.Glob(sub_path)) file_num = len(input_files) logging.info('[task_index=%d] total_file_num=%d task_num=%d' % (self._task_index, file_num, self._task_num)) From 90d81c90fc925269200ae991ccb5d0cef6d39fe2 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Wed, 4 Jun 2025 10:53:35 +0800 Subject: [PATCH 31/51] fix bug of gfile compatibility --- easy_rec/python/core/metrics.py | 2 +- easy_rec/python/input/parquet_input_v3.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/easy_rec/python/core/metrics.py b/easy_rec/python/core/metrics.py index 97496c333..bd7cb0976 100644 --- a/easy_rec/python/core/metrics.py +++ b/easy_rec/python/core/metrics.py @@ -184,7 +184,7 @@ def _distribute_separated_auc_impl(labels, cur_task_index, task_num = get_task_index_and_num() cur_work_device = 'job_' + cur_job_name + '__' + 'task_' + str(cur_task_index) eval_tmp_results_dir = os.environ['eval_tmp_results_dir'] - assert gfile.IsDirectory( + assert tf.gfile.IsDirectory( eval_tmp_results_dir), 'eval_tmp_results_dir not exists' def update_pyfunc(labels, predictions, keys): diff --git a/easy_rec/python/input/parquet_input_v3.py b/easy_rec/python/input/parquet_input_v3.py index 866ec37ab..300a5bd1e 100644 --- a/easy_rec/python/input/parquet_input_v3.py +++ b/easy_rec/python/input/parquet_input_v3.py @@ -3,6 +3,7 @@ import logging import tensorflow as tf + from easy_rec.python.input.input import Input from easy_rec.python.utils.input_utils import get_type_defaults From 5d61a09135f37ff3ebaee9d4f5d5334101528835 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Wed, 10 Sep 2025 11:44:49 +0800 Subject: [PATCH 32/51] update FG document --- docs/images/other/fg.svg | 1 + docs/source/index.rst | 3 +- docs/source/predict/processor.md | 16 ++++ docs/source/quick_start/local_tutorial.md | 6 +- easy_rec/python/input/odps_input_v3.py | 2 +- easy_rec/python/test/odps_input_v3_test.py | 86 ++++++++++++++++++++++ easy_rec/version.py | 2 +- 7 files changed, 111 insertions(+), 5 deletions(-) create mode 100644 docs/images/other/fg.svg create mode 100644 easy_rec/python/test/odps_input_v3_test.py diff --git a/docs/images/other/fg.svg b/docs/images/other/fg.svg new file mode 100644 index 000000000..e39f71a5a --- /dev/null +++ b/docs/images/other/fg.svg @@ -0,0 +1 @@ +
User Feature
Pipelines
[Not supported by viewer]
Feature
Generation
[Not supported by viewer]
Recommender
Service
[Not supported by viewer]
Log Msg Queue
[Not supported by viewer]
Item DB
[Not supported by viewer]
User DB
[Not supported by viewer]
Item Feature
Pipelines
[Not supported by viewer]
Feature
Generation
[Not supported by viewer]
Model Training
Pipelines
[Not supported by viewer]
User Category Preference:
football:0.9; rock-climb:0.5;
skiing:0.1; ... ...

Item Category: rock-climb


User-Category-Prefer: 0.5
[Not supported by viewer]
Ranking
Service
[Not supported by viewer]
Item Index
[Not supported by viewer]
FeatureStore
[Not supported by viewer]
Offline Data Pipelines
[Not supported by viewer] \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 10ed89920..2434ccbc0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -21,8 +21,7 @@ Welcome to easy_rec's documentation! feature/odl_sample.md feature/feature feature/excel_config - feature/rtp_fg - feature/rtp_native + feature/fg .. toctree:: :maxdepth: 3 diff --git a/docs/source/predict/processor.md b/docs/source/predict/processor.md index dabdb7aa1..213063940 100644 --- a/docs/source/predict/processor.md +++ b/docs/source/predict/processor.md @@ -107,6 +107,22 @@ FeatureGenerator作为算子嵌入, 和TFModel联合优化，主要的优化点 easyrec-1.3支持PAI-FeatureStore; 支持从max compute加载数据 easyrec-1.4优化keras model性能; input自动扩展; placement优化 easyrec-1.5graph pattern match bugfix + easyrec-1.6优化keras model性能; input自动扩展; placement优化 + easyrec-1.7fix bug; 旧featurestore最终版本 + easyrec-1.8支持云上版本 FeatureStore + easyrec-1.9修复了 tag feature 带有 kv_separator 时请求报错的问题 + easyrec-2.0修复了feature_column_pass里面unordered_map::at导致的exception + easyrec-2.1fix threadpool bug, 导致模型启动有小概率会hang住; +修复了Concat axis=-1不能被feature_tile识别的问题 + easyrec-2.2FeatureStore 修复最大字符长度的限制，支持自建存储 FeatureDB + easyrec-2.3增加statefulpartitionedcall展开功能, 适配tf2+keras导出的模型优化 + easyrec-2.41. feature store 支持 feature db; 2. feature store 支持 sts token; 3. request 支持 double (float64) 类型 + easyrec-2.5更新新版本 feature store cpp sdk, 支持 featuredb 拉取全量特征，支持设置 hologres 端口，支持默认不从远程拉取特征。 + easyrec-2.6修复 lookup feature 值为 string 时报错的问题 + easyrec-2.7支持 feature store cpp sdk 所有参数 + easyrec-2.8修复输入 text_cnn 的特征不能 tile 的 bug + easyrec-2.9集成新版fg lib 0.7.0 + easyrec-3.0集成新版fg lib 0.7.3; 修复无法解析新版fg新增算子的 bug; 优化`fg_mode=normal`模式的性能 ### 部署 diff --git a/docs/source/quick_start/local_tutorial.md b/docs/source/quick_start/local_tutorial.md index 434e3c434..72287c4f6 100644 --- a/docs/source/quick_start/local_tutorial.md +++ b/docs/source/quick_start/local_tutorial.md @@ -32,6 +32,7 @@ pip install tensorflow_probability==0.5.0 | 2.7.0 | 0.15.0 | | 2.8.0 | 0.16.0 | | 2.10 | 0.18.0 | +| 2.11 | 0.19.0 | | 2.12 | 0.20.0 | 其他版本对应关系请查看链接：[Releases · tensorflow/probability](https://github.com/tensorflow/probability/releases)。 @@ -59,9 +60,12 @@ docker exec -it bash 可选镜像： +- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-paitf1.12-0.8.5 [只能跑在DLC环境] +- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-paitf1.15-0.8.5 - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15.5-0.8.5 -- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-0.8.5 - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15.5-gpu-0.8.5 +- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py39-tf2.11-0.8.5 +- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-0.8.5 - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-gpu-0.8.5 ##### 方法二：自行构建Docker镜像 diff --git a/easy_rec/python/input/odps_input_v3.py b/easy_rec/python/input/odps_input_v3.py index a8657d25d..f16f3e5a6 100644 --- a/easy_rec/python/input/odps_input_v3.py +++ b/easy_rec/python/input/odps_input_v3.py @@ -33,7 +33,7 @@ def __init__(self, self._num_epoch = 0 if common_io is None: logging.error("""please install common_io pip install - https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.1.0-cp37-cp37m-linux_x86_64.whl""" + https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.2%2Btunnel-py2.py3-none-any.whl""" ) sys.exit(1) diff --git a/easy_rec/python/test/odps_input_v3_test.py b/easy_rec/python/test/odps_input_v3_test.py new file mode 100644 index 000000000..fba4c79a2 --- /dev/null +++ b/easy_rec/python/test/odps_input_v3_test.py @@ -0,0 +1,86 @@ +# -*- encoding:utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +"""Test odps input v3.""" + +import tensorflow as tf +from easy_rec.python.input.odps_input_v3 import OdpsInputV3 +from google.protobuf import text_format +from easy_rec.python.protos.dataset_pb2 import DatasetConfig +from easy_rec.python.protos.feature_config_pb2 import FeatureConfig + + +class OdpsInputV3Test(tf.test.TestCase): + + def __init__(self, methodName='OdpsInputV3'): + super(OdpsInputV3Test, self).__init__(methodName=methodName) + # must put .odps_config.ini in HOME directory + self._input_path = 'odps://xingqudao_saas/tables/xingqudao_dbmtl_v1_training_set_fg_encoded_v1/ds=20250526' + + def test_read_data(self): + data_config_str = """ + input_fields { + input_name: 'user_id' + input_type: STRING + } + input_fields { + input_name: 'item_id' + input_type: STRING + } + input_fields { + input_name: 'is_click' + input_type: INT64 + } + input_fields { + input_name: 'is_collect_like_comment' + input_type: INT64 + } + input_fields { + input_name: 'features' + input_type: STRING + } + label_fields: 'is_click' + batch_size: 1024 + num_epochs: 1 + prefetch_size: 1 + """ + feature_config_str = """ + input_names: 'user_id' + feature_type: IdFeature + embedding_dim: 32 + hash_bucket_size: 2000 + """ + dataset_config = DatasetConfig() + text_format.Merge(data_config_str, dataset_config) + feature_config = FeatureConfig() + text_format.Merge(feature_config_str, feature_config) + feature_configs = [feature_config] + empty_config = FeatureConfig() + empty_config.CopyFrom(feature_config) + while len(empty_config.input_names) > 0: + empty_config.input_names.pop() + for input_name in ['item_id', 'features']: + tmp_config = FeatureConfig() + tmp_config.CopyFrom(empty_config) + tmp_config.input_names.append(input_name) + feature_configs.append(tmp_config) + + train_input_fn = OdpsInputV3(dataset_config, feature_configs, + self._input_path).create_input() + dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) + iterator = dataset.make_initializable_iterator() + tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) + features, labels = iterator.get_next() + init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) + gpu_options = tf.GPUOptions(allow_growth=True) + session_config = tf.ConfigProto( + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=False) + with self.test_session(config=session_config) as sess: + sess.run(init_op) + feature_dict, label_dict = sess.run([features, labels]) + print("feature:", feature_dict, "label:", label_dict) + + +if __name__ == '__main__': + tf.test.main() diff --git a/easy_rec/version.py b/easy_rec/version.py index 759f7a8b3..fa13b61b9 100644 --- a/easy_rec/version.py +++ b/easy_rec/version.py @@ -1,4 +1,4 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -__version__ = '0.8.5' +__version__ = '0.8.6' From d7f0d77c5e4a1c9fefbd3d657e19ca6d4df4d5cc Mon Sep 17 00:00:00 2001 From: yangxudong Date: Wed, 10 Sep 2025 11:49:31 +0800 Subject: [PATCH 33/51] update FG document --- docs/source/feature/fg.md | 302 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100644 docs/source/feature/fg.md diff --git a/docs/source/feature/fg.md b/docs/source/feature/fg.md new file mode 100644 index 000000000..7d7017aba --- /dev/null +++ b/docs/source/feature/fg.md @@ -0,0 +1,302 @@ +# Feature Generator + +[特征生成（FeatureGenerator，下文简称FG）](https://help.aliyun.com/zh/airec/what-is-pai-rec/user-guide/feature-generation-overview-and-configuration) +是一套把原始输入转换为模型所需输入（特征）的数据变换过程，用来保证离线、在线样本生成结果的一致性。 +特征生成也可以理解为特征变换，对单个特征或者多个特征做变换。我们提供了各种类型的FG算子来完成各种特征变换操作。 + +特征生成只关注同时需要在离线和在线样本生成过程中的变换操作。如果某个变换操作只需要作用在离线阶段，则不需要定义为FG的操作。 + +FG模块在推荐系统架构中的位置如下图所示： + +![fg](../../images/other/fg.svg) + +特征生成过程由一系列特征变换算子（下文简称为FG算子）按照配置文件定义的DAG图的拓扑顺序并行执行。 + +- FG，解决实时预测需要的特征工程需求；它是属于模型之外的前置数据加工模块。详见[FG文档](https://help.aliyun.com/zh/airec/what-is-pai-rec/user-guide/feature-generator/)。 +- FG能够以比较高的效率生成一些复杂的交叉特征，如`expr feature`和`lookup feature`等，详见文档《[内置特征算子](https://help.aliyun.com/zh/airec/what-is-pai-rec/user-guide/built-in-feature-operator)》。 +- 其生成的特征可以接入EasyRec进行训练。 +- 线上部署的时候提供带FG功能的[EasyRec Processor](../predict/processor.md)一键部署。 +- FG模块支持复杂类型(array, map)；支持[python API](https://help.aliyun.com/zh/airec/what-is-pai-rec/user-guide/quick-start)；支持[自定义特征变换算子](https://help.aliyun.com/zh/airec/what-is-pai-rec/user-guide/custom-feature-operator)；支持分箱操作（离散化）；支持特征间互相依赖。 +- FG模块以多线程的方式执行由配置文件`fg.json`定义的特征变换DAG图。 + +### 训练 + +#### 编写配置文件`fg.json` + +- 包含了features配置和全局配置两个部分, 示例: + +```json +{ + "features": [ + {"expression": "user:user_id", "feature_name": "user_id", "feature_type":"id_feature", "value_type":"String"}, + {"expression": "user:cms_segid", "feature_name": "cms_segid", "feature_type":"id_feature", "value_type":"String"}, + ... + {"expression": "item:price", "feature_name": "price", "feature_type":"raw_feature", "value_type":"float"}, + {"expression": "item:pid", "feature_name": "pid", "feature_type":"id_feature", "value_type":"String"}, + {"expression": "user:tag_category_list", "feature_name": "user_tag_cate", "feature_type":"id_feature"}, + {"map": "user:tag_brand_list", "key":"item:brand", "feature_name": "combo_brand", "feature_type":"lookup_feature"}, + {"map": "user:tag_category_list", "key":"item:cate_id", "feature_name": "combo_cate_id", "feature_type":"lookup_feature"} + ], + + + "reserves": [ + "user_id", "campaign_id", "clk" + ] +} +``` + +- Feature配置说明请[查看文档](https://help.aliyun.com/zh/airec/what-is-pai-rec/user-guide/feature-generation-overview-and-configuration#bf3dfc9387m0l) + +#### 生成样本 + +- 详见文档：《[在MaxCompute Job中使用FG](https://help.aliyun.com/zh/airec/what-is-pai-rec/user-guide/use-fg-in-offline-tasks)》 + +#### 配置EasyRec的config + +FG支持的特征变换算子与EasyRec支持的特征（`Feature Column`）之间没有严格的对应关系，大致可以参加如下表格： + +| FG 算子 | EasyRec Feature Column | +|:-------------|:------------------------------------| +| id_feature | IdFeature 或 TagFeature | +| raw_feature | RawFeature | +| expr_feature | RawFeature | +| combo_feature | IdFeature 或 TagFeature | +| lookup_feature | RawFeature 或 IdFeature 或 TagFeature | +| match_feature | RawFeature 或 IdFeature 或 TagFeature | +| overlap_feature | RawFeature | +| sequence_feature | SequenceFeature 或 TagFeature | +| bm25_feature | RawFeature | +| kv_dot_product | RawFeature | +| tokenize_feature | TagFeature | +| text_normalizer | IdFeature | +| regex_replace_feature | IdFeature | + +备注：**FG的执行结果输出给EasyRec模型，两种之间是串联的关系**。 + +#### 启动训练 + +- 上传fg.config到oss +- 启动训练 + +```sql +pai -name easy_rec_ext +-Dversion='0.8.5' +-Dcmd=train +-Dconfig=oss://bucket-name/easy_rec_test/fg.config +-Dtrain_tables=odps://project-name/tables/taobao_fg_train_out +-Deval_tables=odps://project-name/tables/taobao_fg_test_out +-Dcluster='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}' +-Darn=acs:ram::xxx:role/ev-ext-test-oss +-Dbuckets=oss://bucket-name/ +-DossHost=oss-cn-xxx.aliyuncs.com +-Deval_method=separate; +``` + +- 参数说明: [请参考](../train.md#on-pai) + +#### 模型导出 + +```sql +pai -name easy_rec_ext + -Dversion='0.8.5' + -Dcmd=export + -Dconfig=oss://easyrec/easy_rec_test/fg.config + -Dexport_dir=oss:///export_dir + -Dbuckets=oss:/// + -Darn=acs:ram::xxx:role/aliyunodpspaidefaultrole + -DossHost=oss-hangzhou-internal.aliyuncs.com + -Dedit_config_json='{"export_config.multi_placeholder":true, "feature_config.features[:].max_partitions":1}'; + +``` + +- 参数说明: [请参考](../export.md#pai) +- 注意事项: + - 请检查fg.config, 保证导出的模型是支持多个placeholder的输入\[每个特征一个placeholder\] + + ``` + export_config { + multi_placeholder: true + } + ``` + + 如果不是, 可以通过-Dedit_config_json='{"export_config.multi_placeholder":true}' 进行修改 + + - 如果有设置feature_config.features.max_partitions, 请加入下面的命令重置: + - -Dedit_config_json='{"feature_config.features\[:\].max_partitions":1}'进行修改, 可以获得更好的性能 + + +#### 特征筛选 + +- 可以筛选fg.json里面的部分特征用于训练 +- 方法: 在easyrec.config的model_config.feature_groups里面把不需要的特征注释掉即可 + +### 预测 + +#### 服务部署 + +- 部署的 service.json 示例如下 + +```shell +bizdate=$1 +cat << EOF > echo.json +{ + "name":"ali_rec_rnk", + "metadata": { + "resource": "eas-r-xxxx", + "cpu": 8, + "memory": 20000, + "instance": 2, + "rpc": { + "enable_jemalloc": 1, + "max_queue_size": 100 + } + }, + "model_config": { + "remote_type": "hologres", + "url": "postgresql://:@<域名>:/", + "tables": [{"name":".","key":"","value": ""}], + "period": 2880, + "fg_mode": "tf", + "outputs":"probs_ctr,probs_cvr", + }, + "processor": "easyrec-3.0", + "storage": [ + { + "mount_path": "/home/admin/docker_ml/workspace/model/", + "oss": { + "endpoint": "oss-cn-hangzhou-internal.aliyuncs.com", + "path": "oss://easyrec/ali_rec_sln_acc_rnk/20221122/export/final_with_fg" + } + } + ] +} + +EOF +# 执行部署命令。 +eascmd -i -k -e create echo.json +eascmd -i -k -e update ali_rec_rnk -s echo.json +``` + +详见文档：[EasyRec Processor（推荐打分服务）](https://help.aliyun.com/zh/pai/user-guide/easyrec) + +- processor: easyrec processor, 目前最新的版本为easyrec-3.0, [历史版本](../predict/processor.md#release). +- model_config: eas 部署配置。主要控制把 item 特征加载到内存中。目前数据源支持redis和holo + - period: item feature reload period, 单位minutes + - url: holo url + - fg_mode: 支持tf和normal两种模式, tf模式表示fg是以TF算子的方式执行的, 性能更好 + - tables: item特征存储在hologres表里面, 支持分多个表存储 + - key: 必填, itemId列的名字; + - value: 可选，需要加载的列名, 多个列名之间用逗号(,)分割; + - condition: 可选，where子语句支持筛选item, 如itemId \< 10000; + - timekey: 可选，用于item的增量更新，支持的格式: timestamp和int + - static: 可选, 表示是静态特征，不用周期性更新 + - 支持多个item表, 如果多张表有重复的列, 后面的表覆盖前面的表 + - hologres表里面每一列存储一个item特征,示例: + + + + + +
adgroup_idcate_idcampaign_idcustomerbrandprice
10003854803744811718260774
10003910344122588965901428797
..................
+- storage: 将oss的模型目录mount到docker的指定目录下 + - mount_path: docker内部的挂载路径, 与示例保持一致即可 + - 配置了storage就不需要配置model_path了 + - 优点: 部署速度快 +- model_path: 将模型拷贝到docker内部 + - 缺点: 部署速度慢, 需要将模型保存到docker内部 + - 建议仅在无法通过storage挂载的情况下使用model_path +- 其它参数是所有EAS服务通用的, 请参考[EAS文档](https://help.aliyun.com/zh/pai/user-guide/parameters-of-model-services). + +#### 客户端请求 + +和TFRequest类似, EasyRec Processor也是使用ProtoBuffer 作为传输协议. [proto文件定义](https://github.com/pai-eas/eas-java-sdk/blob/master/src/main/proto/easyrec_predict.proto). Java客户端可以通过PAI-EAS Java SDK调用服务, 在pom.xml里面加入: + +``` + + com.aliyun.openservices.eas + eas-sdk + 2.0.9 + +``` + +代码参考： + +```java +import com.aliyun.openservices.eas.predict.http.*; +import com.aliyun.openservices.eas.predict.request.EasyRecRequest; + +PredictClient client = new PredictClient(new HttpConfig()); + +// 使用网络直连功能, 为了提升吞吐降低RT, 建议使用网络直连 +// Endpoint需要根据相应的region进行修改 +client.setDirectEndpoint("pai-eas-vpc.cn-hangzhou.aliyuncs.com"); + +// 通过普通网关访问时，需要使用以用户UID开头的Endpoint +// 在PAI-EAS控制台服务的调用信息中可以获得该信息 +client.setEndpoint("xxxxxxx.vpc.cn-hangzhou.pai-eas.aliyuncs.com"); + +client.setModelName("ali_rec_rnk"); +// 设置服务Token信息 +client.setToken("atxjzk****"); + +EasyRecRequest easyrecRequest = new EasyRecRequest(separator); +easyrecRequest.appendUserFeatureString(userFeatures); +easyrecRequest.appendContextFeatureString(contextFeatures); +easyrecRequest.appendItemStr(itemIdStr, ","); + +PredictProtos.PBResponse response = client.predict(easyrecRequest); + +for (Map.Entry entry : response.getResultsMap().entrySet()) { + String key = entry.getKey(); + PredictProtos.Results value = entry.getValue(); + System.out.print("key: " + key); + for (int i = 0; i < value.getScoresCount(); i++) { + System.out.format("value: %.6g\n", value.getScores(i)); + } +} +``` + +- client.setDirectEndpoint: [网络直连](https://help.aliyun.com/zh/pai/user-guide/call-a-service-over-the-vpc-direct-connection-channel)可以减少网络传输时间, 显著提升性能 + + - 请从上述文档查看不同region对应的direct endpoint地址 + +- EasyRecRequest类方法列表 + + + + + + + + + + + +
方法描述
setSeparator(String sep)设置user特征分隔符, 默认是"\u0002", 即CTRL_B
addUserFeature(String key, T value)增加一个user特征, key是特征名, value是特征值, value支持的类型包括String, float, long, int
appendUserFeatureString增加一组user特征,特征之间以separator分隔,特征内部使用":"分隔特征名和特征值
appendItemId(String itemId)增加一个itemId
appendItemStr(String itemIdStr)增加一组itemIds, 以","分隔
addContextFeature(String key, List contextFeatures)增加一个context特征, key是特征名, contextFeatures是特征值的列表, 列表中的元素和itemId一一对应
addContextFeatureString(String contextStr)增加一个context特征,特征名和特征值,特征值和特征值之间以":"分割,分割后的长度应该等于len(itemIds) + 1
getRequest()返回build好的EasyRecPredictProtos.PBRequest
+ +- 验证特征一致性 + + ```java + // 获取FG之后的特征，以便和离线的特征对比一致性 + // 将DebugLevel设置成1，即可返回生成的特征 + easyrecRequest.setDebugLevel(1); + PredictProtos.PBResponse response = client.predict(easyrecRequest); + Map genFeas = response.getGenerateFeaturesMap(); + for(String itemId: genFeas.keySet()) { + System.out.println(itemId); + System.out.println(genFeas.get(itemId)); + } + ``` + +- setDebugLevel: 设置调试标志, 方便排查问题, 参数的取值范围如下: + + - 0: 仅返回预测结果, 不返回调试信息 + - 1: 只返回FG之后特征的值, 格式为key:value格式, 不返回预测结果 + - 2: 返回预测结果和FG之后的特征值 + - 3: 返回FG之后特征值, 格式为表格格式, 特征之间用\\u0002分隔, 适用于ODL[实时样本](./odl_sample.md)构造的场景 + - 4: 将FG之后生成的特征值直接写入datahub, 不通过rpc返回, 适用于ODL[实时样本](./odl_sample.md)构造的场景 + - 100: 保存请求到模型目录下, 同时返回预测结果 + - 101: 保存timeline + - 102: 适用于召回模型, 返回user向量和Faiss检索结果 + +- 注意: 生产环境调用的时候设置debug_level=0，否则会导致rt上升, qps下降. From 37123ff4446b330fce817c5266b07b800e5b731c Mon Sep 17 00:00:00 2001 From: yangxudong Date: Wed, 10 Sep 2025 14:05:23 +0800 Subject: [PATCH 34/51] update FG document --- docker/Dockerfile_paitf115 | 36 +++++++++ docker/Dockerfile_tf211 | 36 +++++++++ docs/images/other/fg.svg | 2 +- docs/source/feature/fg.md | 34 ++++----- docs/source/quick_start/local_tutorial.md | 2 +- easy_rec/python/compat/early_stopping.py | 2 +- easy_rec/python/input/odps_input_v3.py | 5 +- easy_rec/python/test/odps_input_v3_test.py | 86 ---------------------- easy_rec/python/test/train_eval_test.py | 2 +- setup.cfg | 2 +- 10 files changed, 97 insertions(+), 110 deletions(-) create mode 100644 docker/Dockerfile_paitf115 create mode 100644 docker/Dockerfile_tf211 delete mode 100644 easy_rec/python/test/odps_input_v3_test.py diff --git a/docker/Dockerfile_paitf115 b/docker/Dockerfile_paitf115 new file mode 100644 index 000000000..bd383adea --- /dev/null +++ b/docker/Dockerfile_paitf115 @@ -0,0 +1,36 @@ +FROM dsw-registry.cn-shanghai.cr.aliyuncs.com/pai/tensorflow-training:1.15PAI-cpu-py36-ubuntu18.04 + +COPY docker/sources_18.04.list /etc/apt/sources.list + +# necessary for later commands to take effect +RUN md5sum /etc/apt/sources.list + +RUN apt-get update +RUN apt-get install apt-utils inetutils-ping wget curl telnet vim strace libpq-dev curl libsasl2-dev gcc g++ unzip openjdk-8-jdk -y + +RUN mkdir /EasyRec +COPY requirements /EasyRec/requirements +COPY requirements.txt /EasyRec/ +COPY easy_rec /EasyRec/easy_rec/ +COPY setup.cfg /EasyRec/ +COPY setup.py /EasyRec/ +COPY MANIFEST.in /EasyRec/ +COPY README.md /EasyRec/ +COPY scripts /EasyRec/scripts + +RUN curl "http://easyrec.oss-cn-beijing.aliyuncs.com/tools/odpscmd_public_0.45.0.zip" -o /EasyRec/odpscmd_public.zip +RUN mkdir /usr/local/odps_clt/ && cd /usr/local/odps_clt/ && unzip /EasyRec/odpscmd_public.zip +RUN ln -s /usr/local/odps_clt/bin/odpscmd /usr/local/bin/odpscmd +RUN pip3 install --upgrade pip +RUN pip3 install pystack-debugger idna kafka-python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install -r /EasyRec/requirements/runtime.txt +RUN pip3 install -r /EasyRec/requirements/extra.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-1.1.0-cp36-cp36m-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/releases/pai_automl-0.0.1rc1-py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.2%2Btunnel-py2.py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install tensorflow_probability==0.8 -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN cd /EasyRec && pip install . +RUN rm -rf /EasyRec +RUN python -c "import easy_rec; easy_rec.help(); import pyhive; import datahub; import kafka" + +COPY docker/hadoop_env.sh /opt/hadoop_env.sh diff --git a/docker/Dockerfile_tf211 b/docker/Dockerfile_tf211 new file mode 100644 index 000000000..52f227cca --- /dev/null +++ b/docker/Dockerfile_tf211 @@ -0,0 +1,36 @@ +FROM dsw-registry.cn-shanghai.cr.aliyuncs.com/pai/tensorflow:2.11-cpu-py39-ubuntu20.04-1 +COPY docker/sources_20.04.list /etc/apt/sources.list + +# necessary for later commands to take effect +RUN md5sum /etc/apt/sources.list + +RUN apt-get update +RUN apt-get install apt-utils inetutils-ping wget curl telnet vim strace libpq-dev curl libsasl2-dev gcc g++ unzip openjdk-8-jdk -y + +RUN mkdir /EasyRec +COPY requirements /EasyRec/requirements +COPY requirements.txt /EasyRec/ +COPY easy_rec /EasyRec/easy_rec/ +COPY setup.cfg /EasyRec/ +COPY setup.py /EasyRec/ +COPY MANIFEST.in /EasyRec/ +COPY README.md /EasyRec/ +COPY scripts /EasyRec/scripts + +RUN curl "http://easyrec.oss-cn-beijing.aliyuncs.com/tools/odpscmd_public_0.45.0.zip" -o /EasyRec/odpscmd_public.zip +RUN mkdir /usr/local/odps_clt/ && cd /usr/local/odps_clt/ && unzip /EasyRec/odpscmd_public.zip +RUN ln -s /usr/local/odps_clt/bin/odpscmd /usr/local/bin/odpscmd +RUN python -m pip install --upgrade pip +RUN pip3 install pystack-debugger idna kafka-python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install -r /EasyRec/requirements/runtime.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install -r /EasyRec/requirements/extra.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install https://tzrec.oss-cn-beijing.aliyuncs.com/third_party/graphlearn-1.2.0-cp39-cp39-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +# RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/releases/pai_automl-0.0.1rc1-py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN pip3 install tensorflow_probability==0.19.0 -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +#RUN pip3 install encodings +RUN pip3 install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.j2%2Btunnel-py2.py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +RUN cd /EasyRec && pip install . +RUN rm -rf /EasyRec +# RUN python -c "import easy_rec; easy_rec.help(); import pyhive; import datahub; import kafka" + +COPY docker/hadoop_env.sh /opt/hadoop_env.sh diff --git a/docs/images/other/fg.svg b/docs/images/other/fg.svg index e39f71a5a..22df9f5f2 100644 --- a/docs/images/other/fg.svg +++ b/docs/images/other/fg.svg @@ -1 +1 @@ -
User Feature
Pipelines
[Not supported by viewer]
Feature
Generation
[Not supported by viewer]
Recommender
Service
[Not supported by viewer]
Log Msg Queue
[Not supported by viewer]
Item DB
[Not supported by viewer]
User DB
[Not supported by viewer]
Item Feature
Pipelines
[Not supported by viewer]
Feature
Generation
[Not supported by viewer]
Model Training
Pipelines
[Not supported by viewer]
User Category Preference:
football:0.9; rock-climb:0.5;
skiing:0.1; ... ...

Item Category: rock-climb


User-Category-Prefer: 0.5
[Not supported by viewer]
Ranking
Service
[Not supported by viewer]
Item Index
[Not supported by viewer]
FeatureStore
[Not supported by viewer]
Offline Data Pipelines
[Not supported by viewer] \ No newline at end of file +
User Feature
Pipelines
[Not supported by viewer]
Feature
Generation
[Not supported by viewer]
Recommender
Service
[Not supported by viewer]
Log Msg Queue
[Not supported by viewer]
Item DB
[Not supported by viewer]
User DB
[Not supported by viewer]
Item Feature
Pipelines
[Not supported by viewer]
Feature
Generation
[Not supported by viewer]
Model Training
Pipelines
[Not supported by viewer]
User Category Preference:
football:0.9; rock-climb:0.5;
skiing:0.1; ... ...

Item Category: rock-climb


User-Category-Prefer: 0.5
[Not supported by viewer]
Ranking
Service
[Not supported by viewer]
Item Index
[Not supported by viewer]
FeatureStore
[Not supported by viewer]
Offline Data Pipelines
[Not supported by viewer] diff --git a/docs/source/feature/fg.md b/docs/source/feature/fg.md index 7d7017aba..acaa204f4 100644 --- a/docs/source/feature/fg.md +++ b/docs/source/feature/fg.md @@ -4,7 +4,7 @@ 是一套把原始输入转换为模型所需输入（特征）的数据变换过程，用来保证离线、在线样本生成结果的一致性。 特征生成也可以理解为特征变换，对单个特征或者多个特征做变换。我们提供了各种类型的FG算子来完成各种特征变换操作。 -特征生成只关注同时需要在离线和在线样本生成过程中的变换操作。如果某个变换操作只需要作用在离线阶段，则不需要定义为FG的操作。 +特征生成只关注同时需要在离线和在线样本生成过程中的变换操作。如果某个变换操作只需要作用在离线阶段，则不需要定义为FG的操作。 FG模块在推荐系统架构中的位置如下图所示： @@ -55,21 +55,21 @@ FG模块在推荐系统架构中的位置如下图所示： FG支持的特征变换算子与EasyRec支持的特征（`Feature Column`）之间没有严格的对应关系，大致可以参加如下表格： -| FG 算子 | EasyRec Feature Column | -|:-------------|:------------------------------------| -| id_feature | IdFeature 或 TagFeature | -| raw_feature | RawFeature | -| expr_feature | RawFeature | -| combo_feature | IdFeature 或 TagFeature | -| lookup_feature | RawFeature 或 IdFeature 或 TagFeature | -| match_feature | RawFeature 或 IdFeature 或 TagFeature | -| overlap_feature | RawFeature | -| sequence_feature | SequenceFeature 或 TagFeature | -| bm25_feature | RawFeature | -| kv_dot_product | RawFeature | -| tokenize_feature | TagFeature | -| text_normalizer | IdFeature | -| regex_replace_feature | IdFeature | +| FG 算子 | EasyRec Feature Column | +| :-------------------- | :---------------------------------- | +| id_feature | IdFeature 或 TagFeature | +| raw_feature | RawFeature | +| expr_feature | RawFeature | +| combo_feature | IdFeature 或 TagFeature | +| lookup_feature | RawFeature 或 IdFeature 或 TagFeature | +| match_feature | RawFeature 或 IdFeature 或 TagFeature | +| overlap_feature | RawFeature | +| sequence_feature | SequenceFeature 或 TagFeature | +| bm25_feature | RawFeature | +| kv_dot_product | RawFeature | +| tokenize_feature | TagFeature | +| text_normalizer | IdFeature | +| regex_replace_feature | IdFeature | 备注：**FG的执行结果输出给EasyRec模型，两种之间是串联的关系**。 @@ -122,8 +122,8 @@ pai -name easy_rec_ext 如果不是, 可以通过-Dedit_config_json='{"export_config.multi_placeholder":true}' 进行修改 - 如果有设置feature_config.features.max_partitions, 请加入下面的命令重置: - - -Dedit_config_json='{"feature_config.features\[:\].max_partitions":1}'进行修改, 可以获得更好的性能 + - -Dedit_config_json='{"feature_config.features\[:\].max_partitions":1}'进行修改, 可以获得更好的性能 #### 特征筛选 diff --git a/docs/source/quick_start/local_tutorial.md b/docs/source/quick_start/local_tutorial.md index 72287c4f6..4a00140b4 100644 --- a/docs/source/quick_start/local_tutorial.md +++ b/docs/source/quick_start/local_tutorial.md @@ -60,7 +60,7 @@ docker exec -it bash 可选镜像： -- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-paitf1.12-0.8.5 [只能跑在DLC环境] +- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-paitf1.12-0.8.5 \[只能跑在DLC环境\] - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-paitf1.15-0.8.5 - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15.5-0.8.5 - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15.5-gpu-0.8.5 diff --git a/easy_rec/python/compat/early_stopping.py b/easy_rec/python/compat/early_stopping.py index fe4c12132..fc850fb62 100644 --- a/easy_rec/python/compat/early_stopping.py +++ b/easy_rec/python/compat/early_stopping.py @@ -21,9 +21,9 @@ import os import threading import time -from distutils.version import LooseVersion import tensorflow as tf +from distutils.version import LooseVersion from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import init_ops diff --git a/easy_rec/python/input/odps_input_v3.py b/easy_rec/python/input/odps_input_v3.py index f16f3e5a6..6ec737ca1 100644 --- a/easy_rec/python/input/odps_input_v3.py +++ b/easy_rec/python/input/odps_input_v3.py @@ -32,8 +32,9 @@ def __init__(self, task_num, check_mode, pipeline_config) self._num_epoch = 0 if common_io is None: - logging.error("""please install common_io pip install - https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.2%2Btunnel-py2.py3-none-any.whl""" + logging.error(''' + please install common_io pip install + https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.2%2Btunnel-py2.py3-none-any.whl''' ) sys.exit(1) diff --git a/easy_rec/python/test/odps_input_v3_test.py b/easy_rec/python/test/odps_input_v3_test.py deleted file mode 100644 index fba4c79a2..000000000 --- a/easy_rec/python/test/odps_input_v3_test.py +++ /dev/null @@ -1,86 +0,0 @@ -# -*- encoding:utf-8 -*- -# Copyright (c) Alibaba, Inc. and its affiliates. -"""Test odps input v3.""" - -import tensorflow as tf -from easy_rec.python.input.odps_input_v3 import OdpsInputV3 -from google.protobuf import text_format -from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.protos.feature_config_pb2 import FeatureConfig - - -class OdpsInputV3Test(tf.test.TestCase): - - def __init__(self, methodName='OdpsInputV3'): - super(OdpsInputV3Test, self).__init__(methodName=methodName) - # must put .odps_config.ini in HOME directory - self._input_path = 'odps://xingqudao_saas/tables/xingqudao_dbmtl_v1_training_set_fg_encoded_v1/ds=20250526' - - def test_read_data(self): - data_config_str = """ - input_fields { - input_name: 'user_id' - input_type: STRING - } - input_fields { - input_name: 'item_id' - input_type: STRING - } - input_fields { - input_name: 'is_click' - input_type: INT64 - } - input_fields { - input_name: 'is_collect_like_comment' - input_type: INT64 - } - input_fields { - input_name: 'features' - input_type: STRING - } - label_fields: 'is_click' - batch_size: 1024 - num_epochs: 1 - prefetch_size: 1 - """ - feature_config_str = """ - input_names: 'user_id' - feature_type: IdFeature - embedding_dim: 32 - hash_bucket_size: 2000 - """ - dataset_config = DatasetConfig() - text_format.Merge(data_config_str, dataset_config) - feature_config = FeatureConfig() - text_format.Merge(feature_config_str, feature_config) - feature_configs = [feature_config] - empty_config = FeatureConfig() - empty_config.CopyFrom(feature_config) - while len(empty_config.input_names) > 0: - empty_config.input_names.pop() - for input_name in ['item_id', 'features']: - tmp_config = FeatureConfig() - tmp_config.CopyFrom(empty_config) - tmp_config.input_names.append(input_name) - feature_configs.append(tmp_config) - - train_input_fn = OdpsInputV3(dataset_config, feature_configs, - self._input_path).create_input() - dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) - iterator = dataset.make_initializable_iterator() - tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) - features, labels = iterator.get_next() - init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) - gpu_options = tf.GPUOptions(allow_growth=True) - session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=False) - with self.test_session(config=session_config) as sess: - sess.run(init_op) - feature_dict, label_dict = sess.run([features, labels]) - print("feature:", feature_dict, "label:", label_dict) - - -if __name__ == '__main__': - tf.test.main() diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 1327de687..72d1eaa5f 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -7,11 +7,11 @@ import threading import time import unittest -from distutils.version import LooseVersion import numpy as np import six import tensorflow as tf +from distutils.version import LooseVersion from tensorflow.python.platform import gfile from easy_rec.python.main import predict diff --git a/setup.cfg b/setup.cfg index d8ed85f21..f0223c47a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,7 @@ multi_line_output = 7 force_single_line = true known_standard_library = setuptools known_first_party = easy_rec -known_third_party = absl,common_io,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,scipy,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,tensorflow_probability,yaml +known_third_party = absl,common_io,distutils,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,scipy,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,tensorflow_probability,yaml no_lines_before = LOCALFOLDER default_section = THIRDPARTY skip = easy_rec/python/protos From 29ee292ce5360e93072248d766f05cdc11e8fb06 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 11 Sep 2025 11:32:25 +0800 Subject: [PATCH 35/51] update FG document --- easy_rec/python/test/train_eval_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 72d1eaa5f..fe5f8fa0c 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -817,6 +817,7 @@ def test_fit_on_eval(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_on_taobao.config', self._test_dir, + total_steps=10, num_evaluator=1, fit_on_eval=True) self.assertTrue(self._success) From 7897820613aa60d3cbbe575f5f191705fcf2b3db Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 11 Sep 2025 16:57:07 +0800 Subject: [PATCH 36/51] update FG document --- easy_rec/python/utils/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easy_rec/python/utils/test_utils.py b/easy_rec/python/utils/test_utils.py index b71249ac8..f790c2d85 100644 --- a/easy_rec/python/utils/test_utils.py +++ b/easy_rec/python/utils/test_utils.py @@ -43,7 +43,7 @@ def get_hdfs_tmp_dir(test_dir): return test_rand_dir -def proc_wait(proc, timeout=1200): +def proc_wait(proc, timeout=1800): t0 = time.time() while proc.poll() is None and time.time() - t0 < timeout: time.sleep(1) From 07cf59b216d2c54282be64cfa86ae3603047fc33 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 11 Sep 2025 22:23:30 +0800 Subject: [PATCH 37/51] update FG document --- easy_rec/python/test/train_eval_test.py | 1 + easy_rec/python/utils/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index fe5f8fa0c..60745f16f 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -813,6 +813,7 @@ def test_train_with_ps_worker(self): 'samples/model_config/multi_tower_on_taobao.config', self._test_dir) self.assertTrue(self._success) + @unittest.skip("Timeout on CI machine") def test_fit_on_eval(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_on_taobao.config', diff --git a/easy_rec/python/utils/test_utils.py b/easy_rec/python/utils/test_utils.py index f790c2d85..b71249ac8 100644 --- a/easy_rec/python/utils/test_utils.py +++ b/easy_rec/python/utils/test_utils.py @@ -43,7 +43,7 @@ def get_hdfs_tmp_dir(test_dir): return test_rand_dir -def proc_wait(proc, timeout=1800): +def proc_wait(proc, timeout=1200): t0 = time.time() while proc.poll() is None and time.time() - t0 < timeout: time.sleep(1) From f4c138d4238f299dd7094886807fb5cf12740145 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Fri, 12 Sep 2025 18:26:35 +0800 Subject: [PATCH 38/51] update FG document --- docs/source/feature/feature.rst | 21 --------------------- docs/source/train.md | 9 --------- 2 files changed, 30 deletions(-) diff --git a/docs/source/feature/feature.rst b/docs/source/feature/feature.rst index 384b8dc13..0ca35be5e 100644 --- a/docs/source/feature/feature.rst +++ b/docs/source/feature/feature.rst @@ -462,27 +462,6 @@ ExprFeature：表达式特征 - 当前版本未定义"&","|"的符号优先级，建议使用括号保证优先级。 - customized normalization: "tf.math.log1p(user_age) / 10.0" -EmbeddingVariable ----------------------------------------------------------------- -Key Value Hash, 减少hash冲突, 支持特征准入和特征淘汰。 - -.. code:: protobuf - - model_config { - model_class: "MultiTower" - ... - ev_params { - filter_freq: 2 - } - } -- 配置方式: - - feature_config单独配置ev_params - - model_config里面统一配置ev_params - -- ev_params : EVParams - - filter_freq: 频次过滤, 低频特征噪声大,过滤噪声让模型更鲁棒 - - steps_to_live: 特征淘汰, 淘汰过期特征,防止模型过大 -- Note: 仅在安装PAI-TF/DeepRec时可用 特征选择 ---------------------------------------------------------------- diff --git a/docs/source/train.md b/docs/source/train.md index 85dd4af0b..c85c7085f 100644 --- a/docs/source/train.md +++ b/docs/source/train.md @@ -209,15 +209,6 @@ pai -name easy_rec_ext -project algo_public ... } ``` -- 如使用key-Value Embedding, 需要设置model_config.ev_params - ``` - model_config { - ... - ev_params { - } - ... - } - ``` ### 命令 From dc5aa67a8917ad1fe2e6480ddfa144cfb190213d Mon Sep 17 00:00:00 2001 From: yangxudong Date: Wed, 24 Sep 2025 17:57:49 +0800 Subject: [PATCH 39/51] fix fg input size check --- docs/source/feature/fg.md | 7 ++++++- easy_rec/python/utils/config_util.py | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/source/feature/fg.md b/docs/source/feature/fg.md index acaa204f4..cdc7a1b13 100644 --- a/docs/source/feature/fg.md +++ b/docs/source/feature/fg.md @@ -182,7 +182,7 @@ eascmd -i -k -e update ali_rec_rn - processor: easyrec processor, 目前最新的版本为easyrec-3.0, [历史版本](../predict/processor.md#release). - model_config: eas 部署配置。主要控制把 item 特征加载到内存中。目前数据源支持redis和holo - period: item feature reload period, 单位minutes - - url: holo url + - url: holo url, 格式为postgresql://:@<域名>:/ - fg_mode: 支持tf和normal两种模式, tf模式表示fg是以TF算子的方式执行的, 性能更好 - tables: item特征存储在hologres表里面, 支持分多个表存储 - key: 必填, itemId列的名字; @@ -191,6 +191,8 @@ eascmd -i -k -e update ali_rec_rn - timekey: 可选，用于item的增量更新，支持的格式: timestamp和int - static: 可选, 表示是静态特征，不用周期性更新 - 支持多个item表, 如果多张表有重复的列, 后面的表覆盖前面的表 + - "tables": [{"key":"table1", ...},{"key":"table2", ...}] + - 如果多张表有重复的列，后面的表将覆盖前面的表 - hologres表里面每一列存储一个item特征,示例: @@ -198,6 +200,9 @@ eascmd -i -k -e update ali_rec_rn
adgroup_idcate_idcampaign_idcustomerbrandprice
10003910344122588965901428797
..................
+ - remote_type: Item特征数据源, 目前支持：`hologres`, `none` + - hologres：通过SQL接口进行数据读取和写入，适用于海量数据的存储和查询 + - none: 不使用Item特征缓存，item特征通过请求传入，此时tables应设置为[] - storage: 将oss的模型目录mount到docker的指定目录下 - mount_path: docker内部的挂载路径, 与示例保持一致即可 - 配置了storage就不需要配置model_path了 diff --git a/easy_rec/python/utils/config_util.py b/easy_rec/python/utils/config_util.py index 54c17dfda..3c6f385e7 100644 --- a/easy_rec/python/utils/config_util.py +++ b/easy_rec/python/utils/config_util.py @@ -440,11 +440,15 @@ def get_input_name_from_fg_json(fg_json): input_names = [] for fea in fg_json['features']: if 'feature_name' in fea: + if 'stub_type' in fea and fea['stub_type']: + continue input_names.append(fea['feature_name']) elif 'sequence_name' in fea: sequence_name = fea['sequence_name'] for seq_fea in fea['features']: assert 'feature_name' in seq_fea + if 'stub_type' in seq_fea and seq_fea['stub_type']: + continue feature_name = seq_fea['feature_name'] input_names.append(sequence_name + '__' + feature_name) return input_names From 4776a7b38e53b9b19a07100b97997566a3da94e6 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Fri, 7 Nov 2025 18:06:47 +0800 Subject: [PATCH 40/51] upgrade zero inflated lognormal loss, support export structure path --- .pre-commit-config.yaml | 2 +- docs/source/export.md | 7 +- docs/source/feature/fg.md | 6 +- docs/source/models/loss.md | 60 ++++++++++++++- easy_rec/python/builders/loss_builder.py | 17 ++++- .../compat/feature_column/feature_column.py | 4 +- .../feature_column/feature_column_v2.py | 4 +- easy_rec/python/inference/predictor.py | 9 ++- easy_rec/python/layers/cmbf.py | 1 + .../python/loss/zero_inflated_lognormal.py | 76 +++++++++++++++---- easy_rec/python/main.py | 18 ++++- easy_rec/python/model/easy_rec_estimator.py | 6 +- easy_rec/python/model/rank_model.py | 25 +++++- easy_rec/python/protos/loss.proto | 11 +++ easy_rec/python/test/odps_run.py | 2 +- easy_rec/python/test/train_eval_test.py | 2 +- easy_rec/version.py | 2 +- pai_jobs/run.py | 2 +- 18 files changed, 213 insertions(+), 41 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2132ed9e..9cbceeecb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: - id: mixed-line-ending args: ["--fix=lf"] - repo: https://github.com/myint/docformatter - rev: v1.3.1 + rev: v1.7.7 hooks: - id: docformatter args: ["--in-place", "--wrap-descriptions", "0", "--wrap-summaries", "0"] diff --git a/docs/source/export.md b/docs/source/export.md index 9ab9b31d4..de5e30469 100644 --- a/docs/source/export.md +++ b/docs/source/export.md @@ -88,8 +88,8 @@ pai -name easy_rec_ext -project algo_public - -Dconfig: 同训练 - -Dcmd: export 模型导出 - -Dexport_dir: 导出的目录 -- -Dcheckpoint_path: 使用指定的checkpoint_path -- -Darn: rolearn  注意这个的arn要替换成客户自己的。可以从dataworks的设置中查看arn。 +- -Dcheckpoint_path: 可选参数，使用指定的checkpoint_path导出 +- -Darn: rolearn 注意这个的arn要替换成客户自己的。可以从dataworks的设置中查看arn。 - -DossHost: ossHost地址 - -Dbuckets: config所在的bucket和保存模型的bucket; 如果有多个bucket，逗号分割 - 如果是pai内部版,则不需要指定arn和ossHost, arn和ossHost放在-Dbuckets里面 @@ -98,6 +98,9 @@ pai -name easy_rec_ext -project algo_public - --export_done_file: 导出完成标志文件名, 导出完成后，在导出目录下创建一个文件表示导出完成了 - --clear_export: 删除旧的导出文件目录 - --place_embedding_on_cpu: 将embedding相关的操作放在cpu上，有助于提升模型在gpu环境下的推理速度 + - --asset_files: 需要导出的asset文件路径, 可设置多个, 逗号分隔； + - 如果需要导出到assets目录的子目录下，使用`${target_path}:${source_path}`的格式；（从版本0.8.7开始支持） + - e.g. '--asset_files custom_fg_lib/fg.json:oss://${bucket}/path/to/fg.json' - 模型导出之后可以使用(EasyRecProcessor)\[./predict/在线预测.md\]部署到PAI-EAS平台 ### 双塔召回模型 diff --git a/docs/source/feature/fg.md b/docs/source/feature/fg.md index cdc7a1b13..f230e3f4d 100644 --- a/docs/source/feature/fg.md +++ b/docs/source/feature/fg.md @@ -182,7 +182,7 @@ eascmd -i -k -e update ali_rec_rn - processor: easyrec processor, 目前最新的版本为easyrec-3.0, [历史版本](../predict/processor.md#release). - model_config: eas 部署配置。主要控制把 item 特征加载到内存中。目前数据源支持redis和holo - period: item feature reload period, 单位minutes - - url: holo url, 格式为postgresql://:@<域名>:/ + - url: holo url, 格式为postgresql://:@\<域名>:/ - fg_mode: 支持tf和normal两种模式, tf模式表示fg是以TF算子的方式执行的, 性能更好 - tables: item特征存储在hologres表里面, 支持分多个表存储 - key: 必填, itemId列的名字; @@ -191,7 +191,7 @@ eascmd -i -k -e update ali_rec_rn - timekey: 可选，用于item的增量更新，支持的格式: timestamp和int - static: 可选, 表示是静态特征，不用周期性更新 - 支持多个item表, 如果多张表有重复的列, 后面的表覆盖前面的表 - - "tables": [{"key":"table1", ...},{"key":"table2", ...}] + - "tables": \[{"key":"table1", ...},{"key":"table2", ...}\] - 如果多张表有重复的列，后面的表将覆盖前面的表 - hologres表里面每一列存储一个item特征,示例: @@ -202,7 +202,7 @@ eascmd -i -k -e update ali_rec_rn
- remote_type: Item特征数据源, 目前支持：`hologres`, `none` - hologres：通过SQL接口进行数据读取和写入，适用于海量数据的存储和查询 - - none: 不使用Item特征缓存，item特征通过请求传入，此时tables应设置为[] + - none: 不使用Item特征缓存，item特征通过请求传入，此时tables应设置为\[\] - storage: 将oss的模型目录mount到docker的指定目录下 - mount_path: docker内部的挂载路径, 与示例保持一致即可 - 配置了storage就不需要配置model_path了 diff --git a/docs/source/models/loss.md b/docs/source/models/loss.md index e098aa0a6..dc2139a48 100644 --- a/docs/source/models/loss.md +++ b/docs/source/models/loss.md @@ -53,7 +53,7 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 下面的配置可以同时使用`F1_REWEIGHTED_LOSS`和`PAIR_WISE_LOSS`，总的loss为这两个损失函数的加权求和。 -``` +```protobuf losses { loss_type: F1_REWEIGHTED_LOSS weight: 1.0 @@ -71,7 +71,7 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 可以调节二分类模型recall/precision相对权重的损失函数，配置如下： - ``` + ```protobuf { loss_type: F1_REWEIGHTED_LOSS f1_reweight_loss { @@ -134,6 +134,10 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 - session_name: list分组的字段名，比如user_id - 参考论文：《 [Joint Optimization of Ranking and Calibration with Contextualized Hybrid Model](https://arxiv.org/pdf/2208.06164.pdf) 》 - 使用示例: [dbmtl_with_jrc_loss.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/dbmtl_on_taobao_with_multi_loss.config) + - 有几个注意点： + 1. JRC_Loss不要和普通二分类loss一起使用，因为它内部已经包含了二分类loss了，最好是先单独使用 + 1. JRC_Loss依赖mini-batch类的同session样本对，因此样本不能全局随机打散； 要把样本按照session_id 分组，同一组的样本需要shuffle到一起；（考验sql功力，如果搞不定不分组也可以，但需要保证同一个session的样本尽量排在一起，即group by session_id） + 1. 模型训练时`batch_size`尽可能大，在内存能够支撑的前提下`batch_size`调到最大（比如，8192）之后，再调整其他参数（如果需要的话） - LISTWISE_RANK_LOSS 的参数配置 @@ -142,6 +146,33 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 - label_is_logits: bool, 标记label是否为teacher模型的输出logits，默认为false - scale_logits: bool, 是否需要对模型的logits进行线性缩放，默认为false +- ZILN_LOSS 的参数配置 + + - mu_regularization: mu参数的正则化系数，默认值为0.01 + - sigma_regularization: sigma参数的正则化系数，默认值为0.01 + - max_sigma: sigma参数的最大值，默认值为5.0（sigma>5 就会让均值乘上 exp(0.5\*25) ≈ 2.7e5 的因子，已经很激进） + - max_log_clip_value: log(预测值)的最大值，默认值为20.0（最大预测值默认为exp(20)） + - return_log_pred_value: 是否返回log(预测值)，默认值为false + - classification_weight: 分类任务的权重，默认值为1.0 + - regression_weight: 回归任务的权重，默认值为1.0；零值越多，建议分类权重越小，回归权重越大 + - 配置示例如下 + ```protobuf + losses { + loss_type: ZILN_LOSS + weight: 1.0 + loss_name: "LTV" + ziln_loss { + mu_regularization: 0.01 + sigma_regularization: 0.01 + max_log_clip_value: 20.0 + max_sigma: 5.0 + return_log_pred_value: false + classification_weight: 1.0 + regression_weight: 1.0 + } + } + ``` + 排序模型同时使用多个损失函数的完整示例： [cmbf_with_multi_loss.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/cmbf_with_multi_loss.config) @@ -183,6 +214,31 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 - loss_weight_strategy: Random - 表示损失函数的权重设定为归一化的随机数 +### 根据样本设定损失函数权重（Masked Loss） + +多目标学习任务中，通常需要根据样本的属性来设定损失函数的权重。 + +#### 根据样本属性设定损失函数权重 + +在某个目标的tower里配置`task_space_indicator_name`和`task_space_indicator_value` + +- task_space_indicator_name 是特征名 +- task_space_indicator_value 是特征值 +- in_task_space_weight 目标样本的loss权重，默认值为1.0 +- out_task_space_weight 非目标样本的loss权重，默认值为1.0 + +如果样本的特征值与你配置的`task_space_indicator_value`相等, loss 权重 * `in_task_space_weight`; +不相等则loss权重 * `out_task_space_weight`。 + +`out_task_space_weight`的值改为0.0，则可实现`Masked Loss`。 + +#### 根据样本label来设定损失函数权重 + +在某个目标的tower里配置`task_space_indicator_label`这个字段，标记一个 label 的名字， +如果这个label的值大于0, 则 loss 权重 \*`in_task_space_weight`; 否则 loss权重 * `out_task_space_weight`。 + +`out_task_space_weight`的值改为0.0，则可实现`Masked Loss`。 + ### 参考论文： - 《 Multi-Task Learning Using Uncertainty to Weigh Losses for Scene Geometry and Semantics 》 diff --git a/easy_rec/python/builders/loss_builder.py b/easy_rec/python/builders/loss_builder.py index 720dfdd9e..66fd8dcd2 100644 --- a/easy_rec/python/builders/loss_builder.py +++ b/easy_rec/python/builders/loss_builder.py @@ -50,7 +50,22 @@ def build(loss_type, return tf.losses.mean_squared_error( labels=label, predictions=pred, weights=loss_weight, **kwargs) elif loss_type == LossType.ZILN_LOSS: - loss = zero_inflated_lognormal_loss(label, pred) + if loss_param is None: + loss = zero_inflated_lognormal_loss(label, pred) + else: + mu_reg = loss_param.mu_regularization + sigma_reg = loss_param.sigma_regularization + max_sigma = loss_param.max_sigma + class_weight = loss_param.classification_weight + reg_weight = loss_param.regression_weight + loss = zero_inflated_lognormal_loss( + label, + pred, + max_sigma=max_sigma, + mu_reg=mu_reg, + sigma_reg=sigma_reg, + class_weight=class_weight, + reg_weight=reg_weight) if np.isscalar(loss_weight) and loss_weight != 1.0: return loss * loss_weight return loss diff --git a/easy_rec/python/compat/feature_column/feature_column.py b/easy_rec/python/compat/feature_column/feature_column.py index 73a568d9c..1981ac4a3 100644 --- a/easy_rec/python/compat/feature_column/feature_column.py +++ b/easy_rec/python/compat/feature_column/feature_column.py @@ -2693,7 +2693,7 @@ class _NumericColumn( collections.namedtuple( '_NumericColumn', ['key', 'shape', 'default_value', 'dtype', 'normalizer_fn'])): - """see `numeric_column`.""" + """See `numeric_column`.""" @property def name(self): @@ -3126,7 +3126,7 @@ class _HashedCategoricalColumn(_CategoricalColumn, collections.namedtuple( '_HashedCategoricalColumn', ['key', 'hash_bucket_size', 'dtype'])): - """see `categorical_column_with_hash_bucket`.""" + """See `categorical_column_with_hash_bucket`.""" @property def name(self): diff --git a/easy_rec/python/compat/feature_column/feature_column_v2.py b/easy_rec/python/compat/feature_column/feature_column_v2.py index 01b9fc93f..4e785a566 100644 --- a/easy_rec/python/compat/feature_column/feature_column_v2.py +++ b/easy_rec/python/compat/feature_column/feature_column_v2.py @@ -2636,7 +2636,7 @@ class NumericColumn( collections.namedtuple('NumericColumn', ('feature_name', 'key', 'shape', 'default_value', 'dtype', 'normalizer_fn'))): - """see `numeric_column`.""" + """See `numeric_column`.""" @property def _is_v2_column(self): @@ -3873,7 +3873,7 @@ class HashedCategoricalColumn( collections.namedtuple('HashedCategoricalColumn', ('feature_name', 'key', 'hash_bucket_size', 'dtype')) ): - """see `categorical_column_with_hash_bucket`.""" + """See `categorical_column_with_hash_bucket`.""" @property def _is_v2_column(self): diff --git a/easy_rec/python/inference/predictor.py b/easy_rec/python/inference/predictor.py index c57092ab5..fd5779212 100644 --- a/easy_rec/python/inference/predictor.py +++ b/easy_rec/python/inference/predictor.py @@ -263,9 +263,10 @@ def _build_model(self): type_name = asset_file.tensor_info.name.split(':')[0] asset_path = os.path.join(model_path, constants.ASSETS_DIRECTORY, asset_file.filename) - assert gfile.Exists( - asset_path), '%s is missing in saved model' % asset_path - self._assets[type_name] = asset_path + # assert gfile.Exists( + # asset_path), '%s is missing in saved model' % asset_path + if gfile.Exists(asset_path): + self._assets[type_name] = asset_path logging.info(self._assets) # get export config @@ -275,7 +276,7 @@ def _build_model(self): # self._export_config = json.loads(export_config_collection[0]) # logging.info('load export config info %s' % export_config_collection[0]) else: - raise ValueError('currently only savedmodel is supported') + raise ValueError('currently only saved_model is supported') def predict(self, input_data_dict, output_names=None): """Predict input data with loaded model. diff --git a/easy_rec/python/layers/cmbf.py b/easy_rec/python/layers/cmbf.py index e5f1caeb2..37b27a288 100644 --- a/easy_rec/python/layers/cmbf.py +++ b/easy_rec/python/layers/cmbf.py @@ -329,6 +329,7 @@ def __call__(self, is_training, *args, **kwargs): if not is_training: self._model_config.hidden_dropout_prob = 0.0 self._model_config.attention_probs_dropout_prob = 0.0 + self._model_config.text_seq_emb_dropout_prob = 0.0 # shape: [batch_size, image_num/image_dim, hidden_size] img_attention_fea = self.image_self_attention_tower() diff --git a/easy_rec/python/loss/zero_inflated_lognormal.py b/easy_rec/python/loss/zero_inflated_lognormal.py index e3ae3110e..e5b916794 100644 --- a/easy_rec/python/loss/zero_inflated_lognormal.py +++ b/easy_rec/python/loss/zero_inflated_lognormal.py @@ -1,6 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Zero-inflated lognormal loss for lifetime value prediction.""" +import logging + import tensorflow as tf import tensorflow_probability as tfp @@ -10,27 +12,53 @@ tf = tf.compat.v1 -def zero_inflated_lognormal_pred(logits): +def log_sigmoid(x): + return -tf.nn.softplus(-x) # 兼容 TF 1.12 + + +def zero_inflated_lognormal_pred(logits, + max_sigma=5.0, + max_log_clip=20.0, + return_log=False): """Calculates predicted mean of zero inflated lognormal logits. Arguments: logits: [batch_size, 3] tensor of logits. + max_sigma: max value of sigma + return_log: whether return log value + max_log_clip: max clip value of log space Returns: positive_probs: [batch_size, 1] tensor of positive probability. preds: [batch_size, 1] tensor of predicted mean. """ + logging.info('max_sigma=%f, max_log_clip=%f' % (max_sigma, max_log_clip)) logits = tf.convert_to_tensor(logits, dtype=tf.float32) positive_probs = tf.keras.backend.sigmoid(logits[..., :1]) - loc = logits[..., 1:2] - scale = tf.keras.backend.softplus(logits[..., 2:]) - preds = ( - positive_probs * - tf.keras.backend.exp(loc + 0.5 * tf.keras.backend.square(scale))) - return positive_probs, preds + log_positive_probs = log_sigmoid(logits[..., :1]) + mu = logits[..., 1:2] + sigma = tf.keras.backend.softplus(logits[..., 2:]) + sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), + max_sigma) + log_mean_pos = mu + 0.5 * tf.keras.backend.square(sigma) + log_preds = log_positive_probs + log_mean_pos + if return_log: + logging.info('return_log=true') + return positive_probs, log_preds + else: + log_preds = tf.clip_by_value(log_preds, -100.0, max_log_clip) + preds = tf.keras.backend.exp(log_preds) + return positive_probs, preds -def zero_inflated_lognormal_loss(labels, logits, name=''): +def zero_inflated_lognormal_loss(labels, + logits, + max_sigma=5.0, + mu_reg=0.01, + sigma_reg=0.01, + class_weight=1.0, + reg_weight=1.0, + name=''): """Computes the zero inflated lognormal loss. Usage with tf.keras API: @@ -43,12 +71,20 @@ def zero_inflated_lognormal_loss(labels, logits, name=''): Arguments: labels: True targets, tensor of shape [batch_size, 1]. logits: Logits of output layer, tensor of shape [batch_size, 3]. + max_sigma: max value of sigma + mu_reg: regularize coefficient of mu + sigma_reg: regularize coefficient of sigma + class_weight: weight of classification loss + reg_weight: weight of regression loss name: the name of loss Returns: Zero inflated lognormal loss value. """ loss_name = name if name else 'ziln_loss' + logging.info( + '%s max_sigma=%f, mu_reg=%f, sigma_reg=%f, classify weight:%f, regression weight %f' + % (loss_name, max_sigma, mu_reg, sigma_reg, class_weight, reg_weight)) labels = tf.cast(labels, dtype=tf.float32) if labels.shape.ndims == 1: labels = tf.expand_dims(labels, 1) # [B, 1] @@ -64,13 +100,23 @@ def zero_inflated_lognormal_loss(labels, logits, name=''): classification_loss = tf.keras.backend.mean(classification_loss) tf.summary.scalar('loss/%s_classify' % loss_name, classification_loss) - loc = logits[..., 1:2] - scale = tf.math.maximum( - tf.keras.backend.softplus(logits[..., 2:]), - tf.math.sqrt(tf.keras.backend.epsilon())) + mu = logits[..., 1:2] + sigma = tf.keras.backend.softplus(logits[..., 2:]) + sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), + max_sigma) + safe_labels = positive * labels + ( 1 - positive) * tf.keras.backend.ones_like(labels) - regression_loss = -tf.keras.backend.mean( - positive * tfd.LogNormal(loc=loc, scale=scale).log_prob(safe_labels)) + logprob = tfd.LogNormal(loc=mu, scale=sigma).log_prob(safe_labels) + num_pos = tf.reduce_sum(positive) + 1e-8 + regression_loss = -(tf.reduce_sum(positive * logprob) / num_pos) tf.summary.scalar('loss/%s_regression' % loss_name, regression_loss) - return classification_loss + regression_loss + + # add regular terms + loc_penalty = mu_reg * tf.reduce_mean(tf.square(mu)) + scale_penalty = sigma_reg * tf.reduce_mean(tf.square(logits[..., 2:])) + tf.summary.scalar('loss/%s_loc_penalty' % loss_name, loc_penalty) + tf.summary.scalar('loss/%s_scale_penalty' % loss_name, scale_penalty) + + total_loss = class_weight * classification_loss + reg_weight * regression_loss + return total_loss + loc_penalty + scale_penalty diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index 1747155fe..abce86237 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -776,9 +776,9 @@ def export(export_dir, feature_configs = config_util.get_compatible_feature_configs(pipeline_config) # create estimator params = {'log_device_placement': verbose} + asset_file_dict = {} if asset_files: logging.info('will add asset files: %s' % asset_files) - asset_file_dict = {} for asset_file in asset_files.split(','): asset_file = asset_file.strip() if ':' not in asset_file or asset_file.startswith( @@ -836,6 +836,22 @@ def export(export_dir, with tf.gfile.GFile(saved_pb_path, 'wb') as fout: fout.write(saved_model.SerializeToString()) + # modify export path + assets_dir = os.path.join(final_export_dir, 'assets') + for asset_rel_path in asset_file_dict: + if not os.path.dirname(asset_rel_path): + continue + asset_file = os.path.basename(asset_file_dict[asset_rel_path]) + asset_file = os.path.join(assets_dir, asset_file) + if not tf.gfile.Exists(asset_file): + logging.info('asset file %s does not exist' % asset_file) + continue + target_file = os.path.join(assets_dir, asset_rel_path) + target_dir = os.path.dirname(target_file) + if not tf.gfile.Exists(target_dir): + tf.gfile.MakeDirs(target_dir) + logging.info('will rename %s to %s' % (asset_file, target_file)) + tf.gfile.Rename(asset_file, target_file) logging.info('model has been exported to %s successfully' % final_export_dir) return final_export_dir diff --git a/easy_rec/python/model/easy_rec_estimator.py b/easy_rec/python/model/easy_rec_estimator.py index 95385936c..aca664d05 100644 --- a/easy_rec/python/model/easy_rec_estimator.py +++ b/easy_rec/python/model/easy_rec_estimator.py @@ -624,7 +624,11 @@ def _export_model_fn(self, features, labels, run_config, params): for asset_file in export_config.asset_files: if asset_file.startswith('!'): asset_file = asset_file[1:] - _, asset_name = os.path.split(asset_file) + if ':' not in asset_file or asset_file.startswith( + 'oss:') or asset_file.startswith('hdfs:'): + _, asset_name = os.path.split(asset_file) + else: + asset_name, asset_file = asset_file.split(':', 1) ops.add_to_collection( ops.GraphKeys.ASSET_FILEPATHS, tf.constant(asset_file, dtype=tf.string, name=asset_name)) diff --git a/easy_rec/python/model/rank_model.py b/easy_rec/python/model/rank_model.py index dc3771daf..ebed369dd 100644 --- a/easy_rec/python/model/rank_model.py +++ b/easy_rec/python/model/rank_model.py @@ -57,7 +57,8 @@ def _output_to_prediction_impl(self, output, loss_type, num_class=1, - suffix=''): + suffix='', + **kwargs): prediction_dict = {} binary_loss_type = { LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, @@ -82,7 +83,14 @@ def _output_to_prediction_impl(self, prediction_dict['probs' + suffix] = probs[:, 1] elif loss_type == LossType.ZILN_LOSS: assert num_class == 3, 'num_class must be 3 when loss type is ZILN_LOSS' - probs, preds = zero_inflated_lognormal_pred(output) + max_log_clip_val = kwargs.get('max_log_clip_value', 20.0) + return_log = kwargs.get('return_log_pred_value', False) + max_sigma = kwargs.get('max_sigma', 5.0) + probs, preds = zero_inflated_lognormal_pred( + output, + max_sigma=max_sigma, + max_log_clip=max_log_clip_val, + return_log=return_log) tf.summary.scalar('prediction/probs', tf.reduce_mean(probs)) tf.summary.scalar('prediction/y', tf.reduce_mean(preds)) prediction_dict['logits' + suffix] = output @@ -121,8 +129,19 @@ def _add_to_prediction_dict(self, output): self._prediction_dict.update(prediction_dict) else: for loss in self._losses: + kwargs = {} + if loss.loss_type == LossType.ZILN_LOSS: + loss_param = loss.WhichOneof('loss_param') + if loss_param is not None: + loss_param = getattr(loss, loss_param) + kwargs['max_log_clip_value'] = loss_param.max_log_clip_value + kwargs['return_log_pred_value'] = loss_param.return_log_pred_value + kwargs['max_sigma'] = loss_param.max_sigma prediction_dict = self._output_to_prediction_impl( - output, loss_type=loss.loss_type, num_class=self._num_class) + output, + loss_type=loss.loss_type, + num_class=self._num_class, + **kwargs) self._prediction_dict.update(prediction_dict) def build_rtp_output_dict(self): diff --git a/easy_rec/python/protos/loss.proto b/easy_rec/python/protos/loss.proto index 4416111a8..c5dfed839 100644 --- a/easy_rec/python/protos/loss.proto +++ b/easy_rec/python/protos/loss.proto @@ -44,9 +44,20 @@ message Loss { PairwiseHingeLoss pairwise_hinge_loss = 110; ListwiseRankLoss listwise_rank_loss = 111; ListwiseDistillLoss listwise_distill_loss = 112; + ZILNLoss ziln_loss = 113; } }; +message ZILNLoss { + optional float mu_regularization = 1 [default = 0.01]; + optional float sigma_regularization = 2 [default = 0.01]; + optional float max_log_clip_value = 3 [default = 20.0]; + optional float max_sigma = 4 [default = 5.0]; + optional bool return_log_pred_value = 5 [default = false]; + optional float classification_weight = 6 [default = 1.0]; + optional float regression_weight = 7 [default = 1.0]; +} + message SoftmaxCrossEntropyWithNegativeMining { required uint32 num_negative_samples = 1; required float margin = 2 [default = 0]; diff --git a/easy_rec/python/test/odps_run.py b/easy_rec/python/test/odps_run.py index 84bd44f9b..1eb77c926 100644 --- a/easy_rec/python/test/odps_run.py +++ b/easy_rec/python/test/odps_run.py @@ -24,7 +24,7 @@ class TestPipelineOnOdps(tf.test.TestCase): - """train eval export test on odps.""" + """Train eval export test on odps.""" def test_deepfm(self): start_files = ['deep_fm/create_inner_deepfm_table.sql'] diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 60745f16f..14b511a63 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -813,7 +813,7 @@ def test_train_with_ps_worker(self): 'samples/model_config/multi_tower_on_taobao.config', self._test_dir) self.assertTrue(self._success) - @unittest.skip("Timeout on CI machine") + @unittest.skip('Timeout on CI machine') def test_fit_on_eval(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_on_taobao.config', diff --git a/easy_rec/version.py b/easy_rec/version.py index fa13b61b9..80ce168e3 100644 --- a/easy_rec/version.py +++ b/easy_rec/version.py @@ -1,4 +1,4 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -__version__ = '0.8.6' +__version__ = '0.8.7' diff --git a/pai_jobs/run.py b/pai_jobs/run.py index 309ec4e7a..e3abc891d 100644 --- a/pai_jobs/run.py +++ b/pai_jobs/run.py @@ -485,7 +485,7 @@ def main(argv): FLAGS.job_name, eval_method='none') - assert len(FLAGS.worker_hosts.split(',')) == 1, 'export only need 1 woker' + assert len(FLAGS.worker_hosts.split(',')) == 1, 'export only need 1 worker' config_util.auto_expand_share_feature_configs(pipeline_config) export_dir = FLAGS.export_dir From 47c681d218998ddf98b778adbac5507eecd3022f Mon Sep 17 00:00:00 2001 From: yangxudong Date: Mon, 10 Nov 2025 10:02:34 +0800 Subject: [PATCH 41/51] upgrade zero inflated lognormal loss, support export structure path --- docs/source/train.md | 2 +- .../test/zero_inflated_lognormal_test.py | 61 +++++++++++-------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/docs/source/train.md b/docs/source/train.md index 15907d3ab..19073b971 100644 --- a/docs/source/train.md +++ b/docs/source/train.md @@ -60,7 +60,7 @@ - 使用SyncReplicasOptimizer进行分布式训练(同步模式) - 仅在train_distribute为NoStrategy时可以设置成true，其它情况应该设置为false - PS异步训练也设置为false - - 注意在设置为 true 时，总共的训练步数为：min(total_sample_num \* num_epochs / batch_size, num_steps) / num_workers + - 注意在设置为 true 时，总共的训练步数为：min(total_sample_num * num_epochs / batch_size, num_steps) / num_workers - train_distribute: 默认不开启Strategy(NoStrategy), strategy确定分布式执行的方式, 可以分成两种模式: PS-Worker模式 和 All-Reduce模式 diff --git a/easy_rec/python/test/zero_inflated_lognormal_test.py b/easy_rec/python/test/zero_inflated_lognormal_test.py index f512e48e8..627db542d 100644 --- a/easy_rec/python/test/zero_inflated_lognormal_test.py +++ b/easy_rec/python/test/zero_inflated_lognormal_test.py @@ -9,19 +9,6 @@ if tf.__version__ >= '2.0': tf = tf.compat.v1 -# Absolute error tolerance in asserting array near. -_ERR_TOL = 1e-6 - - -# softplus function that calculates log(1+exp(x)) -def _softplus(x): - return np.log(1.0 + np.exp(x)) - - -# sigmoid function that calculates 1/(1+exp(-x)) -def _sigmoid(x): - return 1 / (1 + np.exp(-x)) - class ZeroInflatedLognormalLossTest(tf.test.TestCase): @@ -30,21 +17,47 @@ def setUp(self): self.logits = np.array([[.1, .2, .3], [.4, .5, .6]]) self.labels = np.array([[0.], [1.5]]) - def zero_inflated_lognormal(self, labels, logits): - positive_logits = logits[..., :1] - loss_zero = _softplus(positive_logits) - loc = logits[..., 1:2] - scale = np.maximum( - _softplus(logits[..., 2:]), np.sqrt(tf.keras.backend.epsilon())) - log_prob_non_zero = stats.lognorm.logpdf( - x=labels, s=scale, loc=0, scale=np.exp(loc)) - loss_non_zero = _softplus(-positive_logits) - log_prob_non_zero - return np.mean(np.where(labels == 0., loss_zero, loss_non_zero), axis=-1) + def zero_inflated_lognormal(self, labels, logits, max_sigma=5.0): + labels = labels.astype(np.float32) + if labels.ndim == 1: + labels = labels[:, None] + logits = logits.astype(np.float32) + positive = (labels > 0).astype(np.float32) # [B,1] + + z = logits[..., :1] + mu = logits[..., 1:2] + sigma = np.log1p(np.exp(logits[..., 2:])) # softplus + sigma = np.clip(sigma, np.sqrt(tf.keras.backend.epsilon()), max_sigma) + + # 分类损失：BCE with logits，按 batch 平均 + # bce(z, y) = max(z,0) - z*y + log(1 + exp(-|z|)) + bce = np.maximum(z, 0.0) - z * positive + np.log1p(np.exp(-np.abs(z))) + classification_loss = np.mean(bce) + + # 回归损失：仅对正样本的 log_prob 求平均（按正样本数） + # 避免对 y=0 求 logpdf + mask = (labels > 0).squeeze(-1) + logprob = np.zeros_like(labels, dtype=np.float64) + if np.any(mask): + # scipy 参数化：s=shape= sigma, scale=exp(mu), loc=0 + logprob[mask, 0] = stats.lognorm.logpdf( + x=labels[mask, 0].astype(np.float64), + s=sigma[mask, 0].astype(np.float64), + loc=0.0, + scale=np.exp(mu[mask, 0].astype(np.float64))) + num_pos = np.sum(positive) + 1e-8 + regression_loss = -(np.sum(positive * logprob) / num_pos) + # 与实现一致的组合（此处测试会把正则项设为0） + total = classification_loss + regression_loss + return float(total) def test_loss_value(self): expected_loss = self.zero_inflated_lognormal(self.labels, self.logits) expected_loss = np.average(expected_loss) - loss = zero_inflated_lognormal_loss(self.labels, self.logits) + loss = zero_inflated_lognormal_loss( + self.labels, self.logits, mu_reg=0, sigma_reg=0) + # Absolute error tolerance in asserting array near. + _ERR_TOL = 1e-6 self.assertNear(self.evaluate(loss), expected_loss, _ERR_TOL) From 39a3fc203bf756acb2f1a927e1eb815b6e72d866 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Mon, 10 Nov 2025 15:19:04 +0800 Subject: [PATCH 42/51] upgrade zero inflated lognormal loss, support export structure path, upgrade pre-commit hooks --- .gitignore | 2 + .pre-commit-config.yaml | 20 +- README.md | 4 +- docs/post_fix.py | 2 +- docs/source/_ext/post_process.py | 9 +- docs/source/automl/finetune_config.md | 2 +- docs/source/automl/hpo_config.md | 140 +- docs/source/automl/hpo_emr.md | 18 +- docs/source/automl/hpo_pai.md | 14 +- docs/source/automl/hpo_res.md | 2 +- docs/source/automl/pai_nni_hpo.md | 10 +- docs/source/benchmark.md | 10 +- docs/source/component/backbone.md | 6 +- docs/source/component/component.md | 454 +-- docs/source/conf.py | 53 +- docs/source/eval.md | 18 +- docs/source/export.md | 6 +- docs/source/faq.md | 2 +- docs/source/feature/excel_config.md | 4 +- docs/source/feature/fg.md | 38 +- docs/source/feature/fg_docs/ComboFeature.md | 10 +- docs/source/feature/fg_docs/IdFeature.md | 22 +- docs/source/feature/fg_docs/OverLapFeature.md | 34 +- docs/source/feature/fg_docs/RawFeature.md | 20 +- docs/source/feature/odl_sample.md | 4 +- docs/source/feature/rtp_fg.md | 18 +- docs/source/feature/rtp_native.md | 10 +- docs/source/incremental_train.md | 2 +- docs/source/intro.md | 4 +- docs/source/models/aitm.md | 2 +- docs/source/models/cmbf.md | 6 +- docs/source/models/dbmtl.md | 6 +- docs/source/models/deepfm.md | 2 +- docs/source/models/dlrm.md | 2 +- docs/source/models/dssm.md | 2 +- docs/source/models/dssm_neg_sampler.md | 2 +- docs/source/models/esmm.md | 4 +- docs/source/models/loss.md | 46 +- docs/source/models/mind.md | 4 +- docs/source/models/multi_tower.md | 6 +- docs/source/models/rocket_launching.md | 2 +- docs/source/models/uniter.md | 4 +- docs/source/models/wide_and_deep.md | 2 +- docs/source/online_train.md | 6 +- ...73\347\272\277\351\242\204\346\265\213.md" | 2 +- ...73\347\272\277\351\242\204\346\265\213.md" | 8 +- docs/source/predict/processor.md | 12 +- docs/source/quick_start/designer_tutorial.md | 78 +- docs/source/quick_start/emr_tutorial.md | 2 +- docs/source/quick_start/local_tutorial.md | 22 +- docs/source/quick_start/mc_tutorial.md | 6 +- docs/source/quick_start/mc_tutorial_inner.md | 2 +- docs/source/tf_on_yarn.md | 4 +- docs/source/train.md | 6 +- docs/source/vector_retrieve.md | 24 +- easy_rec/__init__.py | 10 +- .../python/builders/hyperparams_builder.py | 32 +- easy_rec/python/builders/loss_builder.py | 268 +- easy_rec/python/builders/optimizer_builder.py | 120 +- easy_rec/python/builders/strategy_builder.py | 17 +- easy_rec/python/compat/adam_s.py | 109 +- easy_rec/python/compat/array_ops.py | 45 +- easy_rec/python/compat/dynamic_variable.py | 204 +- easy_rec/python/compat/early_stopping.py | 297 +- easy_rec/python/compat/embedding_ops.py | 117 +- .../python/compat/embedding_parallel_saver.py | 222 +- easy_rec/python/compat/estimator_train.py | 65 +- easy_rec/python/compat/exporter.py | 180 +- .../compat/feature_column/feature_column.py | 1829 ++++++------ .../feature_column/feature_column_v2.py | 2543 ++++++++++------- .../feature_column/sequence_feature_column.py | 297 +- .../python/compat/feature_column/utils.py | 45 +- easy_rec/python/compat/layers.py | 219 +- easy_rec/python/compat/optimizers.py | 331 ++- easy_rec/python/compat/queues.py | 68 +- easy_rec/python/compat/regularizers.py | 66 +- easy_rec/python/compat/sok_optimizer.py | 149 +- .../python/compat/sync_replicas_optimizer.py | 163 +- .../python/compat/weight_decay_optimizers.py | 175 +- .../python/core/easyrec_metrics/__init__.py | 5 +- .../distribute_metrics_impl_pai.py | 1976 +++++++------ .../distribute_metrics_impl_tf.py | 2049 +++++++------ easy_rec/python/core/learning_schedules.py | 152 +- easy_rec/python/core/metrics.py | 159 +- easy_rec/python/core/sampler.py | 464 +-- easy_rec/python/eval.py | 67 +- easy_rec/python/export.py | 89 +- .../python/feature_column/feature_column.py | 351 ++- .../python/feature_column/feature_group.py | 3 +- easy_rec/python/hpo/emr_hpo.py | 131 +- easy_rec/python/hpo/generate_hpo_sql.py | 55 +- easy_rec/python/hpo/pai_hpo.py | 202 +- .../python/inference/client/client_demo.py | 39 +- .../inference/client/easyrec_request.py | 3 +- easy_rec/python/inference/csv_predictor.py | 74 +- .../inference/hive_parquet_predictor.py | 82 +- easy_rec/python/inference/hive_predictor.py | 76 +- easy_rec/python/inference/odps_predictor.py | 44 +- .../python/inference/parquet_predictor.py | 81 +- .../python/inference/parquet_predictor_v2.py | 81 +- easy_rec/python/inference/predictor.py | 185 +- easy_rec/python/inference/processor/test.py | 81 +- easy_rec/python/inference/vector_retrieve.py | 80 +- easy_rec/python/input/batch_tfrecord_input.py | 98 +- easy_rec/python/input/criteo_binary_reader.py | 153 +- easy_rec/python/input/criteo_input.py | 68 +- easy_rec/python/input/csv_input.py | 124 +- easy_rec/python/input/csv_input_ex.py | 56 +- easy_rec/python/input/csv_input_v2.py | 61 +- easy_rec/python/input/datahub_input.py | 166 +- easy_rec/python/input/dummy_input.py | 33 +- easy_rec/python/input/hive_input.py | 91 +- easy_rec/python/input/hive_parquet_input.py | 89 +- easy_rec/python/input/hive_rtp_input.py | 124 +- easy_rec/python/input/input.py | 551 ++-- easy_rec/python/input/kafka_dataset.py | 78 +- easy_rec/python/input/kafka_input.py | 154 +- easy_rec/python/input/load_parquet.py | 128 +- easy_rec/python/input/odps_input.py | 80 +- easy_rec/python/input/odps_input_v2.py | 98 +- easy_rec/python/input/odps_input_v3.py | 77 +- easy_rec/python/input/odps_rtp_input.py | 173 +- easy_rec/python/input/odps_rtp_input_v2.py | 34 +- easy_rec/python/input/parquet_input.py | 188 +- easy_rec/python/input/parquet_input_v2.py | 57 +- easy_rec/python/input/parquet_input_v3.py | 114 +- easy_rec/python/input/rtp_input.py | 152 +- easy_rec/python/input/rtp_input_v2.py | 106 +- easy_rec/python/input/tfrecord_input.py | 74 +- easy_rec/python/layers/backbone.py | 79 +- easy_rec/python/layers/capsule_layer.py | 91 +- easy_rec/python/layers/cmbf.py | 248 +- easy_rec/python/layers/common_layers.py | 97 +- easy_rec/python/layers/dnn.py | 57 +- easy_rec/python/layers/embed_input_layer.py | 3 +- easy_rec/python/layers/input_layer.py | 212 +- easy_rec/python/layers/keras/__init__.py | 34 +- easy_rec/python/layers/keras/activation.py | 22 +- easy_rec/python/layers/keras/attention.py | 89 +- .../python/layers/keras/auxiliary_loss.py | 10 +- easy_rec/python/layers/keras/blocks.py | 111 +- easy_rec/python/layers/keras/bst.py | 92 +- easy_rec/python/layers/keras/custom_ops.py | 128 +- easy_rec/python/layers/keras/data_augment.py | 43 +- easy_rec/python/layers/keras/din.py | 16 +- easy_rec/python/layers/keras/einsum_dense.py | 371 +-- easy_rec/python/layers/keras/embedding.py | 13 +- easy_rec/python/layers/keras/fibinet.py | 61 +- easy_rec/python/layers/keras/interaction.py | 171 +- easy_rec/python/layers/keras/layer_norm.py | 123 +- easy_rec/python/layers/keras/mask_net.py | 42 +- .../layers/keras/multi_head_attention.py | 308 +- easy_rec/python/layers/keras/multi_task.py | 19 +- .../layers/keras/numerical_embedding.py | 113 +- easy_rec/python/layers/keras/ppnet.py | 107 +- easy_rec/python/layers/keras/transformer.py | 46 +- easy_rec/python/layers/layer_norm.py | 14 +- easy_rec/python/layers/mmoe.py | 38 +- easy_rec/python/layers/multihead_attention.py | 58 +- .../layers/multihead_cross_attention.py | 483 ++-- easy_rec/python/layers/senet.py | 36 +- easy_rec/python/layers/seq_input_layer.py | 83 +- .../python/layers/sequence_feature_layer.py | 226 +- easy_rec/python/layers/uniter.py | 181 +- easy_rec/python/layers/utils.py | 50 +- .../layers/variational_dropout_layer.py | 54 +- easy_rec/python/loss/circle_loss.py | 39 +- easy_rec/python/loss/contrastive_loss.py | 8 +- easy_rec/python/loss/f1_reweight_loss.py | 14 +- easy_rec/python/loss/focal_loss.py | 39 +- easy_rec/python/loss/jrc_loss.py | 57 +- easy_rec/python/loss/listwise_loss.py | 99 +- easy_rec/python/loss/multi_similarity.py | 37 +- easy_rec/python/loss/pairwise_loss.py | 175 +- .../loss/softmax_loss_with_negative_mining.py | 58 +- .../python/loss/zero_inflated_lognormal.py | 51 +- easy_rec/python/main.py | 496 ++-- easy_rec/python/model/autoint.py | 42 +- easy_rec/python/model/cmbf.py | 39 +- .../model/collaborative_metric_learning.py | 121 +- easy_rec/python/model/dat.py | 89 +- easy_rec/python/model/dbmtl.py | 90 +- easy_rec/python/model/dcn.py | 41 +- easy_rec/python/model/deepfm.py | 88 +- easy_rec/python/model/dlrm.py | 42 +- easy_rec/python/model/dropoutnet.py | 155 +- easy_rec/python/model/dssm.py | 109 +- easy_rec/python/model/dssm_senet.py | 105 +- easy_rec/python/model/dummy_model.py | 26 +- easy_rec/python/model/easy_rec_estimator.py | 516 ++-- easy_rec/python/model/easy_rec_model.py | 205 +- easy_rec/python/model/esmm.py | 163 +- easy_rec/python/model/fm.py | 41 +- easy_rec/python/model/match_model.py | 193 +- easy_rec/python/model/mind.py | 321 ++- easy_rec/python/model/mmoe.py | 57 +- easy_rec/python/model/multi_task_model.py | 185 +- easy_rec/python/model/multi_tower.py | 43 +- easy_rec/python/model/multi_tower_bst.py | 150 +- easy_rec/python/model/multi_tower_din.py | 86 +- easy_rec/python/model/multi_tower_recall.py | 54 +- easy_rec/python/model/pdn.py | 162 +- easy_rec/python/model/ple.py | 92 +- easy_rec/python/model/rank_model.py | 353 +-- easy_rec/python/model/rocket_launching.py | 223 +- easy_rec/python/model/simple_multi_task.py | 38 +- easy_rec/python/model/uniter.py | 39 +- easy_rec/python/model/wide_and_deep.py | 63 +- easy_rec/python/ops/gen_kafka_ops.py | 129 +- easy_rec/python/ops/gen_str_avx_op.py | 5 +- easy_rec/python/ops/incr_record.py | 11 +- easy_rec/python/predict.py | 148 +- easy_rec/python/test/csv_input_test.py | 65 +- .../python/test/custom_early_stop_func.py | 5 +- easy_rec/python/test/dh_local_run.py | 50 +- easy_rec/python/test/embed_test.py | 10 +- easy_rec/python/test/emr_run.py | 69 +- easy_rec/python/test/eval_metric_test.py | 86 +- easy_rec/python/test/excel_convert_test.py | 23 +- easy_rec/python/test/export_test.py | 370 ++- easy_rec/python/test/fg_test.py | 33 +- easy_rec/python/test/hive_input_test.py | 55 +- easy_rec/python/test/hpo_test.py | 110 +- easy_rec/python/test/kafka_test.py | 160 +- easy_rec/python/test/local_incr_test.py | 78 +- easy_rec/python/test/loss_test.py | 72 +- easy_rec/python/test/odps_command.py | 22 +- easy_rec/python/test/odps_local_run.py | 33 +- easy_rec/python/test/odps_run.py | 164 +- easy_rec/python/test/odps_test_prepare.py | 45 +- easy_rec/python/test/odps_test_util.py | 54 +- easy_rec/python/test/pre_check_test.py | 19 +- easy_rec/python/test/predictor_test.py | 273 +- easy_rec/python/test/rtp_convert_test.py | 54 +- easy_rec/python/test/run.py | 42 +- easy_rec/python/test/train_eval_test.py | 921 +++--- easy_rec/python/test/util_test.py | 41 +- .../test/zero_inflated_lognormal_test.py | 12 +- .../python/tools/add_boundaries_to_config.py | 31 +- .../tools/add_feature_info_to_config.py | 62 +- .../python/tools/convert_config_format.py | 16 +- easy_rec/python/tools/convert_rtp_data.py | 13 +- easy_rec/python/tools/convert_rtp_fg.py | 122 +- .../python/tools/create_config_from_excel.py | 170 +- easy_rec/python/tools/criteo/convert_data.py | 70 +- easy_rec/python/tools/edit_lookup_graph.py | 68 +- easy_rec/python/tools/faiss_index_pai.py | 51 +- easy_rec/python/tools/feature_selection.py | 177 +- easy_rec/python/tools/hit_rate_ds.py | 100 +- easy_rec/python/tools/hit_rate_pai.py | 61 +- easy_rec/python/tools/pre_check.py | 66 +- easy_rec/python/tools/predict_and_chk.py | 70 +- easy_rec/python/tools/read_kafka.py | 21 +- easy_rec/python/tools/split_model_pai.py | 107 +- easy_rec/python/tools/split_pdn_model_pai.py | 107 +- easy_rec/python/tools/test_saved_model.py | 40 +- easy_rec/python/tools/view_saved_model.py | 12 +- easy_rec/python/tools/write_kafka.py | 32 +- easy_rec/python/train_eval.py | 222 +- easy_rec/python/utils/activation.py | 27 +- easy_rec/python/utils/check_utils.py | 17 +- easy_rec/python/utils/config_util.py | 105 +- easy_rec/python/utils/constant.py | 4 +- easy_rec/python/utils/convert_rtp_fg.py | 168 +- easy_rec/python/utils/dag.py | 15 +- easy_rec/python/utils/distribution_utils.py | 177 +- easy_rec/python/utils/ds_util.py | 4 +- easy_rec/python/utils/embedding_utils.py | 4 +- easy_rec/python/utils/estimator_utils.py | 378 +-- easy_rec/python/utils/export_big_model.py | 427 +-- easy_rec/python/utils/expr_util.py | 6 +- easy_rec/python/utils/fg_util.py | 5 +- easy_rec/python/utils/hit_rate_utils.py | 99 +- easy_rec/python/utils/hive_utils.py | 46 +- easy_rec/python/utils/hpo_util.py | 11 +- easy_rec/python/utils/hvd_utils.py | 7 +- easy_rec/python/utils/input_utils.py | 41 +- easy_rec/python/utils/io_util.py | 86 +- easy_rec/python/utils/load_class.py | 38 +- easy_rec/python/utils/meta_graph_editor.py | 375 +-- easy_rec/python/utils/multi_optimizer.py | 3 +- easy_rec/python/utils/numpy_utils.py | 1 - easy_rec/python/utils/odps_util.py | 12 +- easy_rec/python/utils/pai_util.py | 9 +- easy_rec/python/utils/proto_util.py | 4 +- easy_rec/python/utils/restore_filter.py | 3 +- easy_rec/python/utils/shape_utils.py | 101 +- easy_rec/python/utils/test_utils.py | 329 ++- easy_rec/python/utils/tf_utils.py | 25 +- .../data/amazon_books_data/process_amazon.py | 41 +- examples/data/criteo/process_criteo_kaggle.py | 9 +- examples/data/movielens_1m/process_ml_1m.py | 56 +- examples/match_model/dssm.md | 2 +- examples/match_model/dssm_negative_sample.md | 2 +- examples/match_model/readme.md | 2 +- examples/rank_model/DeepFM.md | 2 +- examples/rank_model/wide_and_deep.md | 2 +- examples/readme.md | 6 +- git-lfs/git_lfs.py | 90 +- pai_jobs/run.py | 489 ++-- scripts/ci_test_change_files.py | 9 +- scripts/pre-commit | 2 +- setup.cfg | 14 +- setup.py | 43 +- 304 files changed, 20122 insertions(+), 15311 deletions(-) diff --git a/.gitignore b/.gitignore index 227b49663..5b1c97f62 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,5 @@ pai_jobs/easy_rec*.tar.gz .DS_Store .python-version +easy_rec/python/test/odps_input_v3_test.py +easy_rec/python/test.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9cbceeecb..fa89dc5c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,20 +6,16 @@ repos: additional_dependencies: [ 'flake8-docstrings==1.5.0' ] - - repo: https://github.com/asottile/seed-isort-config - rev: v2.2.0 - hooks: - - id: seed-isort-config - - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + - repo: https://github.com/pycqa/isort + rev: 5.12.0 hooks: - id: isort - - repo: https://github.com/pre-commit/mirrors-yapf - rev: v0.30.0 + - repo: https://github.com/google/yapf + rev: v0.43.0 hooks: - id: yapf - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 + rev: v6.0.0 hooks: - id: trailing-whitespace args: ["--no-markdown-linebreak-ext"] @@ -30,15 +26,15 @@ repos: - id: check-merge-conflict - id: mixed-line-ending args: ["--fix=lf"] - - repo: https://github.com/myint/docformatter + - repo: https://github.com/PyCQA/docformatter rev: v1.7.7 hooks: - id: docformatter args: ["--in-place", "--wrap-descriptions", "0", "--wrap-summaries", "0"] - repo: https://github.com/executablebooks/mdformat - rev: 0.7.1 + rev: 0.7.22 hooks: - id: mdformat additional_dependencies: [ - 'mdformat-tables==0.4.0' + 'mdformat-tables' ] diff --git a/README.md b/README.md index b3f88506d..df14a6bee 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # EasyRec Introduction -🎉 See our ongoing recommendation framework **[TorchEasyRec](https://github.com/alibaba/TorchEasyRec) !** 🎉 This evolution of EasyRec is built on **PyTorch**, featuring **GPU acceleration** and **hybrid parallelism** for enhanced performance. +🎉 See our ongoing recommendation framework **[TorchEasyRec](https://github.com/alibaba/TorchEasyRec) !** 🎉 This evolution of EasyRec is built on **PyTorch**, featuring **GPU acceleration** and **hybrid parallelism** for enhanced performance.   @@ -44,7 +44,7 @@ Running Platform: - Flexible feature config and simple model config - [Build models by combining some components](docs/source/component/backbone.md) -- Efficient and robust feature generation\[used in taobao\] +- Efficient and robust feature generation[used in taobao] - Nice web interface in development ### It is smart diff --git a/docs/post_fix.py b/docs/post_fix.py index ce279bed8..920787310 100644 --- a/docs/post_fix.py +++ b/docs/post_fix.py @@ -10,6 +10,6 @@ for line_str in lines: if '_static/searchtools.js' in line_str: fout.write( - ' \n' + ' \n' ) fout.write(line_str) diff --git a/docs/source/_ext/post_process.py b/docs/source/_ext/post_process.py index 8f46285e6..221dcdaa0 100644 --- a/docs/source/_ext/post_process.py +++ b/docs/source/_ext/post_process.py @@ -1,5 +1,4 @@ import logging - from docutils import nodes from docutils.transforms import Transform @@ -18,7 +17,7 @@ def _visit(node): for child in node.children: if isinstance(child, nodes.Element): if 'refuri' in child.attributes and '.md#' in child.attributes[ - 'refuri']: + 'refuri']: src = child.attributes['refuri'] dst = src.replace('.md#', '.html#') logging.info('[PostFixLink] replace %s to %s' % (src, dst)) @@ -32,7 +31,7 @@ def setup(app): app.add_post_transform(PostFixLink) return { - 'version': '0.1', - 'parallel_read_safe': True, - 'parallel_write_safe': True, + 'version': '0.1', + 'parallel_read_safe': True, + 'parallel_write_safe': True, } diff --git a/docs/source/automl/finetune_config.md b/docs/source/automl/finetune_config.md index 8b7131059..b07eac9cd 100644 --- a/docs/source/automl/finetune_config.md +++ b/docs/source/automl/finetune_config.md @@ -109,7 +109,7 @@ metric_dict={'auc_is_like':0.25, 'auc_is_valid_play':0.5, 'auc_is_comment':0.25} 与begin训练的`差异点`: - 每个配置模块支持jinja模版渲染 -- 配置finetune日期{% set date_list = \[20220616,20220617\] %} +- 配置finetune日期{% set date_list = [20220616,20220617] %} - 配置finetune开始日期{% set date_begin = 20220616 %}，Dfine_tune_checkpoint开始日期和后续日期采取的model路径不一样 - 假设每天finetune： - {bizdate} 必须保留，将会在代码中根据当天日期进行替换 diff --git a/docs/source/automl/hpo_config.md b/docs/source/automl/hpo_config.md index 5f99e42e6..a8c47e97a 100644 --- a/docs/source/automl/hpo_config.md +++ b/docs/source/automl/hpo_config.md @@ -24,13 +24,13 @@ assessor: 支持将该组中的实验结果和同组中的所有历史进行比较，如果不满足比较标准（例如小于中位数），则停止该组超参数的运行。比如说设置最大运行次数max_trial_num， 实际使用量会显著小于max_trial_num，但具体数量就和实际跑的任务及随机到的超参有关系了。例如max_trial_num=50时，可能最终可能不到 25 次，并且差不多已经是完整探索了50组超参。 ``` -| PAIAssessor | 描述 | 值 | -| ------------- | ----------------------------- | ----------------- | -| optimize_mode | 最大化优化的方向 | maximize/minimize | -| start_step | 从第几步开始进行早停判定 | 2 | -| moving_avg | 早停判断时，采用所有历史的滑动平均值作为判断标准 | True | +| PAIAssessor | 描述 | 值 | +| ------------- | ------------------------------------------------ | ----------------- | +| optimize_mode | 最大化优化的方向 | maximize/minimize | +| start_step | 从第几步开始进行早停判定 | 2 | +| moving_avg | 早停判断时，采用所有历史的滑动平均值作为判断标准 | True | | proportion | 本次超参搜索的最优值和历史记录的proportion值比较 | 0.5 | -| patience | metric指标连续下降几次，就停止 | 10 | +| patience | metric指标连续下降几次，就停止 | 10 | ### 示例 @@ -80,7 +80,7 @@ assessor: - cmd = cmd.replace('${params}', params)->支持参数标识路径，例如lr0.001_batchsize64 注意其中可能含有浮点数，请确定是否支持用来标识数据/数据表 -- cmd = cmd.replace(p, str(v)) 将搜索的参数替换为搜索的值，搜索参数可以使用${batch_size}、${lr}来标记，需要和search_space.json中的key匹配使用 +- cmd = cmd.replace(p, str(v)) 将搜索的参数替换为搜索的值，搜索参数可以使用${batch_size}、${lr}来标记，需要和search_space.json中的key匹配使用 ### jinja渲染 @@ -98,37 +98,37 @@ metric_source_{{bizdate}}=oss://automl-nni/easyrec/finetune/{{bizdate}}_finetune ### 字段介绍 -| 配置模块 | 描述 | 是否可选 | -| --------------- | ----------------------------------------------------------------- | ---- | -| platform_config | 用于标记任务执行的平台以及对应的执行命令 | 必选 | -| metric_config | 用于标记任务metric的获取来源、metric的key以及对应权重、metric类型、最终metric的方式 | 必选 | -| output_config | 如果使用服务版，可以配置output_config用来获取最优模型配置summary_path，用于配制tensorboard路径 | 可选 | -| schedule_config | 如果任务在指定时间内调度任务，则需要配置schedule_config,修改对应的schedule_config的值 | 可选 | -| params_config | 如果用户的参数是保存在文件中，则需要配置params_config, 用于标记需要修改参数的源文件路径和目标路径 | 可选 | -| oss_config | 如果任务需要使用OSS存储，则需要配置OSS config | 可选 | -| odps_config | 如果任务需要使用maxcompute平台执行任务，则需要配置odps config | 可选 | -| ts_config | 如果任务需要使用trainingservice平台执行任务，则需要配置ts config | 可选 | -| paiflow_config | 如果任务需要执行工作流任务，则需要配置paiflow_config,修改对应的paiflow_config的值 | 可选 | -| dlc_config | 如果任务需要执行dlc任务，则需要配置dlc_config,修改对应的dlc_config的值 | 可选 | -| monitor_config | 支持失败告警,最优metric更新时提醒 | 可选 | +| 配置模块 | 描述 | 是否可选 | +| --------------- | ------------------------------------------------------------------------------------------------- | -------- | +| platform_config | 用于标记任务执行的平台以及对应的执行命令 | 必选 | +| metric_config | 用于标记任务metric的获取来源、metric的key以及对应权重、metric类型、最终metric的方式 | 必选 | +| output_config | 如果使用服务版，可以配置output_config用来获取最优模型配置summary_path，用于配制tensorboard路径 | 可选 | +| schedule_config | 如果任务在指定时间内调度任务，则需要配置schedule_config,修改对应的schedule_config的值 | 可选 | +| params_config | 如果用户的参数是保存在文件中，则需要配置params_config, 用于标记需要修改参数的源文件路径和目标路径 | 可选 | +| oss_config | 如果任务需要使用OSS存储，则需要配置OSS config | 可选 | +| odps_config | 如果任务需要使用maxcompute平台执行任务，则需要配置odps config | 可选 | +| ts_config | 如果任务需要使用trainingservice平台执行任务，则需要配置ts config | 可选 | +| paiflow_config | 如果任务需要执行工作流任务，则需要配置paiflow_config,修改对应的paiflow_config的值 | 可选 | +| dlc_config | 如果任务需要执行dlc任务，则需要配置dlc_config,修改对应的dlc_config的值 | 可选 | +| monitor_config | 支持失败告警,最优metric更新时提醒 | 可选 | ## platform_config -| platform_config | 描述 | 值 | -| --------------- | ------------------------------------------------------------ | -------------------------------------------------------------- | -| name | 用于标记任务执行的平台 | DLC/MaxCompute/DataScience/LOCAL/PAI/PAIFLOW | -| cmdxx | 用于标记执行的命令，以cmd开头 | dlc submit pytorch --name=test_nni\_${exp_id}\_${trial_id} xxx | +| platform_config | 描述 | 值 | +| --------------- | ----------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | +| name | 用于标记任务执行的平台 | DLC/MaxCompute/DataScience/LOCAL/PAI/PAIFLOW | +| cmdxx | 用于标记执行的命令，以cmd开头 | dlc submit pytorch --name=test_nni\_${exp_id}\_${trial_id} xxx | | resume | 1表示开启续跑模式；用于用户一次运行时，比如说第一行任务成功，第二行由于资源不足失败，可以开启续跑，从第二行命令开始运行 | 0/1 | ## metric_config -| metric_config | 描述 | 值 | -| ---------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| metric_type | metric类型 | summary/table/api/json/stdout | -| metric_source | metric来源（可以为多个以metric_source开头的，具体可以看maxcompute_crossvalidation案例） | 对应为具体的路径或者job | -| final_mode | 如果任务运行过程中，存在很多中间metric，那么需要确定最终metric的计算方式 | final/best/avg | +| metric_config | 描述 | 值 | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| metric_type | metric类型 | summary/table/api/json/stdout | +| metric_source | metric来源（可以为多个以metric_source开头的，具体可以看maxcompute_crossvalidation案例） | 对应为具体的路径或者job | +| final_mode | 如果任务运行过程中，存在很多中间metric，那么需要确定最终metric的计算方式 | final/best/avg | | source_list_final_mode | 可选，默认值为final_mode，可选值为final/best/avg,用于有多个metric_source时最终metric如何计算，具体可以看maxcompute_crossvalidation案例 | final/best/avg | -| metric_dict | 对应查询的key以及对应的权重;可以为负值 | metric_dict={'auc_is_like':0.25, 'auc_is_valid_play':0.5, 'auc_is_comment':0.25, 'loss_play_time':-0.25} metric=val(’auc_is_valid_play’)\*0.5+val(’auc_is_like’)\*0.25+val(’auc_is_comment’)\*0.25-val(’loss_play_time’)\*0.25 | +| metric_dict | 对应查询的key以及对应的权重;可以为负值 | metric_dict={'auc_is_like':0.25, 'auc_is_valid_play':0.5, 'auc_is_comment':0.25, 'loss_play_time':-0.25} metric=val(’auc_is_valid_play’)\*0.5+val(’auc_is_like’)\*0.25+val(’auc_is_comment’)\*0.25-val(’loss_play_time’)\*0.25 | - 如果metric_type=stdout类型，则metric_dict对应的key为正则表达式，value为对应的权重 @@ -191,31 +191,31 @@ metric_source=cmd1 ## output_config -| output_config | 描述 | 值 | -| ------------- | ------------------------------------ | --- | -| model_path | 如果使用服务版，可以配置model_path用来获取最优模型 | 路径 | -| summary_path | 如果使用单机版，可以配置summary用于本地查看TensorBoard | 路径 | +| output_config | 描述 | 值 | +| ------------- | ------------------------------------------------------ | ---- | +| model_path | 如果使用服务版，可以配置model_path用来获取最优模型 | 路径 | +| summary_path | 如果使用单机版，可以配置summary用于本地查看TensorBoard | 路径 | ## schedule_config -| schedule_config | 描述 | 值 | -| --------------- | -------------------- | ---------------- | +| schedule_config | 描述 | 值 | +| --------------- | ---------------------------------- | ---------------- | | day | 支持在指定时间范围内调度AutoML任务 | everyday/weekend | -| start_time | 指定调度开始时间 | 00:00-23:59 | -| end_time | 指定调度结束时间 | 00:00-23:59 | +| start_time | 指定调度开始时间 | 00:00-23:59 | +| end_time | 指定调度结束时间 | 00:00-23:59 | ## params_config 如果用户的参数是保存在文件中，则需要配置params_config, -| params_config | 描述 | 值 | -| -------------------------- | ------------------------------------------------------ | ----------------- | +| params_config | 描述 | 值 | +| -------------------------- | ------------------------------------------------------------------------------------ | ----------------- | | params_src_dst_filepath1xx | 用于标记需要修改参数的源文件路径和目标路径,可以为多个，以params_src_dst_filepath开头 | src_path,dst_path | -| params_src_dst_filepath2xx | xx | xx | +| params_src_dst_filepath2xx | xx | xx | ## oss_config -| oss_config | 描述 | 值 | +| oss_config | 描述 | 值 | | --------------- | -------- | -------------------------------------------------------------------------- | | endpoint | endpoint | [http://oss-cn-shanghai.aliyuncs.com](http://oss-cn-shanghai.aliyuncs.com) | | accessKeyID | ak | ak | @@ -224,28 +224,28 @@ metric_source=cmd1 ## odps_config -| odps_config | 描述 | 值 | -| ------------- | ------------ | -------------------------------------------------------------------------------------- | -| access_id | ak | ak | -| access_key | sk | ak | -| project_name | project_name | xxx | -| end_point | end_point | 弹外: http://service.odps.aliyun.com/api 弹内：http://service-corp.odps.aliyun-inc.com/api | -| log_view_host | logview host | 弹外：http://logview.odps.aliyun.com 弹内：http://logview.alibaba-inc.com | -| role_arn | role_arn | acs:ram::xxx:role/aliyunserviceroleforpaiautoml | +| odps_config | 描述 | 值 | +| ------------- | ------------ | ------------------------------------------------------------------------------------------ | +| access_id | ak | ak | +| access_key | sk | ak | +| project_name | project_name | xxx | +| end_point | end_point | 弹外: http://service.odps.aliyun.com/api 弹内：http://service-corp.odps.aliyun-inc.com/api | +| log_view_host | logview host | 弹外：http://logview.odps.aliyun.com 弹内：http://logview.alibaba-inc.com | +| role_arn | role_arn | acs:ram::xxx:role/aliyunserviceroleforpaiautoml | ## dlc_config -| dlc_config | 描述 | 值 | -| ---------- | ----------- | ----------------------------------------------------------------- | -| access_id | ak | ak | -| access_key | sk | ak | +| dlc_config | 描述 | 值 | +| ---------- | ----------- | ----------------------------------------------------------------------- | +| access_id | ak | ak | +| access_key | sk | ak | | end_point | end_point | 弹外：pai-dlc.cn-shanghai.aliyuncs.com 弹内：pai-dlc-share.aliyuncs.com | -| region | cn-shanghai | cn-shanghai | -| protocol | protocol | http/https | +| region | cn-shanghai | cn-shanghai | +| protocol | protocol | http/https | ## ts_config -| ts_config | 描述 | 值 | +| ts_config | 描述 | 值 | | ----------------- | --------- | ---------------------------- | | access_key_id | ak | ak | | access_key_secret | sk | ak | @@ -254,7 +254,7 @@ metric_source=cmd1 ## paiflow_config -| paiflow_config | 描述 | 值 | +| paiflow_config | 描述 | 值 | | ----------------- | ------------ | ------- | | access_key_id | ak | ak | | access_key_secret | sk | ak | @@ -267,24 +267,24 @@ metric_source=cmd1 - 点击阿里钉头像->机器人管理-自定义机器人->群组选择工作通知 - 点击阿里钉头像->机器人管理-自定义机器人->群组：选择对应的群号 -| monitor_config | 描述 | 值 | -| -------------- | -------------------------------------------- | ----------------------------------------------------- | -| url | url为创建自定义机器人对应的Webhook地址 | https://oapi.dingtalk.com/robot/send?access_token=xxx | -| keyword | 添加自定义机器人：自定义关键词 | monitor | -| at_mobiles | 在content里添加@人的手机号，且只有在群内的成员才可被@，非群内成员手机号会被脱敏 | \['11xx'\] | -| at_user_ids | 被@人的用户userid。即工号 | \[\] | -| is_at_all | 是否@所有人 | True/False | +| monitor_config | 描述 | 值 | +| -------------- | ------------------------------------------------------------------------------- | ----------------------------------------------------- | +| url | url为创建自定义机器人对应的Webhook地址 | https://oapi.dingtalk.com/robot/send?access_token=xxx | +| keyword | 添加自定义机器人：自定义关键词 | monitor | +| at_mobiles | 在content里添加@人的手机号，且只有在群内的成员才可被@，非群内成员手机号会被脱敏 | ['11xx'] | +| at_user_ids | 被@人的用户userid。即工号 | [] | +| is_at_all | 是否@所有人 | True/False | ## search_space.json -| search_space | 描述 | 值 | -| ------------ | --------------------------------------------------------------------------------------------------------- | --- | -| key | trial.ini中配置的搜索参数变量 | | +| search_space | 描述 | 值 | +| ------------ | ---------------------------------------------------------------------------------------------------------------------------- | --- | +| key | trial.ini中配置的搜索参数变量 | | | type | nni中定义的搜索类型，相关配置参考[NNI searchSpace参考手册](https://nni.readthedocs.io/en/v2.2/Tutorial/SearchSpaceSpec.html) | | - {”\_type”: “choice”, “\_value”: options}：从options中选取一个。 -- {”\_type”: “randint”, “\_value”: \[lower, upper\]}：\[low,upper)之间选择一个随机整数。 -- {”\_type”: “uniform”, “\_value”: \[low, high\]}：\[low,upper\]之间随机采样。 +- {”\_type”: “randint”, “\_value”: [lower, upper]}：\[low,upper)之间选择一个随机整数。 +- {”\_type”: “uniform”, “\_value”: [low, high]}：[low,upper]之间随机采样。 | | value | value是根据业务、经验设置相关搜索值 | | diff --git a/docs/source/automl/hpo_emr.md b/docs/source/automl/hpo_emr.md index 36cf76c4b..01fb4a00b 100644 --- a/docs/source/automl/hpo_emr.md +++ b/docs/source/automl/hpo_emr.md @@ -17,13 +17,13 @@ python -m easy_rec.python.hpo.emr_hpo --hyperparams hyperparams.json --config_p ### 参数说明 -- --config_path easyrec训练配置文件 -- --exp_dir 调优实验目录 -- --debug 保留本地临时目录 +- --config_path easyrec训练配置文件 +- --exp_dir 调优实验目录 +- --debug 保留本地临时目录 - --metric_name  调优的指标，默认是auc，其它可选指标\[参考../eval.md) - --max_parallel   同一时刻可以并行跑的实验数目，默认4 - --total_trial_num  总共跑多少组实验，默认6 -- --el_submit_params el_submit指定PS/Worker资源的一些参数，包括-t x -m x \[-pn x -pc x -pm x\] -wn x -wc x -wm x -wg x 默认值 +- --el_submit_params el_submit指定PS/Worker资源的一些参数，包括-t x -m x [-pn x -pc x -pm x] -wn x -wc x -wm x -wg x 默认值 ```bash -t standalone -m local -wn 1 -wc 6 -wm 20000 -wg 1 @@ -43,9 +43,9 @@ python -m easy_rec.python.hpo.emr_hpo --hyperparams hyperparams.json --config_p ] ``` -- name: easy_rec pipeline_config里面的参数名称，注意要用全路径 +- name: easy_rec pipeline_config里面的参数名称，注意要用全路径 - feature_config.features\[**input_names\[0\]=field_name1**\].embedding_dim + feature_config.features\[**input_names[0]=field_name1**\].embedding_dim - 由于feature_config.features是一个数组，所以需要用到选择器，根据**属性值**选择部分特征: @@ -68,7 +68,7 @@ python -m easy_rec.python.hpo.emr_hpo --hyperparams hyperparams.json --config_p 有些参数的值是关联的，比如对于deepfm算法，所有的embedding_dim必须是一样的 -- name里面可以指定多个要调整的参数名称，用";"分割feature_config.features\[input_names\[0\]=field1\].embedding_dim;feature_config.features\[input_names\[0\]=field20\].embedding_dim +- name里面可以指定多个要调整的参数名称，用";"分割feature_config.features\[input_names[0]=field1\].embedding_dim;feature_config.features\[input_names[0]=field20\].embedding_dim - 如果name里面包含了多个参数名称，那么candidates也需要有多个参数值，用";"分割如"32;32" - candidates: 候选值 - type: 候选值类型, 支持Categorical, Integer, Real @@ -101,12 +101,12 @@ python -m easy_rec.python.hpo.emr_hpo --hyperparams hyperparams.json --config_p ![image.png](../../images/automl/emr_log.png) 一共做了5组实验，可以看到embedding_dim越小越好。 -- 实验目录信息(exp_dir): hdfs:///user/easy_rec_test/experiment/hpo_test_v8 +- 实验目录信息(exp_dir): hdfs:///user/easy_rec_test/experiment/hpo_test_v8 ![image.png](../../images/automl/emr_exp.png) - 如果设置了--debug，那么将会保留本地临时目录: /tmp/emr_easy_rec_hpo_1600519258 - rewrite\_\[0-4\].json定义了每组实验的参数 + rewrite\_[0-4].json定义了每组实验的参数 ![image.png](../../images/automl/emr_json.png) ![image.png](../../images/automl/emr_json2.png) diff --git a/docs/source/automl/hpo_pai.md b/docs/source/automl/hpo_pai.md index ac2310a41..9f547dc88 100644 --- a/docs/source/automl/hpo_pai.md +++ b/docs/source/automl/hpo_pai.md @@ -45,9 +45,9 @@ accessKeyID = xxx accessKeySecret= xxx ``` -- --bucket oss_bucket +- --bucket oss_bucket -- --role_arn acs:ram::xxx:role/xxx +- --role_arn acs:ram::xxx:role/xxx pai tensorflow 任务访问oss数据的钥匙，[获取方式](https://help.aliyun.com/document_detail/190477.html?spm=h2-url-1)。 @@ -57,13 +57,13 @@ accessKeySecret= xxx - --exp_dir 调优目录, oss上的目录 -- --config_path easyrec训练配置文件 +- --config_path easyrec训练配置文件 -- --metric_name 调优的指标，默认是auc，其它可选指标[参考](../eval.md) +- --metric_name 调优的指标，默认是auc，其它可选指标[参考](../eval.md) -- --max_parallel 同一时刻可以并行跑的实验数目 +- --max_parallel 同一时刻可以并行跑的实验数目 -- --total_trial_num 总共跑多少组实验 +- --total_trial_num 总共跑多少组实验 - --is_outer 内部pai还是外部pai @@ -89,7 +89,7 @@ accessKeySecret= xxx ] ``` -- name: easy_rec pipeline_config里面的参数名称，注意要用全路径 +- name: easy_rec pipeline_config里面的参数名称，注意要用全路径 ``` feature_config.features[input_names[0]=field1].embedding_dim ``` diff --git a/docs/source/automl/hpo_res.md b/docs/source/automl/hpo_res.md index ef3abe85d..a6aa23bca 100644 --- a/docs/source/automl/hpo_res.md +++ b/docs/source/automl/hpo_res.md @@ -1,6 +1,6 @@ ## 调优结果 -在运行实验后，可以在命令行界面中找到如下的Web界面地址 ：\[Your IP\]:\[Your Port\] +在运行实验后，可以在命令行界面中找到如下的Web界面地址 ：\[Your IP\]:[Your Port] ![image.png](../../images/automl/pai_nni_create.jpg) ### 查看概要页面 diff --git a/docs/source/automl/pai_nni_hpo.md b/docs/source/automl/pai_nni_hpo.md index a6e03b6b6..1aa5e8a75 100644 --- a/docs/source/automl/pai_nni_hpo.md +++ b/docs/source/automl/pai_nni_hpo.md @@ -106,8 +106,8 @@ nnictl create --config exp.yml ``` - 启动入口为exp.yml -- 通过trialCommand: python3 -m hpo_tools.core.utils.run --config=./trial.ini 连接用户的具体的启动任务。 -- 通过字段searchSpaceFile: search_space.json 连接 search_space.json; +- 通过trialCommand: python3 -m hpo_tools.core.utils.run --config=./trial.ini 连接用户的具体的启动任务。 +- 通过字段searchSpaceFile: search_space.json 连接 search_space.json; 配置案例均可以在安装目录examples/search目录下，细节请参考[HPO配置介绍](./hpo_config.md) @@ -253,8 +253,8 @@ train_config { [NNI searchSpace参考手册](https://nni.readthedocs.io/en/v2.2/Tutorial/SearchSpaceSpec.html) - {"\_type": "choice", "\_value": options}：从options中选取一个。 -- {"\_type": "randint", "\_value": \[lower, upper\]}：\[low,upper)之间选择一个随机整数。 -- {"\_type": "uniform", "\_value": \[low, high\]}：\[low,upper\]之间随机采样。 +- {"\_type": "randint", "\_value": [lower, upper]}：\[low,upper)之间选择一个随机整数。 +- {"\_type": "uniform", "\_value": [low, high]}：[low,upper]之间随机采样。 #### 高级 @@ -324,7 +324,7 @@ nnictl create --config config.yml --port=8780 - 如果NNICTL一开始成功，后续突然不成功，可以清空ECS环境中的python进程，重试 - 例如/mnt/data/project/project/exp/$experiment_id/log/nnictl_stderr.log中无报错，但是Failed to establish a new connection: \[Errno 111\] Connection refused')) + 例如/mnt/data/project/project/exp/$experiment_id/log/nnictl_stderr.log中无报错，但是Failed to establish a new connection: [Errno 111] Connection refused')) 命令：ps -ef|grep python|grep -v grep|cut -c 9-15|xargs kill -15 diff --git a/docs/source/benchmark.md b/docs/source/benchmark.md index 8a2c5348e..db54d4a57 100644 --- a/docs/source/benchmark.md +++ b/docs/source/benchmark.md @@ -62,11 +62,11 @@ - 测试结果 | model | global_step | ctr auc | masked cvr auc | ctcvr auc | 训练时间 | config | -| --------------- | ----------- | --------- | -------------- | --------- | ---- | -------------------------------------------------------------------------------------------------------------------- | -| SimpleMultiTask | 4100 | 0.592606 | | 0.6306802 | 1小时 | [simple_multi_task.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/simple_multi_task.config) | -| MMoE | 3100 | 0.5869702 | | 0.6330008 | 1小时 | [mmoe.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/mmoe.config) | -| ESMM | 800 | 0.5974812 | 0.6841141 | 0.6362526 | 3小时 | [esmm.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/esmm.config) | -| PLE | 3200 | 0.5874 | | 0.6159 | 2小时 | [ple.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/ple.config) | +| --------------- | ----------- | --------- | -------------- | --------- | -------- | -------------------------------------------------------------------------------------------------------------------- | +| SimpleMultiTask | 4100 | 0.592606 | | 0.6306802 | 1小时 | [simple_multi_task.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/simple_multi_task.config) | +| MMoE | 3100 | 0.5869702 | | 0.6330008 | 1小时 | [mmoe.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/mmoe.config) | +| ESMM | 800 | 0.5974812 | 0.6841141 | 0.6362526 | 3小时 | [esmm.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/esmm.config) | +| PLE | 3200 | 0.5874 | | 0.6159 | 2小时 | [ple.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/ple.config) | ### CENSUS diff --git a/docs/source/component/backbone.md b/docs/source/component/backbone.md index 0e600dbb8..7e64f6350 100644 --- a/docs/source/component/backbone.md +++ b/docs/source/component/backbone.md @@ -316,9 +316,9 @@ x3 = Cross()(x0, x2) MovieLens-1M数据集效果对比： -| Model | Epoch | AUC | -| ----------------- | ----- | ------ | -| DCN （内置） | 1 | 0.8576 | +| Model | Epoch | AUC | +| ------------------- | ----- | ------ | +| DCN （内置） | 1 | 0.8576 | | DCN_v2 （backbone） | 1 | 0.8770 | 备注：新实现的`Cross`组件对应了参数量更多的v2版本的DCN，而内置的DCN模型对应了v1版本的DCN。 diff --git a/docs/source/component/component.md b/docs/source/component/component.md index 8ef90b79e..baab90584 100644 --- a/docs/source/component/component.md +++ b/docs/source/component/component.md @@ -2,61 +2,61 @@ ## 1.基础组件 -| 类名 | 功能 | 说明 | 示例 | -| ----------------- | ------ | ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -| MLP | 多层感知机 | 可定制激活函数、initializer、Dropout、BN等 | [案例1](backbone.html#wide-deep) | -| Highway | 类似残差链接 | 可用来对预训练embedding做增量微调 | [highway network](../models/highway.html) | -| Gate | 门控 | 多个输入的加权求和 | [Cross Decoupling Network](../models/cdn.html#id2) | -| PeriodicEmbedding | 周期激活函数 | 数值特征Embedding | [案例5](backbone.html#dlrm-embedding) | -| AutoDisEmbedding | 自动离散化 | 数值特征Embedding | [dlrm_on_criteo_with_autodis.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/dlrm_on_criteo_with_autodis.config) | -| NaryDisEmbedding | N进制编码 | 数值特征Embedding | [dlrm_on_criteo_with_narydis.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/dlrm_on_criteo_with_narydis.config) | -| TextCNN | 文本卷积 | 提取文本序列的特征 | [text_cnn_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/text_cnn_on_movielens.config) | +| 类名 | 功能 | 说明 | 示例 | +| ----------------- | ------------ | ------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| MLP | 多层感知机 | 可定制激活函数、initializer、Dropout、BN等 | [案例1](backbone.html#wide-deep) | +| Highway | 类似残差链接 | 可用来对预训练embedding做增量微调 | [highway network](../models/highway.html) | +| Gate | 门控 | 多个输入的加权求和 | [Cross Decoupling Network](../models/cdn.html#id2) | +| PeriodicEmbedding | 周期激活函数 | 数值特征Embedding | [案例5](backbone.html#dlrm-embedding) | +| AutoDisEmbedding | 自动离散化 | 数值特征Embedding | [dlrm_on_criteo_with_autodis.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/dlrm_on_criteo_with_autodis.config) | +| NaryDisEmbedding | N进制编码 | 数值特征Embedding | [dlrm_on_criteo_with_narydis.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/dlrm_on_criteo_with_narydis.config) | +| TextCNN | 文本卷积 | 提取文本序列的特征 | [text_cnn_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/text_cnn_on_movielens.config) | **备注**：Gate组件的第一个输入是权重向量，后面的输入拼凑成一个列表，权重向量的长度应等于列表的长度 ## 2.特征交叉组件 -| 类名 | 功能 | 说明 | 示例 | -| -------------- | ---------------- | ------------ | -------------------------------------------------------------------------------------------------------------------------- | -| FM | 二阶交叉 | DeepFM模型的组件 | [案例2](backbone.html#deepfm) | -| DotInteraction | 二阶内积交叉 | DLRM模型的组件 | [案例4](backbone.html#dlrm) | -| Cross | bit-wise交叉 | DCN v2模型的组件 | [案例3](backbone.html#dcn) | -| BiLinear | 双线性 | FiBiNet模型的组件 | [fibinet_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/fibinet_on_movielens.config) | -| FiBiNet | SENet & BiLinear | FiBiNet模型 | [fibinet_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/fibinet_on_movielens.config) | +| 类名 | 功能 | 说明 | 示例 | +| -------------- | ---------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------- | +| FM | 二阶交叉 | DeepFM模型的组件 | [案例2](backbone.html#deepfm) | +| DotInteraction | 二阶内积交叉 | DLRM模型的组件 | [案例4](backbone.html#dlrm) | +| Cross | bit-wise交叉 | DCN v2模型的组件 | [案例3](backbone.html#dcn) | +| BiLinear | 双线性 | FiBiNet模型的组件 | [fibinet_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/fibinet_on_movielens.config) | +| FiBiNet | SENet & BiLinear | FiBiNet模型 | [fibinet_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/fibinet_on_movielens.config) | ## 3.特征重要度学习组件 -| 类名 | 功能 | 说明 | 示例 | -| --------- | ----------------- | ------------ | ----------------------------------------------------- | -| SENet | 建模特征重要度 | FiBiNet模型的组件 | [MMoE](../models/mmoe.html#id4) | -| MaskBlock | 建模特征重要度 | MaskNet模型的组件 | [Cross Decoupling Network](../models/cdn.html#id2) | -| MaskNet | 多个串行或并行的MaskBlock | MaskNet模型 | [DBMTL](../models/dbmtl.html#dbmtl-based-on-backbone) | -| PPNet | 参数个性化网络 | PPNet模型 | [PPNet](../models/ppnet.html#id2) | +| 类名 | 功能 | 说明 | 示例 | +| --------- | ------------------------- | ----------------- | ----------------------------------------------------- | +| SENet | 建模特征重要度 | FiBiNet模型的组件 | [MMoE](../models/mmoe.html#id4) | +| MaskBlock | 建模特征重要度 | MaskNet模型的组件 | [Cross Decoupling Network](../models/cdn.html#id2) | +| MaskNet | 多个串行或并行的MaskBlock | MaskNet模型 | [DBMTL](../models/dbmtl.html#dbmtl-based-on-backbone) | +| PPNet | 参数个性化网络 | PPNet模型 | [PPNet](../models/ppnet.html#id2) | ## 4. 序列特征编码组件 -| 类名 | 功能 | 说明 | 示例 | -| ------------------ | --------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------ | -| DIN | target attention | DIN模型的组件 | [DIN_backbone.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/din_backbone_on_taobao.config) | -| BST | transformer | BST模型的组件 | [BST_backbone.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/bst_backbone_on_taobao.config) | -| SeqAugment | 序列数据增强 | crop, mask, reorder | [CL4SRec](../models/cl4srec.html#id2) | -| Attention | Dot-product attention | Transformer模型的组件 | | -| MultiHeadAttention | Multi-head attention | Transformer模型的组件 | | -| TransformerBlock | Transformer layer | Transformer模型的组件 | | -| TransformerEncoder | Transformer encoder | Transformer模型的组件 | | -| TextEncoder | BERT 模型 | 类似BERT模型 | | +| 类名 | 功能 | 说明 | 示例 | +| ------------------ | --------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| DIN | target attention | DIN模型的组件 | [DIN_backbone.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/din_backbone_on_taobao.config) | +| BST | transformer | BST模型的组件 | [BST_backbone.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/bst_backbone_on_taobao.config) | +| SeqAugment | 序列数据增强 | crop, mask, reorder | [CL4SRec](../models/cl4srec.html#id2) | +| Attention | Dot-product attention | Transformer模型的组件 | | +| MultiHeadAttention | Multi-head attention | Transformer模型的组件 | | +| TransformerBlock | Transformer layer | Transformer模型的组件 | | +| TransformerEncoder | Transformer encoder | Transformer模型的组件 | | +| TextEncoder | BERT 模型 | 类似BERT模型 | | ## 5. 多目标学习组件 -| 类名 | 功能 | 说明 | 示例 | -| --------- | --------------------------- | --------- | ----------------------------- | -| MMoE | Multiple Mixture of Experts | MMoE模型的组件 | [案例8](backbone.html#mmoe) | -| AITMTower | AITM模型的一个tower | AITM模型的组件 | [AITM](../models/aitm.md#id2) | +| 类名 | 功能 | 说明 | 示例 | +| --------- | --------------------------- | -------------- | ----------------------------- | +| MMoE | Multiple Mixture of Experts | MMoE模型的组件 | [案例8](backbone.html#mmoe) | +| AITMTower | AITM模型的一个tower | AITM模型的组件 | [AITM](../models/aitm.md#id2) | ## 6. 辅助损失函数组件 -| 类名 | 功能 | 说明 | 示例 | -| ------------- | ---------- | --------- | ------------------------ | +| 类名 | 功能 | 说明 | 示例 | +| ------------- | -------------------- | ------------------ | -------------------------- | | AuxiliaryLoss | 用来计算辅助损失函数 | 常用在自监督学习中 | [案例7](backbone.html#id7) | # 组件详细参数 @@ -65,74 +65,74 @@ - MLP （多层感知机） -| 参数 | 类型 | 默认值 | 说明 | -| ----------------------- | ---- | ---------- | --------------------------- | -| hidden_units | list | | 各隐层单元数 | -| dropout_ratio | list | | 各隐层dropout rate | -| activation | str | relu | 每层的激活函数 | -| use_bn | bool | true | 是否使用batch normalization | -| use_final_bn | bool | true | 最后一层是否使用batch normalization | -| use_bias | bool | false | 是否使用偏置项 | -| use_final_bias | bool | false | 最后一层是否使用偏置项 | -| final_activation | str | relu | 最后一层的激活函数 | +| 参数 | 类型 | 默认值 | 说明 | +| ----------------------- | ---- | ---------- | ------------------------------------- | +| hidden_units | list | | 各隐层单元数 | +| dropout_ratio | list | | 各隐层dropout rate | +| activation | str | relu | 每层的激活函数 | +| use_bn | bool | true | 是否使用batch normalization | +| use_final_bn | bool | true | 最后一层是否使用batch normalization | +| use_bias | bool | false | 是否使用偏置项 | +| use_final_bias | bool | false | 最后一层是否使用偏置项 | +| final_activation | str | relu | 最后一层的激活函数 | | initializer | str | he_uniform | 权重初始化方法，参考keras Dense layer | | use_bn_after_activation | bool | false | 是否在激活函数之后做batch norm | - HighWay -| 参数 | 类型 | 默认值 | 说明 | -| -------------- | ------ | ---- | ------------ | -| emb_size | uint32 | None | embedding维度 | -| activation | str | gelu | 激活函数 | -| dropout_rate | float | 0 | dropout rate | -| init_gate_bias | float | -3.0 | 门控网络的bias初始值 | -| num_layers | int | 1 | 网络层数 | +| 参数 | 类型 | 默认值 | 说明 | +| -------------- | ------ | ------ | -------------------- | +| emb_size | uint32 | None | embedding维度 | +| activation | str | gelu | 激活函数 | +| dropout_rate | float | 0 | dropout rate | +| init_gate_bias | float | -3.0 | 门控网络的bias初始值 | +| num_layers | int | 1 | 网络层数 | - PeriodicEmbedding -| 参数 | 类型 | 默认值 | 说明 | -| ------------------ | ------ | ----- | ------------------------------------------------- | -| embedding_dim | uint32 | | embedding维度 | -| sigma | float | | 初始化自定义参数时的标准差，**效果敏感、小心调参** | -| add_linear_layer | bool | true | 是否在embedding之后添加额外的层 | -| linear_activation | str | relu | 额外添加的层的激活函数 | -| output_tensor_list | bool | false | 是否同时输出embedding列表 | -| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | +| 参数 | 类型 | 默认值 | 说明 | +| ------------------ | ------ | ------ | -------------------------------------------------------------- | +| embedding_dim | uint32 | | embedding维度 | +| sigma | float | | 初始化自定义参数时的标准差，**效果敏感、小心调参** | +| add_linear_layer | bool | true | 是否在embedding之后添加额外的层 | +| linear_activation | str | relu | 额外添加的层的激活函数 | +| output_tensor_list | bool | false | 是否同时输出embedding列表 | +| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | - AutoDisEmbedding -| 参数 | 类型 | 默认值 | 说明 | -| ------------------ | ------ | ----- | ------------------------------------------------- | -| embedding_dim | uint32 | | embedding维度 | -| num_bins | uint32 | | 虚拟分桶数量 | -| keep_prob | float | 0.8 | 残差链接的权重 | -| temperature | float | | softmax函数的温度系数 | -| output_tensor_list | bool | false | 是否同时输出embedding列表 | -| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | +| 参数 | 类型 | 默认值 | 说明 | +| ------------------ | ------ | ------ | -------------------------------------------------------------- | +| embedding_dim | uint32 | | embedding维度 | +| num_bins | uint32 | | 虚拟分桶数量 | +| keep_prob | float | 0.8 | 残差链接的权重 | +| temperature | float | | softmax函数的温度系数 | +| output_tensor_list | bool | false | 是否同时输出embedding列表 | +| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | - NaryDisEmbedding -| 参数 | 类型 | 默认值 | 说明 | -| ------------------ | ------ | ----- | --------------------------------------------------- | -| embedding_dim | uint32 | | embedding维度 | -| carries | list | | N-ary 数值特征需要编码的进制列表 | -| multiplier | float | 1.0 | 针对float类型的特征，放大`multiplier`倍再取整后进行进制编码 | -| intra_ary_pooling | string | sum | 同一进制的不同位的数字embedding如何聚合成最终的embedding, 可选：sum, mean | -| num_replicas | uint32 | 1 | 每个特征输出多少个embedding表征 | -| output_tensor_list | bool | false | 是否同时输出embedding列表 | -| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | +| 参数 | 类型 | 默认值 | 说明 | +| ------------------ | ------ | ------ | ------------------------------------------------------------------------- | +| embedding_dim | uint32 | | embedding维度 | +| carries | list | | N-ary 数值特征需要编码的进制列表 | +| multiplier | float | 1.0 | 针对float类型的特征，放大`multiplier`倍再取整后进行进制编码 | +| intra_ary_pooling | string | sum | 同一进制的不同位的数字embedding如何聚合成最终的embedding, 可选：sum, mean | +| num_replicas | uint32 | 1 | 每个特征输出多少个embedding表征 | +| output_tensor_list | bool | false | 是否同时输出embedding列表 | +| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | 备注：该组件依赖自定义Tensorflow OP，可能在某些版本的TF上无法使用 - TextCNN -| 参数 | 类型 | 默认值 | 说明 | -| ------------------- | ------------ | ---- | ---------------- | -| num_filters | list | | 卷积核个数列表 | -| filter_sizes | list | | 卷积核步长列表 | -| activation | string | relu | 卷积操作的激活函数 | -| pad_sequence_length | uint32 | | 序列补齐或截断的长度 | -| mlp | MLP | | protobuf message | +| 参数 | 类型 | 默认值 | 说明 | +| ------------------- | ------------ | ------ | -------------------- | +| num_filters | list | | 卷积核个数列表 | +| filter_sizes | list | | 卷积核步长列表 | +| activation | string | relu | 卷积操作的激活函数 | +| pad_sequence_length | uint32 | | 序列补齐或截断的长度 | +| mlp | MLP | | protobuf message | 备注：pad_sequence_length 参数必须要配置，否则模型predict的分数可能不稳定 @@ -140,128 +140,128 @@ - FM -| 参数 | 类型 | 默认值 | 说明 | -| ----------- | ---- | ----- | -------------------------- | -| use_variant | bool | false | 是否使用FM的变体：所有二阶交叉项直接输出，而不求和 | +| 参数 | 类型 | 默认值 | 说明 | +| ----------- | ---- | ------ | -------------------------------------------------- | +| use_variant | bool | false | 是否使用FM的变体：所有二阶交叉项直接输出，而不求和 | - DotInteraction -| 参数 | 类型 | 默认值 | 说明 | -| ---------------- | ---- | ----- | ------------------------------------ | -| self_interaction | bool | false | 是否运行特征自己与自己交叉 | -| skip_gather | bool | false | 一个优化开关，设置为true，可以提高运行速度，但需要占用更多的内存空间 | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------- | ---- | ------ | -------------------------------------------------------------------- | +| self_interaction | bool | false | 是否运行特征自己与自己交叉 | +| skip_gather | bool | false | 一个优化开关，设置为true，可以提高运行速度，但需要占用更多的内存空间 | - Cross -| 参数 | 类型 | 默认值 | 说明 | -| ------------------ | ------ | ---------------- | ------------------------------------------------------------------------------------------------------------------------- | -| projection_dim | uint32 | None | 使用矩阵分解降低计算开销，把大的权重矩阵分解为两个小的矩阵相乘，projection_dim是第一个小矩阵的列数，也是第二个小矩阵的行数 | -| diag_scale | float | 0 | used to increase the diagonal of the kernel W by `diag_scale`, that is, W + diag_scale * I, where I is an identity matrix | -| use_bias | bool | true | whether to add a bias term for this layer. | -| kernel_initializer | string | truncated_normal | Initializer to use on the kernel matrix | -| bias_initializer | string | zeros | Initializer to use on the bias vector | -| kernel_regularizer | string | None | Regularizer to use on the kernel matrix | -| bias_regularizer | string | None | Regularizer to use on bias vector | +| 参数 | 类型 | 默认值 | 说明 | +| ------------------ | ------ | ---------------- | -------------------------------------------------------------------------------------------------------------------------- | +| projection_dim | uint32 | None | 使用矩阵分解降低计算开销，把大的权重矩阵分解为两个小的矩阵相乘，projection_dim是第一个小矩阵的列数，也是第二个小矩阵的行数 | +| diag_scale | float | 0 | used to increase the diagonal of the kernel W by `diag_scale`, that is, W + diag_scale * I, where I is an identity matrix | +| use_bias | bool | true | whether to add a bias term for this layer. | +| kernel_initializer | string | truncated_normal | Initializer to use on the kernel matrix | +| bias_initializer | string | zeros | Initializer to use on the bias vector | +| kernel_regularizer | string | None | Regularizer to use on the kernel matrix | +| bias_regularizer | string | None | Regularizer to use on bias vector | - Bilinear -| 参数 | 类型 | 默认值 | 说明 | -| ---------------- | ------ | ----------- | ---------- | -| type | string | interaction | 双线性类型 | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------- | ------ | ----------- | ---------------- | +| type | string | interaction | 双线性类型 | | use_plus | bool | true | 是否使用plus版本 | -| num_output_units | uint32 | | 输出size | +| num_output_units | uint32 | | 输出size | - FiBiNet -| 参数 | 类型 | 默认值 | 说明 | -| -------- | -------- | --- | ---------------- | -| bilinear | Bilinear | | protobuf message | -| senet | SENet | | protobuf message | -| mlp | MLP | | protobuf message | +| 参数 | 类型 | 默认值 | 说明 | +| -------- | -------- | ------ | ---------------- | +| bilinear | Bilinear | | protobuf message | +| senet | SENet | | protobuf message | +| mlp | MLP | | protobuf message | ## 3.特征重要度学习组件 - SENet -| 参数 | 类型 | 默认值 | 说明 | -| --------------------- | ------ | ---- | ------------------ | -| reduction_ratio | uint32 | 4 | 隐层单元数量缩减倍数 | -| num_squeeze_group | uint32 | 2 | 压缩分组数量 | -| use_skip_connection | bool | true | 是否使用残差连接 | -| use_output_layer_norm | bool | true | 是否在输出层使用layer norm | +| 参数 | 类型 | 默认值 | 说明 | +| --------------------- | ------ | ------ | -------------------------- | +| reduction_ratio | uint32 | 4 | 隐层单元数量缩减倍数 | +| num_squeeze_group | uint32 | 2 | 压缩分组数量 | +| use_skip_connection | bool | true | 是否使用残差连接 | +| use_output_layer_norm | bool | true | 是否在输出层使用layer norm | - MaskBlock -| 参数 | 类型 | 默认值 | 说明 | -| ---------------- | ------ | ---- | ------------------------------- | -| output_size | uint32 | | 输出层单元数 | -| reduction_factor | float | | 隐层单元数缩减因子 | -| aggregation_size | uint32 | | 隐层单元数 | -| input_layer_norm | bool | true | 输入是否需要做layer norm | -| projection_dim | uint32 | | 用两个小矩阵相乘代替原来的输入-隐层权重矩阵，配置小矩阵的维数 | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------- | ------ | ------ | ------------------------------------------------------------- | +| output_size | uint32 | | 输出层单元数 | +| reduction_factor | float | | 隐层单元数缩减因子 | +| aggregation_size | uint32 | | 隐层单元数 | +| input_layer_norm | bool | true | 输入是否需要做layer norm | +| projection_dim | uint32 | | 用两个小矩阵相乘代替原来的输入-隐层权重矩阵，配置小矩阵的维数 | - MaskNet -| 参数 | 类型 | 默认值 | 说明 | -| ------------ | ---- | ---- | ------------- | -| mask_blocks | list | | MaskBlock结构列表 | -| use_parallel | bool | true | 是否使用并行模式 | -| mlp | MLP | 可选 | 顶部mlp | +| 参数 | 类型 | 默认值 | 说明 | +| ------------ | ---- | ------ | ----------------- | +| mask_blocks | list | | MaskBlock结构列表 | +| use_parallel | bool | true | 是否使用并行模式 | +| mlp | MLP | 可选 | 顶部mlp | - PPNet -| 参数 | 类型 | 默认值 | 说明 | -| --------------- | ------ | ----- | -------------------------------------------------- | -| mlp | MLP | | mlp 配置 | -| gate_params | GateNN | | 参数个性化Gate网络的配置 | -| mode | string | eager | 配置参数个性化是作用在MLP的每个layer的输入上还是输出上，可选：\[eager, lazy\] | -| full_gate_input | bool | true | 是否需要添加stop_gradient之后的mlp的输入作为gate网络的输入 | +| 参数 | 类型 | 默认值 | 说明 | +| --------------- | ------ | ------ | --------------------------------------------------------------------------- | +| mlp | MLP | | mlp 配置 | +| gate_params | GateNN | | 参数个性化Gate网络的配置 | +| mode | string | eager | 配置参数个性化是作用在MLP的每个layer的输入上还是输出上，可选：[eager, lazy] | +| full_gate_input | bool | true | 是否需要添加stop_gradient之后的mlp的输入作为gate网络的输入 | 其中，GateNN的参数如下： -| 参数 | 类型 | 默认值 | 说明 | -| ------------ | ------ | --------------- | ----------------------------------------- | +| 参数 | 类型 | 默认值 | 说明 | +| ------------ | ------ | ---------------------- | ----------------------------------------------------------------- | | output_dim | uint32 | mlp前一层的输出units数 | Gate网络的输出维度，eager模式下必须要配置为mlp第一层的输入units数 | -| hidden_dim | uint32 | output_dim | 隐层单元数 | -| dropout_rate | float | 0.0 | 隐层dropout rate | -| activation | str | relu | 隐层的激活函数 | -| use_bn | bool | true | 隐层是否使用batch normalization | +| hidden_dim | uint32 | output_dim | 隐层单元数 | +| dropout_rate | float | 0.0 | 隐层dropout rate | +| activation | str | relu | 隐层的激活函数 | +| use_bn | bool | true | 隐层是否使用batch normalization | ## 4. 序列特征编码组件 - SeqAugment (序列数据增强) -| 参数 | 类型 | 默认值 | 说明 | -| ------------ | ----- | --- | --------------- | -| mask_rate | float | 0.6 | 被mask掉的token比率 | -| crop_rate | float | 0.2 | 裁剪保留的token比率 | -| reorder_rate | float | 0.6 | shuffle的子序列长度占比 | +| 参数 | 类型 | 默认值 | 说明 | +| ------------ | ----- | ------ | ----------------------- | +| mask_rate | float | 0.6 | 被mask掉的token比率 | +| crop_rate | float | 0.2 | 裁剪保留的token比率 | +| reorder_rate | float | 0.6 | shuffle的子序列长度占比 | - DIN -| 参数 | 类型 | 默认值 | 说明 | -| -------------------- | ------ | ------- | ------------------------- | -| attention_dnn | MLP | | attention unit mlp | +| 参数 | 类型 | 默认值 | 说明 | +| -------------------- | ------ | ------- | ----------------------------- | +| attention_dnn | MLP | | attention unit mlp | | need_target_feature | bool | true | 是否返回target item embedding | -| attention_normalizer | string | softmax | softmax or sigmoid | +| attention_normalizer | string | softmax | softmax or sigmoid | - BST -| 参数 | 类型 | 默认值 | 说明 | -| ---------------------------- | ------ | ---- | -------------------------------------- | -| hidden_size | int | | transformer 编码层单元数 | -| num_hidden_layers | int | | transformer层数 | -| num_attention_heads | int | | transformer head数 | -| intermediate_size | int | | transformer中间层单元数 | -| hidden_act | string | gelu | 隐藏层激活函数 | -| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | -| attention_probs_dropout_prob | float | 0.1 | attention层dropout rate | -| max_position_embeddings | int | 512 | 序列最大长度 | -| use_position_embeddings | bool | true | 是否使用位置编码 | -| initializer_range | float | 0.2 | 权重参数初始值的区间范围 | -| output_all_token_embeddings | bool | true | 是否输出所有token embedding | -| target_item_position | string | head | target item的插入位置，可选：head, tail, ignore | -| reserve_target_position | bool | true | 是否为target item保留一个位置 | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------------------- | ------ | ------ | ----------------------------------------------- | +| hidden_size | int | | transformer 编码层单元数 | +| num_hidden_layers | int | | transformer层数 | +| num_attention_heads | int | | transformer head数 | +| intermediate_size | int | | transformer中间层单元数 | +| hidden_act | string | gelu | 隐藏层激活函数 | +| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | +| attention_probs_dropout_prob | float | 0.1 | attention层dropout rate | +| max_position_embeddings | int | 512 | 序列最大长度 | +| use_position_embeddings | bool | true | 是否使用位置编码 | +| initializer_range | float | 0.2 | 权重参数初始值的区间范围 | +| output_all_token_embeddings | bool | true | 是否输出所有token embedding | +| target_item_position | string | head | target item的插入位置，可选：head, tail, ignore | +| reserve_target_position | bool | true | 是否为target item保留一个位置 | - Attention @@ -273,15 +273,15 @@ The calculation follows the steps: 1. Use scores to calculate a softmax distribution with shape (batch_size, Tq, Tv). 1. Use the softmax distribution to create a linear combination of value with shape (batch_size, Tq, dim). -| 参数 | 类型 | 默认值 | 说明 | -| ----------------------- | ------ | ----- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| use_scale | bool | False | If True, will create a scalar variable to scale the attention scores. | -| scale_by_dim | bool | Fasle | whether to scale by dimension | -| score_mode | string | dot | Function to use to compute attention scores, one of {"dot", "concat"}. "dot" refers to the dot product between the query and key vectors. "concat" refers to the hyperbolic tangent of the concatenation of the query and key vectors. | -| dropout | float | 0.0 | Float between 0 and 1. Fraction of the units to drop for the attention scores. | -| seed | int | None | A Python integer to use as random seed incase of dropout. | -| return_attention_scores | bool | False | if True, returns the attention scores (after masking and softmax) as an additional output argument. | -| use_causal_mask | bool | False | Set to True for decoder self-attention. Adds a mask such that position i cannot attend to positions j > i. This prevents the flow of information from the future towards the past. | +| 参数 | 类型 | 默认值 | 说明 | +| ----------------------- | ------ | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| use_scale | bool | False | If True, will create a scalar variable to scale the attention scores. | +| scale_by_dim | bool | Fasle | whether to scale by dimension | +| score_mode | string | dot | Function to use to compute attention scores, one of {"dot", "concat"}. "dot" refers to the dot product between the query and key vectors. "concat" refers to the hyperbolic tangent of the concatenation of the query and key vectors. | +| dropout | float | 0.0 | Float between 0 and 1. Fraction of the units to drop for the attention scores. | +| seed | int | None | A Python integer to use as random seed incase of dropout. | +| return_attention_scores | bool | False | if True, returns the attention scores (after masking and softmax) as an additional output argument. | +| use_causal_mask | bool | False | Set to True for decoder self-attention. Adds a mask such that position i cannot attend to positions j > i. This prevents the flow of information from the future towards the past. | > - inputs: List of the following tensors: > \- query: Query tensor of shape (batch_size, Tq, dim). @@ -293,83 +293,83 @@ The calculation follows the steps: - MultiHeadAttention -| 参数 | 类型 | 默认值 | 说明 | -| ----------------------- | ------ | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 参数 | 类型 | 默认值 | 说明 | +| ----------------------- | ------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | num_heads | uint32 | 无 | Number of attention heads. | -| key_dim | uint32 | | Size of each attention head for query and key. | -| value_dim | uint32 | | Size of each attention head for value. | -| dropout | float | 0.0 | Dropout probability. | -| use_bias | bool | true | whether the dense layers use bias vectors/matrices. | -| return_attention_scores | bool | false | whether the output should be (attention_output, attention_scores) | -| use_causal_mask | bool | false | whether to apply a causal mask to prevent tokens from attending to future tokens (e.g., used in a decoder Transformer). | -| output_shape | uint32 | | The expected shape of an output tensor, besides the batch and sequence dims. If not specified, projects back to the query feature dim (the query input's last dimension). | -| kernel_initializer | string | | Initializer for dense layer kernels. | -| bias_initializer | string | | Initializer for dense layer biases. | +| key_dim | uint32 | | Size of each attention head for query and key. | +| value_dim | uint32 | | Size of each attention head for value. | +| dropout | float | 0.0 | Dropout probability. | +| use_bias | bool | true | whether the dense layers use bias vectors/matrices. | +| return_attention_scores | bool | false | whether the output should be (attention_output, attention_scores) | +| use_causal_mask | bool | false | whether to apply a causal mask to prevent tokens from attending to future tokens (e.g., used in a decoder Transformer). | +| output_shape | uint32 | | The expected shape of an output tensor, besides the batch and sequence dims. If not specified, projects back to the query feature dim (the query input's last dimension). | +| kernel_initializer | string | | Initializer for dense layer kernels. | +| bias_initializer | string | | Initializer for dense layer biases. | - TransformerBlock Transformer encoder 的其中一个layer。 -| 参数 | 类型 | 默认值 | 说明 | -| ---------------------------- | ------ | ---- | ----------------------- | -| hidden_size | int | | transformer 编码层单元数 | -| num_attention_heads | int | | transformer head数 | -| intermediate_size | int | | transformer中间层单元数 | -| hidden_act | string | relu | 隐藏层激活函数 | -| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | -| attention_probs_dropout_prob | float | 0.0 | attention层的dropout rate | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------------------- | ------ | ------ | ------------------------- | +| hidden_size | int | | transformer 编码层单元数 | +| num_attention_heads | int | | transformer head数 | +| intermediate_size | int | | transformer中间层单元数 | +| hidden_act | string | relu | 隐藏层激活函数 | +| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | +| attention_probs_dropout_prob | float | 0.0 | attention层的dropout rate | - TransformerEncoder -| 参数 | 类型 | 默认值 | 说明 | -| ---------------------------- | ------ | ---- | ----------------------- | -| vocab_size | uint32 | | 词汇表大小 | -| hidden_size | uint32 | | transformer 编码层单元数 | -| num_hidden_layers | uint32 | | transformer层数 | -| num_attention_heads | uint32 | | transformer head数 | -| intermediate_size | uint32 | | transformer中间层单元数 | -| hidden_act | string | relu | 隐藏层激活函数 | -| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | -| attention_probs_dropout_prob | float | 0.0 | attention层的dropout rate | -| max_position_embeddings | uint32 | 512 | 序列最大长度 | -| output_all_token_embeddings | bool | true | 是否输出所有token embedding | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------------------- | ------ | ------ | --------------------------- | +| vocab_size | uint32 | | 词汇表大小 | +| hidden_size | uint32 | | transformer 编码层单元数 | +| num_hidden_layers | uint32 | | transformer层数 | +| num_attention_heads | uint32 | | transformer head数 | +| intermediate_size | uint32 | | transformer中间层单元数 | +| hidden_act | string | relu | 隐藏层激活函数 | +| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | +| attention_probs_dropout_prob | float | 0.0 | attention层的dropout rate | +| max_position_embeddings | uint32 | 512 | 序列最大长度 | +| output_all_token_embeddings | bool | true | 是否输出所有token embedding | - TextEncoder BERT模型结构 -| 参数 | 类型 | 默认值 | 说明 | -| ---------------- | ------------------ | --- | ----------------------------- | -| transformer | TransformerEncoder | 无 | transformer 子组件的配置 | -| separator | string | ' ' | 文本分隔符 | -| vocab_file | string | 无 | 词汇表文件路径，不设置时使用hash获得token id | -| default_token_id | int32 | 0 | Out of vocabulary 的token的默认id | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------- | ------------------ | ------ | -------------------------------------------- | +| transformer | TransformerEncoder | 无 | transformer 子组件的配置 | +| separator | string | ' ' | 文本分隔符 | +| vocab_file | string | 无 | 词汇表文件路径，不设置时使用hash获得token id | +| default_token_id | int32 | 0 | Out of vocabulary 的token的默认id | ## 5. 多任务学习组件 - MMoE -| 参数 | 类型 | 默认值 | 说明 | -| ---------- | ------ | --- | ------------ | -| num_task | uint32 | | 任务数 | -| num_expert | uint32 | 0 | expert数量 | -| expert_mlp | MLP | 可选 | expert的mlp参数 | +| 参数 | 类型 | 默认值 | 说明 | +| ---------- | ------ | ------ | --------------- | +| num_task | uint32 | | 任务数 | +| num_expert | uint32 | 0 | expert数量 | +| expert_mlp | MLP | 可选 | expert的mlp参数 | - AITMTower -| 参数 | 类型 | 默认值 | 说明 | -| ------------- | ------ | ---- | ------------------------------ | +| 参数 | 类型 | 默认值 | 说明 | +| ------------- | ------ | ------ | --------------------------------- | | project_dim | uint32 | 可选 | attention Query, Key, Value的维度 | -| stop_gradient | bool | True | 是否需要停用对依赖的输入的梯度 | -| transfer_mlp | MLP | | transfer的mlp参数 | +| stop_gradient | bool | True | 是否需要停用对依赖的输入的梯度 | +| transfer_mlp | MLP | | transfer的mlp参数 | ## 6. 计算辅助损失函数的组件 - AuxiliaryLoss -| 参数 | 类型 | 默认值 | 说明 | -| ----------- | ------ | --- | ------------------------------------- | -| loss_type | string | | 损失函数类型，包括：l2_loss, nce_loss, info_nce | -| loss_weight | float | 1.0 | 损失函数权重 | -| temperature | float | 0.1 | info_nce & nec loss 的参数 | -| 其他 | | | 根据loss_type决定 | +| 参数 | 类型 | 默认值 | 说明 | +| ----------- | ------ | ------ | ----------------------------------------------- | +| loss_type | string | | 损失函数类型，包括：l2_loss, nce_loss, info_nce | +| loss_weight | float | 1.0 | 损失函数权重 | +| temperature | float | 0.1 | info_nce & nec loss 的参数 | +| 其他 | | | 根据loss_type决定 | diff --git a/docs/source/conf.py b/docs/source/conf.py index 5caeb70b6..5cc1d0f5b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,9 +13,8 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # import os -import sys - import sphinx_rtd_theme +import sys import easy_rec @@ -41,11 +40,11 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', - 'sphinx.ext.napoleon', 'recommonmark', 'sphinx_markdown_tables', - 'post_process' + 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon', 'recommonmark', 'sphinx_markdown_tables', + 'post_process' ] # Add any paths that contain templates here, relative to this directory. @@ -115,29 +114,31 @@ # -- Options for LaTeX output ------------------------------------------------ latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'easy_rec.tex', u'easy\\_rec Documentation', u'EasyRec Team', - 'manual'), + ( + master_doc, 'easy_rec.tex', u'easy\\_rec Documentation', u'EasyRec Team', + 'manual' + ), ] # -- Options for manual page output ------------------------------------------ @@ -152,8 +153,10 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'easy_rec', u'easy_rec Documentation', author, 'easy_rec', - 'One line description of project.', 'Miscellaneous'), + ( + master_doc, 'easy_rec', u'easy_rec Documentation', author, 'easy_rec', + 'One line description of project.', 'Miscellaneous' + ), ] # -- Options for Epub output ------------------------------------------------- @@ -186,6 +189,6 @@ todo_include_todos = True autodoc_default_options = { - 'member-order': 'bysource', - 'special-members': '__init__', + 'member-order': 'bysource', + 'special-members': '__init__', } diff --git a/docs/source/eval.md b/docs/source/eval.md index 06271ac7d..816053808 100644 --- a/docs/source/eval.md +++ b/docs/source/eval.md @@ -39,22 +39,22 @@ eval_config { **备注** 在多目标模型里, eval_config 的 metrics_set 无效，请使用 model 里面的 metrics_set 配置 -参见文档 \[MODEL\]->\[多目标模型\] 配置中的 model_config.\[model\].task_towers.metrics_set +参见文档 [MODEL]->[多目标模型] 配置中的 model_config.[model].task_towers.metrics_set ### Metric: | MetricClass | Example | 适用模型 | -| ------------------ | ------------------------- | --------------------------------------------------------------------- | -| Accuracy | accuracy {} | 多分类模型LossType=CLASSIFICATION, num_class > 1 | +| ------------------ | ------------------------- | ------------------------------------------------------------------------- | +| Accuracy | accuracy {} | 多分类模型LossType=CLASSIFICATION, num_class > 1 | | MeanAbsoluteError | mean_absolute_error {} | 回归模型LossType=L2_LOSS | -| RecallAtTopK | recall_at_topk {} | 多分类模型LossType=CLASSIFICATION, num_class > 1; CoMetricLearningI2I模型 | +| RecallAtTopK | recall_at_topk {} | 多分类模型LossType=CLASSIFICATION, num_class > 1; CoMetricLearningI2I模型 | | Max_F1 | max_f1 {} | 分类模型LossType=CLASSIFICATION | | MeanSquaredError | mean_squared_error{} | 回归模型LossType=L2_LOSS | -| AUC | auc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | -| GAUC | gauc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | -| SessionAUC | session_auc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | -| Precision | precision{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | -| Recall | recall{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | +| AUC | auc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | +| GAUC | gauc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | +| SessionAUC | session_auc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | +| Precision | precision{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | +| Recall | recall{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | | AvgPrecisionAtTopK | precision_at_topk{topk=5} | CoMetricLearningI2I模型专用, LossType=CIRCLE_LOSS / MULTI_SIMILARITY_LOSS | **备注** diff --git a/docs/source/export.md b/docs/source/export.md index de5e30469..f5df579b4 100644 --- a/docs/source/export.md +++ b/docs/source/export.md @@ -8,7 +8,7 @@ export_config { ``` - batch_size: 导出模型的batch_size，默认是-1，即可以接收任意batch_size -- exporter_type: 导出类型, best | final | latest | none，默认final +- exporter_type: 导出类型, best | final | latest | none，默认final - best 导出最好的模型 - final 训练结束后导出 - latest 导出最新的模型 @@ -101,11 +101,11 @@ pai -name easy_rec_ext -project algo_public - --asset_files: 需要导出的asset文件路径, 可设置多个, 逗号分隔； - 如果需要导出到assets目录的子目录下，使用`${target_path}:${source_path}`的格式；（从版本0.8.7开始支持） - e.g. '--asset_files custom_fg_lib/fg.json:oss://${bucket}/path/to/fg.json' -- 模型导出之后可以使用(EasyRecProcessor)\[./predict/在线预测.md\]部署到PAI-EAS平台 +- 模型导出之后可以使用(EasyRecProcessor)[./predict/在线预测.md]部署到PAI-EAS平台 ### 双塔召回模型 -如果是双塔召回模型(如dssm, mind等), 模型导出之后, 一般还需要进行模型切分和索引构建, 才能使用(EasyRecProcessor)\[./predict/在线预测.md\]部署到PAI-EAS上. +如果是双塔召回模型(如dssm, mind等), 模型导出之后, 一般还需要进行模型切分和索引构建, 才能使用(EasyRecProcessor)[./predict/在线预测.md]部署到PAI-EAS上. #### 模型切分 diff --git a/docs/source/faq.md b/docs/source/faq.md index 11ef98889..55212b805 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -272,7 +272,7 @@ Original error: #### 输出user塔的embedding时，输出为空 -用tfResponse.getDoubleVals("user_emb")去打印结果出来的是否返回的是\[\]，空的数组 +用tfResponse.getDoubleVals("user_emb")去打印结果出来的是否返回的是[]，空的数组 需要使用 tfResponse.getStringVals("user_emb") diff --git a/docs/source/feature/excel_config.md b/docs/source/feature/excel_config.md index 49c4def69..19900f1ba 100644 --- a/docs/source/feature/excel_config.md +++ b/docs/source/feature/excel_config.md @@ -27,12 +27,12 @@ python -m easy_rec.python.tools.create_config_from_excel --model_type multi_towe - 输入输出文件 - --train_input_path TRAIN_INPUT_PATH --eval_input_path EVAL_INPUT_PATH + --train_input_path TRAIN_INPUT_PATH --eval_input_path EVAL_INPUT_PATH - 默认数据文件(csv)列(column)分割符号是, 列(column)内部里面字符分割符号是| 可以自定义分隔符: - --column_separator $'|' --incol_separator $',' + --column_separator $'|' --incol_separator $',' - 训练数据路径 --train_input_path diff --git a/docs/source/feature/fg.md b/docs/source/feature/fg.md index f230e3f4d..a3edff8ab 100644 --- a/docs/source/feature/fg.md +++ b/docs/source/feature/fg.md @@ -23,7 +23,7 @@ FG模块在推荐系统架构中的位置如下图所示： #### 编写配置文件`fg.json` -- 包含了features配置和全局配置两个部分, 示例: +- 包含了features配置和全局配置两个部分, 示例: ```json { @@ -55,21 +55,21 @@ FG模块在推荐系统架构中的位置如下图所示： FG支持的特征变换算子与EasyRec支持的特征（`Feature Column`）之间没有严格的对应关系，大致可以参加如下表格： -| FG 算子 | EasyRec Feature Column | -| :-------------------- | :---------------------------------- | -| id_feature | IdFeature 或 TagFeature | -| raw_feature | RawFeature | -| expr_feature | RawFeature | -| combo_feature | IdFeature 或 TagFeature | +| FG 算子 | EasyRec Feature Column | +| :-------------------- | :------------------------------------ | +| id_feature | IdFeature 或 TagFeature | +| raw_feature | RawFeature | +| expr_feature | RawFeature | +| combo_feature | IdFeature 或 TagFeature | | lookup_feature | RawFeature 或 IdFeature 或 TagFeature | | match_feature | RawFeature 或 IdFeature 或 TagFeature | -| overlap_feature | RawFeature | -| sequence_feature | SequenceFeature 或 TagFeature | -| bm25_feature | RawFeature | -| kv_dot_product | RawFeature | -| tokenize_feature | TagFeature | -| text_normalizer | IdFeature | -| regex_replace_feature | IdFeature | +| overlap_feature | RawFeature | +| sequence_feature | SequenceFeature 或 TagFeature | +| bm25_feature | RawFeature | +| kv_dot_product | RawFeature | +| tokenize_feature | TagFeature | +| text_normalizer | IdFeature | +| regex_replace_feature | IdFeature | 备注：**FG的执行结果输出给EasyRec模型，两种之间是串联的关系**。 @@ -111,7 +111,7 @@ pai -name easy_rec_ext - 参数说明: [请参考](../export.md#pai) - 注意事项: - - 请检查fg.config, 保证导出的模型是支持多个placeholder的输入\[每个特征一个placeholder\] + - 请检查fg.config, 保证导出的模型是支持多个placeholder的输入[每个特征一个placeholder] ``` export_config { @@ -123,7 +123,7 @@ pai -name easy_rec_ext - 如果有设置feature_config.features.max_partitions, 请加入下面的命令重置: - - -Dedit_config_json='{"feature_config.features\[:\].max_partitions":1}'进行修改, 可以获得更好的性能 + - -Dedit_config_json='{"feature_config.features[:].max_partitions":1}'进行修改, 可以获得更好的性能 #### 特征筛选 @@ -187,11 +187,11 @@ eascmd -i -k -e update ali_rec_rn - tables: item特征存储在hologres表里面, 支持分多个表存储 - key: 必填, itemId列的名字; - value: 可选，需要加载的列名, 多个列名之间用逗号(,)分割; - - condition: 可选，where子语句支持筛选item, 如itemId \< 10000; + - condition: 可选，where子语句支持筛选item, 如itemId < 10000; - timekey: 可选，用于item的增量更新，支持的格式: timestamp和int - static: 可选, 表示是静态特征，不用周期性更新 - 支持多个item表, 如果多张表有重复的列, 后面的表覆盖前面的表 - - "tables": \[{"key":"table1", ...},{"key":"table2", ...}\] + - "tables": [{"key":"table1", ...},{"key":"table2", ...}] - 如果多张表有重复的列，后面的表将覆盖前面的表 - hologres表里面每一列存储一个item特征,示例: @@ -202,7 +202,7 @@ eascmd -i -k -e update ali_rec_rn
- remote_type: Item特征数据源, 目前支持：`hologres`, `none` - hologres：通过SQL接口进行数据读取和写入，适用于海量数据的存储和查询 - - none: 不使用Item特征缓存，item特征通过请求传入，此时tables应设置为\[\] + - none: 不使用Item特征缓存，item特征通过请求传入，此时tables应设置为[] - storage: 将oss的模型目录mount到docker的指定目录下 - mount_path: docker内部的挂载路径, 与示例保持一致即可 - 配置了storage就不需要配置model_path了 diff --git a/docs/source/feature/fg_docs/ComboFeature.md b/docs/source/feature/fg_docs/ComboFeature.md index 5e0495cdd..5d706eafa 100644 --- a/docs/source/feature/fg_docs/ComboFeature.md +++ b/docs/source/feature/fg_docs/ComboFeature.md @@ -18,11 +18,11 @@ combo_feature是多个字段（或表达式）的组合（即笛卡尔积），i ^\]表示多值分隔符，注意这是一个符号，其ASCII编码是"\\x1D"，而不是两个符号 -| user:age_class的取值 | item:item_id的取值 | 输出的feature | -| ----------------- | --------------- | ---------------------------------------------------------------------------------------------------------- | -| 123 | 45678 | comb_u_age_item_123_45678 | -| abc, bcd | 45678 | comb_u_age_item_abc_45678, comb_u_age_item_bcd_45678 | -| abc, bcd | 12345^\]45678 | comb_u_age_item_abc_12345, comb_u_age_item_abc_45678, comb_u_age_item_bcd_12345, comb_u_age_item_bcd_45678 | +| user:age_class的取值 | item:item_id的取值 | 输出的feature | +| -------------------- | ------------------ | ---------------------------------------------------------------------------------------------------------- | +| 123 | 45678 | comb_u_age_item_123_45678 | +| abc, bcd | 45678 | comb_u_age_item_abc_45678, comb_u_age_item_bcd_45678 | +| abc, bcd | 12345^\]45678 | comb_u_age_item_abc_12345, comb_u_age_item_abc_45678, comb_u_age_item_bcd_12345, comb_u_age_item_bcd_45678 | 输出的feature个数等于 diff --git a/docs/source/feature/fg_docs/IdFeature.md b/docs/source/feature/fg_docs/IdFeature.md index 27e2f4c54..b2b06e3f7 100644 --- a/docs/source/feature/fg_docs/IdFeature.md +++ b/docs/source/feature/fg_docs/IdFeature.md @@ -14,23 +14,23 @@ id_feature表示离散特征, 包含单值离散特征和多值离散特征. } ``` -| 字段名 | 含义 | -| ------------ | ----------------------------------------------------------------------------- | -| feature_name | 必选项，feature_name会被当做最终输出的feature的前缀 | -| expression | 必选项，expression描述该feature所依赖的字段来源 | +| 字段名 | 含义 | +| ------------ | ------------------------------------------------------------------------------------------------------------ | +| feature_name | 必选项，feature_name会被当做最终输出的feature的前缀 | +| expression | 必选项，expression描述该feature所依赖的字段来源 | | need_prefix | 可选项，true表示会拼上feature_name作为前缀，false表示不拼，默认为true，通常在shared_embedding的场景会用false | ## 示例: 下面以item侧的特征is_main作为案例来说明在不同配置下特征的输入和输出: -| 类型 | item:is_main的取值 | 输出的feature | -| -------- | --------------- | ---------------------------------- | -| int64_t | 100 | item_is_main_100 | -| double | 5.2 | item_is_main_5（小数部分会被截取） | -| string | abc | item_is_main_abc | -| 多值string | abc^\]bcd | item_is_main_abc^Citem_is_main_bcd | -| 多值int | 123^\]456 | item_is_main_123^Citem_is_main_456 | +| 类型 | item:is_main的取值 | 输出的feature | +| ---------- | ------------------ | ---------------------------------- | +| int64_t | 100 | item_is_main_100 | +| double | 5.2 | item_is_main_5（小数部分会被截取） | +| string | abc | item_is_main_abc | +| 多值string | abc^\]bcd | item_is_main_abc^Citem_is_main_bcd | +| 多值int | 123^\]456 | item_is_main_123^Citem_is_main_456 | - ^\]表示多值分隔符，注意这是一个符号，其ASCII编码是"\\x1D", 也可以写作"\\u001d" - ^C是FG encode之后输出的特征值的分隔符, 其ASCII编码是"\\x03" diff --git a/docs/source/feature/fg_docs/OverLapFeature.md b/docs/source/feature/fg_docs/OverLapFeature.md index b6396db61..534e12ddb 100644 --- a/docs/source/feature/fg_docs/OverLapFeature.md +++ b/docs/source/feature/fg_docs/OverLapFeature.md @@ -6,16 +6,16 @@ 离线推荐使用1.3.56-SNAPSHOT这个版本。 ps: 写fg的时候注意维度，title的维度要大于或等于query的问题（简单来说就是如果title是user特征，那query也只能是user特征，user特征的batch size为1，商品特征的batch size为商品数） -| 方式 | 描述 | 备注 | -| ------------------- | ----------------------------------------------- | ------------------ | -| common_word | 计算query与title间重复term，并输出为fg_common1_common2 | 重复数不超过query term数 | -| diff_word | 计算query与title间不重复term，并输出为fg_diff1_diff2 | 不重复数不超过query term数 | -| query_common_ratio | 计算query与title间重复term数占query中term比例,乘以10取下整 | 取值为\[0,10\] | -| title_common_ratio | 计算query与title间重复term数占title中term比例,乘以100取下整 | 取值为\[0,100\] | -| is_contain | 计算query是否全部包含在title中，保持顺序 | 0表示未包含，1表示包含 | -| is_equal | 计算query是否与title完全相同 | 0表示不完全相同，1表示完全相同 | -| common_word_divided | 计算query与title间重复term，并输出为fg_common1, fg_common2 | 重复数不超过query term数 | -| diff_word_divided | 计算query与title间不重复term，并输出为fg_diff1, fg_diff2 | 重复数不超过query term数 | +| 方式 | 描述 | 备注 | +| ------------------- | ----------------------------------------------------------- | ------------------------------ | +| common_word | 计算query与title间重复term，并输出为fg_common1_common2 | 重复数不超过query term数 | +| diff_word | 计算query与title间不重复term，并输出为fg_diff1_diff2 | 不重复数不超过query term数 | +| query_common_ratio | 计算query与title间重复term数占query中term比例,乘以10取下整 | 取值为[0,10] | +| title_common_ratio | 计算query与title间重复term数占title中term比例,乘以100取下整 | 取值为[0,100] | +| is_contain | 计算query是否全部包含在title中，保持顺序 | 0表示未包含，1表示包含 | +| is_equal | 计算query是否与title完全相同 | 0表示不完全相同，1表示完全相同 | +| common_word_divided | 计算query与title间重复term，并输出为fg_common1, fg_common2 | 重复数不超过query term数 | +| diff_word_divided | 计算query与title间不重复term，并输出为fg_diff1, fg_diff2 | 重复数不超过query term数 | ## 配置方法 @@ -30,14 +30,14 @@ } ``` -| 字段名 | 含义 | -| ------------ | -------------------------------------------------------------------------------------- | -| feature_type | 必选项，描述改feature的类型 | -| feature_name | 必选项，feature_name会被当做最终输出的feature的前缀 | -| query | 必选项，query依赖的表, attr1是一个多值string, 多值string的分隔符使用chr(29) | -| title | 必选项，title依赖的表, attr2是一个多值string | +| 字段名 | 含义 | +| ------------ | ------------------------------------------------------------------------------------------------- | +| feature_type | 必选项，描述改feature的类型 | +| feature_name | 必选项，feature_name会被当做最终输出的feature的前缀 | +| query | 必选项，query依赖的表, attr1是一个多值string, 多值string的分隔符使用chr(29) | +| title | 必选项，title依赖的表, attr2是一个多值string | | method | 可填common_word, diff_word, query_common_ratio, title_common_ratio, is_contain， 对应上图五种方式 | -| separator | 输出结果中的分割字符，不填写我们默认为\_ ，但也可以用户自己定制，具体看例子 | +| separator | 输出结果中的分割字符，不填写我们默认为\_ ，但也可以用户自己定制，具体看例子 | ## 例子 diff --git a/docs/source/feature/fg_docs/RawFeature.md b/docs/source/feature/fg_docs/RawFeature.md index 45f732f78..f81b3ced1 100644 --- a/docs/source/feature/fg_docs/RawFeature.md +++ b/docs/source/feature/fg_docs/RawFeature.md @@ -15,22 +15,22 @@ raw_feature表示连续值特征, 支持数值int、float、double等数值类 } ``` -| 字段名 | 含义 | -| --------------- | ------------------------------------------------------------ | -| feature_name | 必选项, 特征名 | +| 字段名 | 含义 | +| --------------- | -------------------------------------------------------------------------------------- | +| feature_name | 必选项, 特征名 | | expression | 必选项，expression描述该feature所依赖的字段来源, 来源必须是user、item、context中的一种 | -| value_dimension | 可选项，默认值为1，表示输出的字段的维度 | -| normalizer | 可选项，归一化方法，详见后文 | +| value_dimension | 可选项，默认值为1，表示输出的字段的维度 | +| normalizer | 可选项，归一化方法，详见后文 | ## 示例 ^\]表示多值分隔符，注意这是一个符号，其ASCII编码是"\\x1D"，而不是两个符号 -| 类型 | item:ctr的取值 | 输出的feature | -| ------- | ----------- | ---------------------------------------------- | -| int64_t | 100 | (ctr, 100) | -| double | 100.1 | (ctr, 100.1) | -| 多值int | 123^\]456 | (ctr, (123,456)) (注意，输入字段必须与配置的dimension维度一致) | +| 类型 | item:ctr的取值 | 输出的feature | +| ------- | -------------- | -------------------------------------------------------------- | +| int64_t | 100 | (ctr, 100) | +| double | 100.1 | (ctr, 100.1) | +| 多值int | 123^\]456 | (ctr, (123,456)) (注意，输入字段必须与配置的dimension维度一致) | ## Normalizer diff --git a/docs/source/feature/odl_sample.md b/docs/source/feature/odl_sample.md index 0fe6cdda8..02f623b07 100644 --- a/docs/source/feature/odl_sample.md +++ b/docs/source/feature/odl_sample.md @@ -202,7 +202,7 @@ 1. label生成, 目前提供三种[python udf](http://easyrec.oss-cn-beijing.aliyuncs.com/deploy/label_gen.zip): - playtime: sum_over(events, 'playtime') - - click: has_event(events, 'click') + - click: has_event(events, 'click') - min_over / max_over: min_over(events, 'eventTime') - 可以使用python自定义任意udf, [参考文档](https://flink.apache.org/2020/04/09/pyflink-udf-support-flink.html) - udf 上传([vvp-console](https://vvp.console.aliyun.com/)): @@ -299,7 +299,7 @@ ``` - ttl(miliseconds)的设置考虑两个因素: - odl_sample_with_lbl相对请求时间request_time的延迟 - - ttl \< 相对延迟, 就会有样本丢失 + - ttl < 相对延迟, 就会有样本丢失 - 统计相对延迟: - 将odl_sample_with_lbl / odl_callback_log落到MaxCompute - 按request_id join 计算ts的差异 diff --git a/docs/source/feature/rtp_fg.md b/docs/source/feature/rtp_fg.md index 40da4852e..41160cd91 100644 --- a/docs/source/feature/rtp_fg.md +++ b/docs/source/feature/rtp_fg.md @@ -12,7 +12,7 @@ #### 编写配置 [fg.json](https://easyrec.oss-cn-beijing.aliyuncs.com/rtp_fg/fg.json) -- 包含了features配置和全局配置两个部分, 示例: +- 包含了features配置和全局配置两个部分, 示例: ```json { @@ -44,7 +44,7 @@ - 如果设成false, 转换成EasyRec的config时会转成IdFeature, 可以减少字符串分割的开销 - - 多值分隔符使用chr(29)\[ctrl+v ctrl+\], 即"\\u001D". + - 多值分隔符使用chr(29)[ctrl+v ctrl+], 即"\\u001D". - [多值类型说明](./fg_docs/mutiValues.md) @@ -132,13 +132,13 @@ - 双层查找, 根据category和item_id查找value. - - match feature里面多值分隔符可以使用chr(29) (ctrl+v ctrl+\])或者逗号\[,\]， 如: + - match feature里面多值分隔符可以使用chr(29) (ctrl+v ctrl+\])或者逗号[,]， 如: ``` 50011740^107287172:0.2^]36806676:0.3^]122572685:0.5|50006842^16788816:0.1^]10122:0.2^]29889:0.3^]30068:19 ``` - - needWeighting: 生成特征权重，即kv格式, kv之间用\[ctrl+v ctrl+e\]分割, 转换成TagFeature. + - needWeighting: 生成特征权重，即kv格式, kv之间用[ctrl+v ctrl+e]分割, 转换成TagFeature. - [sequence_feature](./fg_docs/SequenceFeature.md) @@ -293,7 +293,7 @@ tunnel download taobao_fg_test_out taobao_fg_test_out.txt -fd=';'; | ----- | ------- | ------- | ------------------------------------------------------------------------------------------------------------- | | 0 | 336811 | 100002 | user_id_100002^Bcms_segid_5^Bcms_group_id_2^Bage_level_2^Bpvalue_level_1^Bshopping_level_3^Boccupation_1^B... | -#### 从配置文件\[fg.json\]生成EasyRec的config +#### 从配置文件[fg.json]生成EasyRec的config 从Git克隆EasyRec @@ -325,7 +325,7 @@ python -m easy_rec.python.tools.convert_rtp_fg --label is_product_detail is_pur - --separator: feature之间的分隔符, 默认是CTRL_B(u0002) -- --selected_cols: 指定输入列，包括label、\[sample_weight\]和features，其中label可以指定多列，表示要使用多个label(一般是多任务模型), 最后一列必须是features, 如: +- --selected_cols: 指定输入列，包括label、[sample_weight]和features，其中label可以指定多列，表示要使用多个label(一般是多任务模型), 最后一列必须是features, 如: ``` label0,label1,sample_weight,features @@ -392,7 +392,7 @@ pai -name easy_rec_ext - 参数说明: [请参考](../export.md#pai) - 注意事项: - - 请检查fg.config, 保证导出的模型是支持多个placeholder的输入\[每个特征一个placeholder\] + - 请检查fg.config, 保证导出的模型是支持多个placeholder的输入[每个特征一个placeholder] ``` export_config { @@ -404,7 +404,7 @@ pai -name easy_rec_ext - 如果有设置feature_config.features.max_partitions, 请加入下面的命令重置: - - -Dedit_config_json='{"feature_config.features\[:\].max_partitions":1}'进行修改, 可以获得更好的性能 + - -Dedit_config_json='{"feature_config.features[:].max_partitions":1}'进行修改, 可以获得更好的性能 #### 增加特征 @@ -499,7 +499,7 @@ eascmd -i -k -e update ali_rec_rn - tables: item特征存储在hologres表里面, 支持分多个表存储 - key: 必填, itemId列的名字; - value: 可选，需要加载的列名, 多个列名之间用逗号(,)分割; - - condition: 可选，where子语句支持筛选item, 如itemId \< 10000; + - condition: 可选，where子语句支持筛选item, 如itemId < 10000; - timekey: 可选，用于item的增量更新，支持的格式: timestamp和int - static: 可选, 表示是静态特征，不用周期性更新 - 支持多个item表, 如果多张表有重复的列, 后面的表覆盖前面的表 diff --git a/docs/source/feature/rtp_native.md b/docs/source/feature/rtp_native.md index 8774041c7..e36815ab8 100644 --- a/docs/source/feature/rtp_native.md +++ b/docs/source/feature/rtp_native.md @@ -6,7 +6,7 @@ #### 编写RTP特征配置 [fg.json](https://easyrec.oss-cn-beijing.aliyuncs.com/rtp_fg/fg.json) -- 包含了features配置和全局配置两个部分, 示例: +- 包含了features配置和全局配置两个部分, 示例: ```json { @@ -34,7 +34,7 @@ - is_multi: id_feature是否是多值属性 - 默认是false, 转换成EasyRec的config时会转成IdFeature - 如果设成true, 转换成EasyRec的config时会转成TagFeature. - - 多值分隔符使用chr(29)\[ctrl+v ctrl+\]. + - 多值分隔符使用chr(29)[ctrl+v ctrl+]. - num_buckets: 当输入是unsigned int类型的时候，并且输入有界的时候，可以指定num_bucket为输入的最大值. - hash_bucket_size: 对应EasyRec feature_config.features的hash_bucket_size。hash_bucket方式是目前RTP唯一支持的embedding方式 - embedding_dimension/embedding_dim: 对应EasyRec feature_config.features里面的embedding_dim. @@ -102,13 +102,13 @@ - [MatchFeature](http://easyrec.oss-cn-beijing.aliyuncs.com/fg_docs/MatchFeature.pdf) - 双层查找, 根据category和item_id查找value. - - match Feature里面多值分隔符可以使用chr(29) (ctrl+v ctrl+\])或者逗号\[,\]， 如: + - match Feature里面多值分隔符可以使用chr(29) (ctrl+v ctrl+\])或者逗号[,]， 如: ``` 50011740^107287172:0.2^]36806676:0.3^]122572685:0.5|50006842^16788816:0.1^]10122:0.2^]29889:0.3^]30068:19 ``` - - needWeighting: 生成特征权重，即kv格式, kv之间用\[ctrl+v ctrl+e\]分割, 转换成TagFeature. + - needWeighting: 生成特征权重，即kv格式, kv之间用[ctrl+v ctrl+e]分割, 转换成TagFeature. - [OverLapFeature](http://easyrec.oss-cn-beijing.aliyuncs.com/fg_docs/OverLapFeature.pdf) @@ -120,7 +120,7 @@ - combiner: 默认是mean, 也可以是sum. - 影响数据生成和 EasyRec feature_config 生成, 主要是多值Feature. - [多值类型说明](http://easyrec.oss-cn-beijing.aliyuncs.com/fg_docs/%E5%A4%9A%E5%80%BC%E7%B1%BB%E5%9E%8B.pdf) - - 多值feature使用chr(29)\[ctrl+v ctrl+\]\]作为分隔符. + - 多值feature使用chr(29)[ctrl+v ctrl+]\]作为分隔符. - 全局配置说明: diff --git a/docs/source/incremental_train.md b/docs/source/incremental_train.md index c2c82f782..6f72a9bf7 100644 --- a/docs/source/incremental_train.md +++ b/docs/source/incremental_train.md @@ -57,4 +57,4 @@ pai -name easy_rec_ext -project algo_public ``` - bizdate在dataworks里面是业务日期，一般是运行日期的前一天。 -- fine_tune_checkpoint: fine_tune_checkpoint的路径，可以指定具体的checkpoint，也可以指定一个目录，将自动定位目录里面最新的checkpoint。 +- fine_tune_checkpoint: fine_tune_checkpoint的路径，可以指定具体的checkpoint，也可以指定一个目录，将自动定位目录里面最新的checkpoint。 diff --git a/docs/source/intro.md b/docs/source/intro.md index 65844ac8d..f735da430 100644 --- a/docs/source/intro.md +++ b/docs/source/intro.md @@ -1,6 +1,6 @@ # EasyRec简介 -🎉 See our ongoing recommendation framework **[TorchEasyRec](https://github.com/alibaba/TorchEasyRec) !** 🎉 This evolution of EasyRec is built on **PyTorch**, featuring **GPU acceleration** and **hybrid parallelism** for enhanced performance. +🎉 See our ongoing recommendation framework **[TorchEasyRec](https://github.com/alibaba/TorchEasyRec) !** 🎉 This evolution of EasyRec is built on **PyTorch**, featuring **GPU acceleration** and **hybrid parallelism** for enhanced performance. ## What is EasyRec? @@ -30,7 +30,7 @@ EasyRec implements state of the art machine learning models used in common recom ### Simple to config - Flexible feature config and simple model config -- Efficient and robust feature generation\[used in taobao\] +- Efficient and robust feature generation[used in taobao] - Nice web interface in development ### It is smart diff --git a/docs/source/models/aitm.md b/docs/source/models/aitm.md index 6f4c57d7b..036567e50 100644 --- a/docs/source/models/aitm.md +++ b/docs/source/models/aitm.md @@ -8,7 +8,7 @@ 1. (a) Expert-Bottom pattern。如 [MMoE](mmoe.md) 1. (b) Probability-Transfer pattern。如 [ESMM](esmm.md) -1. (c) Adaptive Information Transfer Multi-task (AITM) framework. +1. (c) Adaptive Information Transfer Multi-task (AITM) framework. 两个特点： diff --git a/docs/source/models/cmbf.md b/docs/source/models/cmbf.md index 78cf5404a..7761cf7ce 100644 --- a/docs/source/models/cmbf.md +++ b/docs/source/models/cmbf.md @@ -105,7 +105,7 @@ model_config: { - 配置一个名为`general`的feature_group，包含需要做跨模态attention的常规特征，这些特征的`embedding_dim`必须相同。 - 配置一个名为`text`的feature_group，包含需要做跨模态attention的不定长文本序列特征，这些特征的`embedding_dim`必须相同。 - 注意：CMBF 模型要求所有文本侧（包括`text`和`general`两个特征组）输入特征的 embedding_dim 保持一致。 - - \[可选\] 配置一个名为`other`的feature_group，包含不需要做跨模态attention的其他特征，如各类统计特征。 + - [可选] 配置一个名为`other`的feature_group，包含不需要做跨模态attention的其他特征，如各类统计特征。 - cmbf/config: CMBF 模型相关的参数 @@ -115,7 +115,7 @@ model_config: { - text_multi_head_num: 文本单模态学习模块中的 head 数量，默认为1 - image_head_size: 单模态学习模块中的图像tower，multi-headed self-attention的每个head的size - text_head_size: 单模态学习模块中的文本tower，multi-headed self-attention的每个head的size - - image_feature_patch_num: \[可选，默认值为1\] 当只有一个image feature时生效，表示该图像特征是一个复合embedding，维度为`image_feature_patch_num * embedding_size`。 + - image_feature_patch_num: [可选，默认值为1] 当只有一个image feature时生效，表示该图像特征是一个复合embedding，维度为`image_feature_patch_num * embedding_size`。 - image_self_attention_layer_num: 单模态学习模块中的图像tower，multi-headed self-attention的层数；当只有一个图像特征时，可设置为0 - text_self_attention_layer_num: 单模态学习模块中的文本tower，multi-headed self-attention的层数 - cross_modal_layer_num: 跨模态融合模块的层数，建议设在1到5之间，默认为1 @@ -127,7 +127,7 @@ model_config: { - use_position_embeddings: bool, default is true；是否为文本序列添加位置编码 - max_position_embeddings: 文本序列的最大位置，当`use_position_embeddings`为true时，必须配置；并且必须大于或等于所有特征配置`max_seq_len`的最大值 - text_seq_emb_dropout_prob: 文本序列embedding的dropout概率 - - other_feature_dnn: \[可选\] 其他特征的MLP网络配置 + - other_feature_dnn: [可选] 其他特征的MLP网络配置 - cmbf/final_dnn: 输出模块的MLP网络配置 diff --git a/docs/source/models/dbmtl.md b/docs/source/models/dbmtl.md index aa4015aa7..7c6483a2b 100644 --- a/docs/source/models/dbmtl.md +++ b/docs/source/models/dbmtl.md @@ -342,7 +342,7 @@ model_config: { - text_multi_head_num: 文本单模态学习模块中的 head 数量，默认为1 - image_head_size: 单模态学习模块中的图像tower，multi-headed self-attention的每个head的size - text_head_size: 单模态学习模块中的文本tower，multi-headed self-attention的每个head的size - - image_feature_patch_num: \[可选，默认值为1\] 当只有一个image feature时生效，表示该图像特征是一个复合embedding，维度为`image_feature_patch_num * embedding_size`。 + - image_feature_patch_num: [可选，默认值为1] 当只有一个image feature时生效，表示该图像特征是一个复合embedding，维度为`image_feature_patch_num * embedding_size`。 - image_self_attention_layer_num: 单模态学习模块中的图像tower，multi-headed self-attention的层数 - text_self_attention_layer_num: 单模态学习模块中的文本tower，multi-headed self-attention的层数 - cross_modal_layer_num: 跨模态融合模块的层数，建议设在1到5之间，默认为1 @@ -354,7 +354,7 @@ model_config: { - use_position_embeddings: bool, default is true；是否为文本序列添加位置编码 - max_position_embeddings: 文本序列的最大位置，当`use_position_embeddings`为true时，必须配置；并且必须大于或等于所有特征配置`max_seq_len`的最大值 - text_seq_emb_dropout_prob: 文本序列embedding的dropout概率 - - other_feature_dnn: \[可选\] 其他特征的MLP网络配置 + - other_feature_dnn: [可选] 其他特征的MLP网络配置 - 其余与dbmtl一致 #### DBMTL+UNITER @@ -461,7 +461,7 @@ model_config: { - hidden_dropout_prob: multi-headed attention模块中FC layer的dropout概率 - use_position_embeddings: bool, default is true；是否为文本序列添加位置编码 - max_position_embeddings: 文本序列的最大位置，当`use_position_embeddings`为true时，必须配置；并且必须大于或等于所有特征配置`max_seq_len`的最大值 - - other_feature_dnn: \[可选\] 其他特征的MLP网络配置 + - other_feature_dnn: [可选] 其他特征的MLP网络配置 - 其余与dbmtl一致 diff --git a/docs/source/models/deepfm.md b/docs/source/models/deepfm.md index 7dfe58418..c1c8ecc5f 100644 --- a/docs/source/models/deepfm.md +++ b/docs/source/models/deepfm.md @@ -49,7 +49,7 @@ model_config:{ - model_class: 'DeepFM', 不需要修改 - feature_groups: 需要两个feature_group: wide group和deep group, **group name不能变** -- deepfm: deepfm相关的参数 +- deepfm: deepfm相关的参数 - dnn: deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - wide_output_dim: wide部分输出的大小 diff --git a/docs/source/models/dlrm.md b/docs/source/models/dlrm.md index 66ad84e69..58f0617a5 100644 --- a/docs/source/models/dlrm.md +++ b/docs/source/models/dlrm.md @@ -2,7 +2,7 @@ ### 简介 -DLRM(Deep Learning Recommendation Model for Personalization and Recommendation Systems\[Facebook\])是一种DNN模型, 支持使用连续值特征(price/age/...)和ID类特征(user_id/item_id/...), 并对特征之间的交互(interaction)进行了建模(基于内积的方式). +DLRM(Deep Learning Recommendation Model for Personalization and Recommendation Systems[Facebook])是一种DNN模型, 支持使用连续值特征(price/age/...)和ID类特征(user_id/item_id/...), 并对特征之间的交互(interaction)进行了建模(基于内积的方式). ``` output: diff --git a/docs/source/models/dssm.md b/docs/source/models/dssm.md index 243f57a92..1d7a9e206 100644 --- a/docs/source/models/dssm.md +++ b/docs/source/models/dssm.md @@ -55,7 +55,7 @@ model_config:{ - dnn: deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - id: 指定user_id/item_id列 -- simi_func: 向量相似度函数，包括\[COSINE, INNER_PRODUCT, EUCLID\]，默认COSINE，建议使用INNER_PRODUCT +- simi_func: 向量相似度函数，包括[COSINE, INNER_PRODUCT, EUCLID]，默认COSINE，建议使用INNER_PRODUCT - embedding_regularization: 对embedding部分加regularization，防止overfit 支持的metric_set包括: diff --git a/docs/source/models/dssm_neg_sampler.md b/docs/source/models/dssm_neg_sampler.md index 14aa95288..bf8ecac70 100644 --- a/docs/source/models/dssm_neg_sampler.md +++ b/docs/source/models/dssm_neg_sampler.md @@ -99,7 +99,7 @@ model_config:{ - dnn: deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - id: 指定user_id/item_id列 -- simi_func: 向量相似度函数，包括\[COSINE, INNER_PRODUCT, EUCLID\]，默认COSINE，建议使用INNER_PRODUCT +- simi_func: 向量相似度函数，包括[COSINE, INNER_PRODUCT, EUCLID]，默认COSINE，建议使用INNER_PRODUCT - scale_simi: 是否自动缩放相似度便于loss计算，建议设置成false - loss_type: 目前只支持SOFTMAX_CROSS_ENTROPY - embedding_regularization: 对embedding部分加regularization，防止overfit diff --git a/docs/source/models/esmm.md b/docs/source/models/esmm.md index c12c78dae..959232ae1 100644 --- a/docs/source/models/esmm.md +++ b/docs/source/models/esmm.md @@ -94,7 +94,7 @@ model_config: { - feature_groups: 支持多组feature_group - esmm: esmm相关的参数 - groups - - input tower的input必须和feature_groups的group_name对应 + - input tower的input必须和feature_groups的group_name对应 - dnn deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - cvr_tower @@ -114,7 +114,7 @@ model_config: { ESMM模型输出的值有以下几项： - "logits\_" / "probs\_" + ctr_tower的tower_name -- "logits\_" / "probs\_" / "y\_" + cvr_tower的tower_name +- "logits\_" / "probs\_" / "y\_" + cvr_tower的tower_name - "probs_ctcvr" / "y_ctcvr" ESMM模型的指标有以下几项： diff --git a/docs/source/models/loss.md b/docs/source/models/loss.md index dc2139a48..8e5d4485b 100644 --- a/docs/source/models/loss.md +++ b/docs/source/models/loss.md @@ -4,28 +4,28 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 ### 使用单个损失函数 -| 损失函数 | 说明 | -| ------------------------------------------ | ---------------------------------------------------------- | -| CLASSIFICATION | 分类Loss，二分类为sigmoid_cross_entropy；多分类为softmax_cross_entropy | -| L2_LOSS | 平方损失 | -| SIGMOID_L2_LOSS | 对sigmoid函数的结果计算平方损失 | -| CROSS_ENTROPY_LOSS | log loss 负对数损失 | -| BINARY_CROSS_ENTROPY_LOSS | 仅用在知识蒸馏中的BCE损失 | -| KL_DIVERGENCE_LOSS | 仅用在知识蒸馏中的KL散度损失 | -| CIRCLE_LOSS | CoMetricLearningI2I模型专用 | -| MULTI_SIMILARITY_LOSS | CoMetricLearningI2I模型专用 | -| SOFTMAX_CROSS_ENTROPY_WITH_NEGATIVE_MINING | 自动负采样版本的多分类softmax_cross_entropy，用在二分类任务中 | -| BINARY_FOCAL_LOSS | 支持困难样本挖掘和类别平衡的focal loss | -| PAIR_WISE_LOSS | 以优化全局AUC为目标的rank loss | -| PAIRWISE_FOCAL_LOSS | pair粒度的focal loss, 支持自定义pair分组 | -| PAIRWISE_LOGISTIC_LOSS | pair粒度的logistic loss, 支持自定义pair分组 | -| PAIRWISE_HINGE_LOSS | pair粒度的hinge loss, 支持自定义pair分组 | -| JRC_LOSS | 二分类 + listwise ranking loss | -| F1_REWEIGHTED_LOSS | 可以调整二分类召回率和准确率相对权重的损失函数，可有效对抗正负样本不平衡问题 | -| ORDER_CALIBRATE_LOSS | 使用目标依赖关系校正预测结果的辅助损失函数，详见[AITM](aitm.md)模型 | -| LISTWISE_RANK_LOSS | listwise的排序损失 | -| LISTWISE_DISTILL_LOSS | 用来蒸馏给定list排序的损失函数，与listwise rank loss 比较类似 | -| ZILN_LOSS | LTV预测任务的损失函数（num_class必须设置为3） | +| 损失函数 | 说明 | +| ------------------------------------------ | ---------------------------------------------------------------------------- | +| CLASSIFICATION | 分类Loss，二分类为sigmoid_cross_entropy；多分类为softmax_cross_entropy | +| L2_LOSS | 平方损失 | +| SIGMOID_L2_LOSS | 对sigmoid函数的结果计算平方损失 | +| CROSS_ENTROPY_LOSS | log loss 负对数损失 | +| BINARY_CROSS_ENTROPY_LOSS | 仅用在知识蒸馏中的BCE损失 | +| KL_DIVERGENCE_LOSS | 仅用在知识蒸馏中的KL散度损失 | +| CIRCLE_LOSS | CoMetricLearningI2I模型专用 | +| MULTI_SIMILARITY_LOSS | CoMetricLearningI2I模型专用 | +| SOFTMAX_CROSS_ENTROPY_WITH_NEGATIVE_MINING | 自动负采样版本的多分类softmax_cross_entropy，用在二分类任务中 | +| BINARY_FOCAL_LOSS | 支持困难样本挖掘和类别平衡的focal loss | +| PAIR_WISE_LOSS | 以优化全局AUC为目标的rank loss | +| PAIRWISE_FOCAL_LOSS | pair粒度的focal loss, 支持自定义pair分组 | +| PAIRWISE_LOGISTIC_LOSS | pair粒度的logistic loss, 支持自定义pair分组 | +| PAIRWISE_HINGE_LOSS | pair粒度的hinge loss, 支持自定义pair分组 | +| JRC_LOSS | 二分类 + listwise ranking loss | +| F1_REWEIGHTED_LOSS | 可以调整二分类召回率和准确率相对权重的损失函数，可有效对抗正负样本不平衡问题 | +| ORDER_CALIBRATE_LOSS | 使用目标依赖关系校正预测结果的辅助损失函数，详见[AITM](aitm.md)模型 | +| LISTWISE_RANK_LOSS | listwise的排序损失 | +| LISTWISE_DISTILL_LOSS | 用来蒸馏给定list排序的损失函数，与listwise rank loss 比较类似 | +| ZILN_LOSS | LTV预测任务的损失函数（num_class必须设置为3） | - ZILN_LOSS：使用时模型有3个可选的输出（在多目标任务重，输出名有一个目标相关的后缀） - probs: 预估的转化概率 @@ -235,7 +235,7 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 #### 根据样本label来设定损失函数权重 在某个目标的tower里配置`task_space_indicator_label`这个字段，标记一个 label 的名字， -如果这个label的值大于0, 则 loss 权重 \*`in_task_space_weight`; 否则 loss权重 * `out_task_space_weight`。 +如果这个label的值大于0, 则 loss 权重 \*`in_task_space_weight`; 否则 loss权重 * `out_task_space_weight`。 `out_task_space_weight`的值改为0.0，则可实现`Masked Loss`。 diff --git a/docs/source/models/mind.md b/docs/source/models/mind.md index 8a52d6a83..2ae30138a 100644 --- a/docs/source/models/mind.md +++ b/docs/source/models/mind.md @@ -129,7 +129,7 @@ model_config:{ - time_id取值的方式可参考: - - 训练数据: Math.round((2 * Math.log1p((labelTime - itemTime) / 60.) / Math.log(2.))) + 1 + - 训练数据: Math.round((2 * Math.log1p((labelTime - itemTime) / 60.) / Math.log(2.))) + 1 - inference: Math.round((2 * Math.log1p((currentTime - itemTime) / 60.) / Math.log(2.))) + 1 - 此处的时间(labelTime, itemTime, currentTime) 为seconds @@ -137,7 +137,7 @@ model_config:{ - 使用增量训练，增量训练可以防止负采样的穿越。 -- 使用HPO对squash_pow\[0.1 - 1.0\]和simi_pow\[10 - 100\]进行搜索调优。 +- 使用HPO对squash_pow[0.1 - 1.0]和simi_pow[10 - 100]进行搜索调优。 - 要看的指标是召回率，准确率和兴趣损失，三个指标要一起看。 diff --git a/docs/source/models/multi_tower.md b/docs/source/models/multi_tower.md index 9e9d343e5..f65675956 100644 --- a/docs/source/models/multi_tower.md +++ b/docs/source/models/multi_tower.md @@ -80,9 +80,9 @@ model_config: { - group_name: 可以根据实际情况取 - wide_deep: 必须是DEEP - losses: 可选，可以选择同时配置两个loss函数，并且为每个loss配置不同的权重 - - loss_type: CLASSIFICATION \[默认值\] 二分类的sigmoid cross entropy loss - - loss_type: PAIR_WISE_LOSS \[可选\] 以优化AUC为主要目标的 pairwise rank loss - - loss_type: F1_REWEIGHTED_LOSS \[可选\] 可以调节二分类模型recall/precision相对权重的loss; 注意不要与`loss_type: CLASSIFICATION`同时使用 + - loss_type: CLASSIFICATION [默认值] 二分类的sigmoid cross entropy loss + - loss_type: PAIR_WISE_LOSS [可选] 以优化AUC为主要目标的 pairwise rank loss + - loss_type: F1_REWEIGHTED_LOSS [可选] 可以调节二分类模型recall/precision相对权重的loss; 注意不要与`loss_type: CLASSIFICATION`同时使用 - f1_reweight_loss: 可以调节二分类模型`recall/precision`相对权重的损失函数 - f1_beta_square: 大于1的值会导致模型更关注`recall`，小于1的值会导致模型更关注`precision` - F1 分数，又称平衡F分数（balanced F Score），它被定义为精确率和召回率的调和平均数。 diff --git a/docs/source/models/rocket_launching.md b/docs/source/models/rocket_launching.md index 172319e6d..b97af8ed1 100644 --- a/docs/source/models/rocket_launching.md +++ b/docs/source/models/rocket_launching.md @@ -59,7 +59,7 @@ model_config: { - hidden_units: dnn每一层的channel数目，即神经元的数目 - booster_dnn: 助推器网络的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - - light_dnn: 轻量网络的参数配置 + - light_dnn: 轻量网络的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - feature_based_distillation：是否配置特征蒸馏（默认不配） - feature_distillation_function：中间层相似度衡量指标（COSINE; EUCLID; 默认COSINE） diff --git a/docs/source/models/uniter.md b/docs/source/models/uniter.md index 5af92baba..534f37851 100644 --- a/docs/source/models/uniter.md +++ b/docs/source/models/uniter.md @@ -100,7 +100,7 @@ model_config: { - 配置一个名为`general`的feature_group，包含需要做跨模态attention的常规特征，这些特征的`embedding_dim`必须相同。 - 配置一个名为`text`的feature_group，包含需要做跨模态attention的不定长文本序列特征，这些特征的`embedding_dim`必须相同。 - 注意：CMBF 模型要求所有文本侧（包括`text`和`general`两个特征组）输入特征的 embedding_dim 保持一致。 - - \[可选\] 配置一个名为`other`的feature_group，包含不需要做跨模态attention的其他特征，如各类统计特征。 + - [可选] 配置一个名为`other`的feature_group，包含不需要做跨模态attention的其他特征，如各类统计特征。 - uniter/config: UNITER 模型相关的参数 @@ -113,7 +113,7 @@ model_config: { - hidden_dropout_prob: multi-headed attention模块中FC layer的dropout概率 - use_position_embeddings: bool, default is true；是否为文本序列添加位置编码 - max_position_embeddings: 文本序列的最大位置，当`use_position_embeddings`为true时，必须配置；并且必须大于或等于所有特征配置`max_seq_len`的最大值 - - other_feature_dnn: \[可选\] 其他特征的MLP网络配置 + - other_feature_dnn: [可选] 其他特征的MLP网络配置 - uniter/final_dnn: 输出模块的MLP网络配置 diff --git a/docs/source/models/wide_and_deep.md b/docs/source/models/wide_and_deep.md index a225b58d6..d09dc6ed9 100644 --- a/docs/source/models/wide_and_deep.md +++ b/docs/source/models/wide_and_deep.md @@ -52,7 +52,7 @@ model_config:{ 需要两个feature_group: wide group和deep group, **group name不能变** -- wide_and_deep: wide_and_deep 相关的参数 +- wide_and_deep: wide_and_deep 相关的参数 - dnn: deep part的参数配置 diff --git a/docs/source/online_train.md b/docs/source/online_train.md index 3af5d05cf..4c4436c7b 100644 --- a/docs/source/online_train.md +++ b/docs/source/online_train.md @@ -84,7 +84,7 @@ datahub_eval_input{ - akSecret: datahub access_secret -- region: endpoint +- region: endpoint - 注意必须是http的，不能是https，vpc里面只通80端口，不通443端口 @@ -140,7 +140,7 @@ train_config { } ``` -- dense_save_steps: dense参数发送的频率 +- dense_save_steps: dense参数发送的频率 - sparse_save_steps: sparse参数发送的频率 - incr_update: 保存增量更新，可以选fs和kafka: - fs: @@ -325,5 +325,5 @@ train_config { ## A/B实验 -- 推荐引擎: 在推荐引擎\[如PAI-REC\]里面配置一个新的实验，更新[PAI-EAS服务配置](http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/pairec/docs/pairec/html/config/algo.html) +- 推荐引擎: 在推荐引擎[如PAI-REC]里面配置一个新的实验，更新[PAI-EAS服务配置](http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/pairec/docs/pairec/html/config/algo.html) - 报表: 天级报表, 小时级报表 diff --git "a/docs/source/predict/Local \347\246\273\347\272\277\351\242\204\346\265\213.md" "b/docs/source/predict/Local \347\246\273\347\272\277\351\242\204\346\265\213.md" index 40046b86d..1ae0f38d4 100644 --- "a/docs/source/predict/Local \347\246\273\347\272\277\351\242\204\346\265\213.md" +++ "b/docs/source/predict/Local \347\246\273\347\272\277\351\242\204\346\265\213.md" @@ -52,7 +52,7 @@ predictor = Predictor('model/export/') 1. list 格式 1. dict 格式 - 输出是list of dict，dict里面包含一个字段y，即score: 范围在\[0, 1\]之间 + 输出是list of dict，dict里面包含一个字段y，即score: 范围在[0, 1]之间 ``` # interface 1, input is a list of fields, the order of fields diff --git "a/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" "b/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" index dd867a165..cf7082baf 100644 --- "a/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" +++ "b/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" @@ -54,12 +54,12 @@ pai -name easy_rec_ext - 二分类模型(要求num_class=1)，导出字段:logits、probs，对应: sigmoid之前的值/概率 - 回归模型，导出字段: y，对应: 预测值 - 多分类模型(num_class > 1)，导出字段: - - logits: string(json), softmax之前的vector, shape\[num_class\] - - probs: string(json), softmax之后的vector, shape\[num_class\] + - logits: string(json), softmax之前的vector, shape[num_class] + - probs: string(json), softmax之后的vector, shape[num_class] - 如果一个分类目标是is_click, 输出概率的变量名称是probs_is_click - 多目标模型中有一个回归目标是paytime，那么输出回归预测分的变量名称是：y_paytime - - logits_y: logits\[y\], float, 类别y对应的softmax之前的概率 - - probs_y: probs\[y\], float, 类别y对应的概率 + - logits_y: logits[y], float, 类别y对应的softmax之前的概率 + - probs_y: probs[y], float, 类别y对应的概率 - y: 类别id, = argmax(probs_y), int, 概率最大的类别 - 示例: ```sql diff --git a/docs/source/predict/processor.md b/docs/source/predict/processor.md index 213063940..754d55130 100644 --- a/docs/source/predict/processor.md +++ b/docs/source/predict/processor.md @@ -24,8 +24,8 @@ FeatureGenerator作为算子嵌入, 和TFModel联合优化，主要的优化点 - 在线推理User Feature的BatchSize = 1, 因此只需要计算一次, Tile可以放到UserFeature EmbeddingLookUp的后面，和EmbeddingLookUp融合起来 - EmbeddingLookup OpFusion - embedding lookup涉及到GatherV2 / SparseSegmentMean等多个小op, 使用OpFusion可以减少kernel launch以及Tensor内存分配和拷贝的开销 - - embedding lookup的过程需要用到比较多的SparseSegment Mean/Sum操作, 使用avx指令优化向量运算 -- MatchFeature / LookupFeature优化 + - embedding lookup的过程需要用到比较多的SparseSegment Mean/Sum操作, 使用avx指令优化向量运算 +- MatchFeature / LookupFeature优化 - 使用string_view代替std::string，减少string拷贝的开销 - Sequence特征优化 - sequence特征建模通常需要带上side info才能取得比较好的效果, 将side info放在请求中传递过来会带来通信的开销, EasyRec Processor在客户侧缓存了item特征, 因此在请求中只传递item_id sequence, side info sequence通过item_id查找客户侧缓存构建. @@ -39,7 +39,7 @@ FeatureGenerator作为算子嵌入, 和TFModel联合优化，主要的优化点 - 特征: - +
id_featureraw_featurelookup_featuresequence_feature
67 170 756 main seq num: 4 | sideinfo seq num: 32 | max seq len: 50
@@ -52,7 +52,7 @@ FeatureGenerator作为算子嵌入, 和TFModel联合优化，主要的优化点 - CPU型号: [IceLake](https://help.aliyun.com/document_detail/68564.html#p-zpg-gvj-g91) - 测试结果: - +
@@ -63,7 +63,7 @@ FeatureGenerator作为算子嵌入, 和TFModel联合优化，主要的优化点 - 特征数: -
CPU利用率QPSAVG RTTP99
优化前96 20 247ms 333ms
优化后91 55 86ms 113ms
+
id_feature raw_feature lookup_feature match_feature
306 77 60 1000
@@ -76,7 +76,7 @@ FeatureGenerator作为算子嵌入, 和TFModel联合优化，主要的优化点 - CPU型号: [IceLake](https://help.aliyun.com/document_detail/68564.html#p-zpg-gvj-g91) - 测试结果: - +
diff --git a/docs/source/quick_start/designer_tutorial.md b/docs/source/quick_start/designer_tutorial.md index 66a22d15c..168e18bf5 100644 --- a/docs/source/quick_start/designer_tutorial.md +++ b/docs/source/quick_start/designer_tutorial.md @@ -24,33 +24,33 @@ PAI-Designer（Studio 2.0）是基于云原生架构Pipeline Service（PAIFlow ### 输入桩配置 -| 输入桩（从左到右） | 限制数据类型 | 对应PAI命令参数 | 是否必选 | -| ---------- | ----------- | ------------------------------------------------------- | ---- | -| 训练表 | MaxCompute表 | `train_tables` | 是 | -| 评估表 | MaxCompute表 | `eval_tables` | 否 | -| checkpoint | OSS存储的模型 | `edit_config_json`中的`train_config.fine_tune_checkpoint` | 否 | -| 分箱表 | MaxCompute表 | `boundary_table` | 否 | +| 输入桩（从左到右） | 限制数据类型 | 对应PAI命令参数 | 是否必选 | +| ------------------ | ------------- | --------------------------------------------------------- | -------- | +| 训练表 | MaxCompute表 | `train_tables` | 是 | +| 评估表 | MaxCompute表 | `eval_tables` | 否 | +| checkpoint | OSS存储的模型 | `edit_config_json`中的`train_config.fine_tune_checkpoint` | 否 | +| 分箱表 | MaxCompute表 | `boundary_table` | 否 | ### 右侧参数说明 -| 页签 | 参数 | 是否必选 | 描述 | 默认值 | -| ---- | ---------------------- | ---- | ------------------------------------------------------------------------------------------------------ | ------------ | -| 参数设置 | 模型路径 | 否 | 对应PAI命令参数`model_dir` | 工作流自动设置的工作路径 | -| 参数设置 | EasyRec配置 | 是 | 在下方编辑框填写config配置，保存至指定的OSS路径下，对应PAI命令参数`config` | | -| 参数设置 | 指定算法版本 | 否 | 点开高级选项后，可以自定义EasyRec的执行版本。请先参考文档[EasyRec版本更新](../release.md)上传对应版本的tar包到OSS，在这个参数中选中上传的文件。对应参数`script` | 空 | -| 执行调优 | ps数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 2 | -| 执行调优 | ps CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | -| 执行调优 | ps Memory数量（MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | -| 执行调优 | Worker数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | -| 执行调优 | Worker CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | -| 执行调优 | Worker Memory用量（单位为MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | -| 执行调优 | Worker GPU卡数 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 0 | +| 页签 | 参数 | 是否必选 | 描述 | 默认值 | +| -------- | ----------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| 参数设置 | 模型路径 | 否 | 对应PAI命令参数`model_dir` | 工作流自动设置的工作路径 | +| 参数设置 | EasyRec配置 | 是 | 在下方编辑框填写config配置，保存至指定的OSS路径下，对应PAI命令参数`config` | | +| 参数设置 | 指定算法版本 | 否 | 点开高级选项后，可以自定义EasyRec的执行版本。请先参考文档[EasyRec版本更新](../release.md)上传对应版本的tar包到OSS，在这个参数中选中上传的文件。对应参数`script` | 空 | +| 执行调优 | ps数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 2 | +| 执行调优 | ps CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | +| 执行调优 | ps Memory数量（MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | +| 执行调优 | Worker数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | +| 执行调优 | Worker CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | +| 执行调优 | Worker Memory用量（单位为MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | +| 执行调优 | Worker GPU卡数 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 0 | ### 输出桩配置 -| 输出桩（从左到右） | 数据类型 | 对应PAI命令参数 | -| --------- | -------- | ------------ | -| 输出模型 | OSS存储的模型 | `model_dir ` | +| 输出桩（从左到右） | 数据类型 | 对应PAI命令参数 | +| ------------------ | ------------- | --------------- | +| 输出模型 | OSS存储的模型 | `model_dir ` | ### 对应PAI命令 @@ -63,30 +63,30 @@ PAI-Designer（Studio 2.0）是基于云原生架构Pipeline Service（PAIFlow ### 输入桩配置 -| 输入桩（从左到右） | 限制数据类型 | 对应PAI命令参数 | 是否必选 | -| --------- | ----------- | ----------------- | ---- | -| 输入模型 | OSS存储的模型 | `saved_model_dir` | 是 | -| 输入表 | MaxCompute表 | `input_table` | 是 | +| 输入桩（从左到右） | 限制数据类型 | 对应PAI命令参数 | 是否必选 | +| ------------------ | ------------- | ----------------- | -------- | +| 输入模型 | OSS存储的模型 | `saved_model_dir` | 是 | +| 输入表 | MaxCompute表 | `input_table` | 是 | ### 右侧参数说明 -| 页签 | 参数 | 是否必选 | 描述 | 默认值 | -| ---- | ---------------------- | ---- | -------------------------------------------------------------------------------------------------------------- | ----------------- | -| 参数设置 | 输入选择列 | 否 | 从输入表选择特征列给到预测模型，不能与排除列同时使用 | - | -| 参数设置 | 排除列 | 否 | 预测模型不需要使用的输入列，不能和输入选择列同时使用 | - | -| 参数设置 | 输出保留列 | 否 | 在预测结构表中原样输出的列 | - | -| 参数设置 | 预测详情输出列 | 否 | 选择预测模型的输出到MaxCompute表的映射，细节请参见[EasyRec离线预测文档](../predict/MaxCompute%20%E7%A6%BB%E7%BA%BF%E9%A2%84%E6%B5%8B.md) | 默认为"probs double" | -| 参数设置 | miniBatch的大小 | 否 | 对应参数`batch_size` | 1024 | -| 执行调优 | Worker数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | -| 执行调优 | Worker CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | -| 执行调优 | Worker Memory用量（单位为MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | -| 执行调优 | Worker GPU卡数 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 0 | +| 页签 | 参数 | 是否必选 | 描述 | 默认值 | +| -------- | ----------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------- | +| 参数设置 | 输入选择列 | 否 | 从输入表选择特征列给到预测模型，不能与排除列同时使用 | - | +| 参数设置 | 排除列 | 否 | 预测模型不需要使用的输入列，不能和输入选择列同时使用 | - | +| 参数设置 | 输出保留列 | 否 | 在预测结构表中原样输出的列 | - | +| 参数设置 | 预测详情输出列 | 否 | 选择预测模型的输出到MaxCompute表的映射，细节请参见[EasyRec离线预测文档](../predict/MaxCompute%20%E7%A6%BB%E7%BA%BF%E9%A2%84%E6%B5%8B.md) | 默认为"probs double" | +| 参数设置 | miniBatch的大小 | 否 | 对应参数`batch_size` | 1024 | +| 执行调优 | Worker数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | +| 执行调优 | Worker CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | +| 执行调优 | Worker Memory用量（单位为MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | +| 执行调优 | Worker GPU卡数 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 0 | ### 输出桩配置 -| 输出桩（从左到右） | 数据类型 | 对应PAI命令参数 | -| --------- | ----------- | --------------- | -| 输出表 | MaxCompute表 | `output_table ` | +| 输出桩（从左到右） | 数据类型 | 对应PAI命令参数 | +| ------------------ | ------------ | --------------- | +| 输出表 | MaxCompute表 | `output_table ` | ### 对应PAI命令 diff --git a/docs/source/quick_start/emr_tutorial.md b/docs/source/quick_start/emr_tutorial.md index 7d19e50fb..7e8f45db9 100644 --- a/docs/source/quick_start/emr_tutorial.md +++ b/docs/source/quick_start/emr_tutorial.md @@ -158,7 +158,7 @@ resource: - 使用el_submit提交导出任务, **el_submit**相关参数请参考[**tf_on_yarn**](https://help.aliyun.com/document_detail/93031.html) --pipeline_config_path: EasyRec配置文件 ---export_dir: 导出模型目录  +--export_dir: 导出模型目录 --checkpoint_path: 指定checkpoint，默认不指定，不指定则使用model_dir下面最新的checkpoint ### 开源TF模式 diff --git a/docs/source/quick_start/local_tutorial.md b/docs/source/quick_start/local_tutorial.md index 4a00140b4..0fd6ada0d 100644 --- a/docs/source/quick_start/local_tutorial.md +++ b/docs/source/quick_start/local_tutorial.md @@ -24,16 +24,16 @@ pip install tensorflow_probability==0.5.0 常见版本对应关系： | TensorFlow版本 | TensorFlowProbability版本 | -| ------------ | ----------------------- | -| 1.12 | 0.5.0 | -| 1.15 | 0.8.0 | -| 2.5.0 | 0.13.0 | -| 2.6.0 | 0.14.0 | -| 2.7.0 | 0.15.0 | -| 2.8.0 | 0.16.0 | -| 2.10 | 0.18.0 | -| 2.11 | 0.19.0 | -| 2.12 | 0.20.0 | +| -------------- | ------------------------- | +| 1.12 | 0.5.0 | +| 1.15 | 0.8.0 | +| 2.5.0 | 0.13.0 | +| 2.6.0 | 0.14.0 | +| 2.7.0 | 0.15.0 | +| 2.8.0 | 0.16.0 | +| 2.10 | 0.18.0 | +| 2.11 | 0.19.0 | +| 2.12 | 0.20.0 | 其他版本对应关系请查看链接：[Releases · tensorflow/probability](https://github.com/tensorflow/probability/releases)。 @@ -60,7 +60,7 @@ docker exec -it bash 可选镜像： -- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-paitf1.12-0.8.5 \[只能跑在DLC环境\] +- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-paitf1.12-0.8.5 [只能跑在DLC环境] - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-paitf1.15-0.8.5 - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15.5-0.8.5 - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15.5-gpu-0.8.5 diff --git a/docs/source/quick_start/mc_tutorial.md b/docs/source/quick_start/mc_tutorial.md index 0f6065f0c..14e6fedef 100644 --- a/docs/source/quick_start/mc_tutorial.md +++ b/docs/source/quick_start/mc_tutorial.md @@ -11,7 +11,7 @@ 输入一般是MaxCompute表: - train: pai_online_project.dwd_avazu_ctr_deepmodel_train -- test: pai_online_project.dwd_avazu_ctr_deepmodel_test +- test: pai_online_project.dwd_avazu_ctr_deepmodel_test 说明：原则上这两张表是自己odps的表，为了方便，以上提供case的两张表可在国内用户的MaxCompute项目空间中访问。 @@ -49,13 +49,13 @@ pai -name easy_rec_ext -project algo_public - -Dfine_tune_checkpoint: 可选，从checkpoint restore参数，进行finetune - 可以指定directory，将使用directory里面的最新的checkpoint. - -Dmodel_dir: 如果指定了model_dir将会覆盖config里面的model_dir，一般在周期性调度的时候使用。 -- -Darn: rolearn 注意这个的arn要替换成客户自己的。可以从dataworks的设置中查看arn;或者阿里云控制台人工智能平台PAI，左侧菜单"开通和授权"，找到全部云产品依赖->Designer->OSS->查看授权信息。 +- -Darn: rolearn 注意这个的arn要替换成客户自己的。可以从dataworks的设置中查看arn;或者阿里云控制台人工智能平台PAI，左侧菜单"开通和授权"，找到全部云产品依赖->Designer->OSS->查看授权信息。 - -Dbuckets: config所在的bucket和保存模型的bucket; 如果有多个bucket，逗号分割 - -DossHost: ossHost地址 ### 注意： -- dataworks和PAI的project一样，案例都是pai_online_project，用户需要根据自己的环境修改。如果需要使用gpu，PAI的project需要设置开通GPU。链接：[https://pai.data.aliyun.com/console?projectId=®ionId=cn-beijing#/visual](https://pai.data.aliyun.com/console?projectId=%C2%AEionId=cn-beijing#/visual) ，其中regionId可能不一致。 +- dataworks和PAI的project一样，案例都是pai_online_project，用户需要根据自己的环境修改。如果需要使用gpu，PAI的project需要设置开通GPU。链接：[https://pai.data.aliyun.com/console?projectId=®ionId=cn-beijing#/visual](https://pai.data.aliyun.com/console?projectId=%C2%AEionId=cn-beijing#/visual) ，其中regionId可能不一致。 ![mc_gpu](../../images/quick_start/mc_gpu.png) diff --git a/docs/source/quick_start/mc_tutorial_inner.md b/docs/source/quick_start/mc_tutorial_inner.md index d04dc2e8d..5870c6c1b 100644 --- a/docs/source/quick_start/mc_tutorial_inner.md +++ b/docs/source/quick_start/mc_tutorial_inner.md @@ -57,7 +57,7 @@ pai -name easy_rec_ext -project algo_public ### 注意： -- dataworks和pai的project 一样，案例都是pai_online_project，用户需要根据自己的环境修改。如果需要使用gpu，PAI的project需要设置开通GPU。链接：[https://pai.data.aliyun.com/console?projectId=®ionId=cn-beijing#/visual](https://pai.data.aliyun.com/console?projectId=%C2%AEionId=cn-beijing#/visual) ，其中regionId可能不一致。 +- dataworks和pai的project 一样，案例都是pai_online_project，用户需要根据自己的环境修改。如果需要使用gpu，PAI的project需要设置开通GPU。链接：[https://pai.data.aliyun.com/console?projectId=®ionId=cn-beijing#/visual](https://pai.data.aliyun.com/console?projectId=%C2%AEionId=cn-beijing#/visual) ，其中regionId可能不一致。 ![mc_gpu](../../images/quick_start/mc_gpu.png) diff --git a/docs/source/tf_on_yarn.md b/docs/source/tf_on_yarn.md index ba0f22972..afbdc3da0 100644 --- a/docs/source/tf_on_yarn.md +++ b/docs/source/tf_on_yarn.md @@ -18,7 +18,7 @@ Data Science版本的EMR集群支持GPU调度，所以在Core节点，推荐用 参数说明： -- -t APP_TYPE  提交的任务类型，支持三种类型的任务类型\[tensorflow-ps, tensorflow-mpi, standalone\]，三种类型要配合后面运行模式使用 +- -t APP_TYPE  提交的任务类型，支持三种类型的任务类型[tensorflow-ps, tensorflow-mpi, standalone]，三种类型要配合后面运行模式使用 > tensorflow-ps使用的是原生TensorFlow ps 类型 @@ -27,7 +27,7 @@ Data Science版本的EMR集群支持GPU调度，所以在Core节点，推荐用 > tensorflow-worker多worker模式，适用于MultiWorkerMirroredStrategy - -a APP_NAME 提交的任务名称，用户可以根据需要起名 -- -m MODE 提交的运行时环境，目前支持四种类型运行时环境\[local, virtual-env,docker\] +- -m MODE 提交的运行时环境，目前支持四种类型运行时环境[local, virtual-env,docker] > local 使用的是emr-worker上面的python运行环境，所以如果要使用一些第三方python包需要手动在所有机器上进行安装 diff --git a/docs/source/train.md b/docs/source/train.md index 19073b971..564791957 100644 --- a/docs/source/train.md +++ b/docs/source/train.md @@ -2,9 +2,9 @@ ## train_config -- log_step_count_steps: 200 # 每200步打印一行log +- log_step_count_steps: 200 # 每200步打印一行log -- optimizer_config # 优化器相关的参数 +- optimizer_config # 优化器相关的参数 ```protobuf { @@ -55,7 +55,7 @@ - 设置两个optimizer时, 第一个optimizer仅用于wide参数; - 如果要给deep embedding单独设置optimizer, 需要设置3个optimizer. -- sync_replicas: true # 是否同步训练，默认是false +- sync_replicas: true # 是否同步训练，默认是false - 使用SyncReplicasOptimizer进行分布式训练(同步模式) - 仅在train_distribute为NoStrategy时可以设置成true，其它情况应该设置为false diff --git a/docs/source/vector_retrieve.md b/docs/source/vector_retrieve.md index 8d3f7b909..578521179 100644 --- a/docs/source/vector_retrieve.md +++ b/docs/source/vector_retrieve.md @@ -21,19 +21,19 @@ pai -name easy_rec_ext -project algo_public_dev ## 参数说明 -| 参数名 | 默认值 | 参数说明 | -| --------------------- | ------------- | ------------------------------------------------------------------------- | -| query_table | 无 | 输入查询表, schema: (id bigint, vector string) | -| doc_table | 无 | 输入索引表, schema: (id bigint, vector string) | -| output_table | 无 | 输出表, schema: (query_id bigint, doc_id bigint, distance double) | +| 参数名 | 默认值 | 参数说明 | +| --------------------- | ------------- | ---------------------------------------------------------------------------------- | +| query_table | 无 | 输入查询表, schema: (id bigint, vector string) | +| doc_table | 无 | 输入索引表, schema: (id bigint, vector string) | +| output_table | 无 | 输出表, schema: (query_id bigint, doc_id bigint, distance double) | | knn_distance | inner_product | 计算距离的方法：l2、inner_product | -| knn_num_neighbours | 无 | top n, 每个query输出多少个近邻 | -| knn_feature_dims | 无 | 向量维度 | -| knn_feature_delimiter | , | 向量字符串分隔符 | -| knn_index_type | ivfflat | 向量索引类型：'flat', 'ivfflat', 'ivfpq', 'gpu_flat', 'gpu_ivfflat', 'gpu_ivfpg' | -| knn_nlist | 5 | 聚类的簇个数, number of split cluster on each worker | -| knn_nprobe | 2 | 检索时只考虑距离与输入向量最近的簇个数, number of probe part on each worker | -| knn_compress_dim | 8 | 当index_type为`ivfpq` and `gpu_ivfpq`时, 指定压缩的维度，必须为float属性个数的因子 | +| knn_num_neighbours | 无 | top n, 每个query输出多少个近邻 | +| knn_feature_dims | 无 | 向量维度 | +| knn_feature_delimiter | , | 向量字符串分隔符 | +| knn_index_type | ivfflat | 向量索引类型：'flat', 'ivfflat', 'ivfpq', 'gpu_flat', 'gpu_ivfflat', 'gpu_ivfpg' | +| knn_nlist | 5 | 聚类的簇个数, number of split cluster on each worker | +| knn_nprobe | 2 | 检索时只考虑距离与输入向量最近的簇个数, number of probe part on each worker | +| knn_compress_dim | 8 | 当index_type为`ivfpq` and `gpu_ivfpq`时, 指定压缩的维度，必须为float属性个数的因子 | ## 使用示例 diff --git a/easy_rec/__init__.py b/easy_rec/__init__.py index cbbc20e9c..ecfa54f93 100644 --- a/easy_rec/__init__.py +++ b/easy_rec/__init__.py @@ -13,11 +13,13 @@ sys.path.insert(0, parent_dir) logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) # Avoid import tensorflow which conflicts with the version used in EasyRecProcessor if 'PROCESSOR_TEST' not in os.environ: from tensorflow.python.platform import tf_logging + # In DeepRec, logger.propagate of tf_logging is False, should be True tf_logging._logger.propagate = True @@ -66,7 +68,8 @@ def get_ops_dir(): def help(): - print(""" + print( + """ 1 Train 1.1 Train 1gpu CUDA_VISIBLE_DEVICES=0 python -m easy_rec.python.train_eval @@ -111,4 +114,5 @@ def help(): for row in reader: inputs.append({ f : row[fid+1] for fid, f in enumerate(field_keys) }) output_res = self._predictor.predict(inputs, batch_size=32) -""") +""" + ) diff --git a/easy_rec/python/builders/hyperparams_builder.py b/easy_rec/python/builders/hyperparams_builder.py index a1caf537e..b9a1b9a65 100644 --- a/easy_rec/python/builders/hyperparams_builder.py +++ b/easy_rec/python/builders/hyperparams_builder.py @@ -37,16 +37,21 @@ def build_regularizer(regularizer): regularizer_oneof = regularizer.WhichOneof('regularizer_oneof') if regularizer_oneof == 'l1_regularizer': return regularizers.l1_regularizer( - scale=float(regularizer.l1_regularizer.scale)) + scale=float(regularizer.l1_regularizer.scale) + ) if regularizer_oneof == 'l2_regularizer': return regularizers.l2_regularizer( - scale=float(regularizer.l2_regularizer.scale)) + scale=float(regularizer.l2_regularizer.scale) + ) if regularizer_oneof == 'l1_l2_regularizer': return regularizers.l1_l2_regularizer( - scale_l1=float(regularizer.l1_l2_regularizer.scale_l1), - scale_l2=float(regularizer.l1_l2_regularizer.scale_l2)) + scale_l1=float(regularizer.l1_l2_regularizer.scale_l1), + scale_l2=float(regularizer.l1_l2_regularizer.scale_l2) + ) - raise ValueError('Unknown regularizer function: {}'.format(regularizer_oneof)) + raise ValueError( + 'Unknown regularizer function: {}'.format(regularizer_oneof) + ) def build_initializer(initializer): @@ -64,15 +69,20 @@ def build_initializer(initializer): initializer_oneof = initializer.WhichOneof('initializer_oneof') if initializer_oneof == 'truncated_normal_initializer': return tf.truncated_normal_initializer( - mean=initializer.truncated_normal_initializer.mean, - stddev=initializer.truncated_normal_initializer.stddev) + mean=initializer.truncated_normal_initializer.mean, + stddev=initializer.truncated_normal_initializer.stddev + ) if initializer_oneof == 'random_normal_initializer': return tf.random_normal_initializer( - mean=initializer.random_normal_initializer.mean, - stddev=initializer.random_normal_initializer.stddev) + mean=initializer.random_normal_initializer.mean, + stddev=initializer.random_normal_initializer.stddev + ) if initializer_oneof == 'glorot_normal_initializer': return tf.glorot_normal_initializer() if initializer_oneof == 'constant_initializer': return tf.constant_initializer( - [x for x in initializer.constant_initializer.consts]) - raise ValueError('Unknown initializer function: {}'.format(initializer_oneof)) + [x for x in initializer.constant_initializer.consts] + ) + raise ValueError( + 'Unknown initializer function: {}'.format(initializer_oneof) + ) diff --git a/easy_rec/python/builders/loss_builder.py b/easy_rec/python/builders/loss_builder.py index 66fd8dcd2..157d1cbfa 100644 --- a/easy_rec/python/builders/loss_builder.py +++ b/easy_rec/python/builders/loss_builder.py @@ -1,54 +1,54 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import numpy as np import tensorflow as tf +from easy_rec.python.loss.f1_reweight_loss import f1_reweight_sigmoid_cross_entropy # NOQA from easy_rec.python.loss.focal_loss import sigmoid_focal_loss_with_logits from easy_rec.python.loss.jrc_loss import jrc_loss -from easy_rec.python.loss.listwise_loss import listwise_distill_loss -from easy_rec.python.loss.listwise_loss import listwise_rank_loss -from easy_rec.python.loss.pairwise_loss import pairwise_focal_loss -from easy_rec.python.loss.pairwise_loss import pairwise_hinge_loss -from easy_rec.python.loss.pairwise_loss import pairwise_logistic_loss -from easy_rec.python.loss.pairwise_loss import pairwise_loss -from easy_rec.python.protos.loss_pb2 import LossType - +from easy_rec.python.loss.listwise_loss import listwise_distill_loss, listwise_rank_loss # NOQA +from easy_rec.python.loss.pairwise_loss import pairwise_focal_loss, pairwise_hinge_loss, pairwise_logistic_loss, pairwise_loss # NOQA from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_loss # NOQA - -from easy_rec.python.loss.f1_reweight_loss import f1_reweight_sigmoid_cross_entropy # NOQA +from easy_rec.python.protos.loss_pb2 import LossType if tf.__version__ >= '2.0': tf = tf.compat.v1 -def build(loss_type, - label, - pred, - loss_weight=1.0, - num_class=1, - loss_param=None, - **kwargs): +def build( + loss_type, + label, + pred, + loss_weight=1.0, + num_class=1, + loss_param=None, + **kwargs +): loss_name = kwargs.pop('loss_name') if 'loss_name' in kwargs else 'unknown' if loss_type == LossType.CLASSIFICATION: if num_class == 1: return tf.losses.sigmoid_cross_entropy( - label, logits=pred, weights=loss_weight, **kwargs) + label, logits=pred, weights=loss_weight, **kwargs + ) else: assert label.dtype in [tf.int32, tf.int64], \ 'label.dtype must in [tf.int32, tf.int64] when use sparse_softmax_cross_entropy.' return tf.losses.sparse_softmax_cross_entropy( - labels=label, logits=pred, weights=loss_weight, **kwargs) + labels=label, logits=pred, weights=loss_weight, **kwargs + ) elif loss_type == LossType.CROSS_ENTROPY_LOSS: return tf.losses.log_loss(label, pred, weights=loss_weight, **kwargs) elif loss_type == LossType.BINARY_CROSS_ENTROPY_LOSS: - losses = tf.keras.backend.binary_crossentropy(label, pred, from_logits=True) + losses = tf.keras.backend.binary_crossentropy( + label, pred, from_logits=True + ) return tf.reduce_mean(losses) elif loss_type in [LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS]: logging.info('%s is used' % LossType.Name(loss_type)) return tf.losses.mean_squared_error( - labels=label, predictions=pred, weights=loss_weight, **kwargs) + labels=label, predictions=pred, weights=loss_weight, **kwargs + ) elif loss_type == LossType.ZILN_LOSS: if loss_param is None: loss = zero_inflated_lognormal_loss(label, pred) @@ -59,13 +59,14 @@ def build(loss_type, class_weight = loss_param.classification_weight reg_weight = loss_param.regression_weight loss = zero_inflated_lognormal_loss( - label, - pred, - max_sigma=max_sigma, - mu_reg=mu_reg, - sigma_reg=sigma_reg, - class_weight=class_weight, - reg_weight=reg_weight) + label, + pred, + max_sigma=max_sigma, + mu_reg=mu_reg, + sigma_reg=sigma_reg, + class_weight=class_weight, + reg_weight=reg_weight + ) if np.isscalar(loss_weight) and loss_weight != 1.0: return loss * loss_weight return loss @@ -74,26 +75,28 @@ def build(loss_type, if loss_param is None: return jrc_loss(label, pred, session, name=loss_name) return jrc_loss( - label, - pred, - session, - loss_param.alpha, - loss_weight_strategy=loss_param.loss_weight_strategy, - sample_weights=loss_weight, - same_label_loss=loss_param.same_label_loss, - name=loss_name) + label, + pred, + session, + loss_param.alpha, + loss_weight_strategy=loss_param.loss_weight_strategy, + sample_weights=loss_weight, + same_label_loss=loss_param.same_label_loss, + name=loss_name + ) elif loss_type == LossType.PAIR_WISE_LOSS: session = kwargs.get('session_ids', None) margin = 0 if loss_param is None else loss_param.margin temp = 1.0 if loss_param is None else loss_param.temperature return pairwise_loss( - label, - pred, - session_ids=session, - margin=margin, - temperature=temp, - weights=loss_weight, - name=loss_name) + label, + pred, + session_ids=session, + margin=margin, + temperature=temp, + weights=loss_weight, + name=loss_name + ) elif loss_type == LossType.PAIRWISE_LOGISTIC_LOSS: session = kwargs.get('session_ids', None) temp = 1.0 if loss_param is None else loss_param.temperature @@ -103,15 +106,16 @@ def build(loss_type, hinge_margin = loss_param.hinge_margin lbl_margin = False if loss_param is None else loss_param.use_label_margin return pairwise_logistic_loss( - label, - pred, - session_ids=session, - temperature=temp, - hinge_margin=hinge_margin, - ohem_ratio=ohem_ratio, - weights=loss_weight, - use_label_margin=lbl_margin, - name=loss_name) + label, + pred, + session_ids=session, + temperature=temp, + hinge_margin=hinge_margin, + ohem_ratio=ohem_ratio, + weights=loss_weight, + use_label_margin=lbl_margin, + name=loss_name + ) elif loss_type == LossType.PAIRWISE_HINGE_LOSS: session = kwargs.get('session_ids', None) temp, ohem_ratio, margin = 1.0, 1.0, 1.0 @@ -124,36 +128,39 @@ def build(loss_type, use_label_margin = loss_param.use_label_margin use_exponent = loss_param.use_exponent return pairwise_hinge_loss( - label, - pred, - session_ids=session, - temperature=temp, - margin=margin, - ohem_ratio=ohem_ratio, - weights=loss_weight, - label_is_logits=label_is_logits, - use_label_margin=use_label_margin, - use_exponent=use_exponent, - name=loss_name) + label, + pred, + session_ids=session, + temperature=temp, + margin=margin, + ohem_ratio=ohem_ratio, + weights=loss_weight, + label_is_logits=label_is_logits, + use_label_margin=use_label_margin, + use_exponent=use_exponent, + name=loss_name + ) elif loss_type == LossType.PAIRWISE_FOCAL_LOSS: session = kwargs.get('session_ids', None) if loss_param is None: return pairwise_focal_loss( - label, pred, session_ids=session, weights=loss_weight, name=loss_name) + label, pred, session_ids=session, weights=loss_weight, name=loss_name + ) hinge_margin = None if loss_param.HasField('hinge_margin'): hinge_margin = loss_param.hinge_margin return pairwise_focal_loss( - label, - pred, - session_ids=session, - gamma=loss_param.gamma, - alpha=loss_param.alpha if loss_param.HasField('alpha') else None, - hinge_margin=hinge_margin, - ohem_ratio=loss_param.ohem_ratio, - temperature=loss_param.temperature, - weights=loss_weight, - name=loss_name) + label, + pred, + session_ids=session, + gamma=loss_param.gamma, + alpha=loss_param.alpha if loss_param.HasField('alpha') else None, + hinge_margin=hinge_margin, + ohem_ratio=loss_param.ohem_ratio, + temperature=loss_param.temperature, + weights=loss_weight, + name=loss_name + ) elif loss_type == LossType.LISTWISE_RANK_LOSS: session = kwargs.get('session_ids', None) trans_fn, temp, label_is_logits, scale = None, 1.0, False, False @@ -164,14 +171,15 @@ def build(loss_type, if loss_param.HasField('transform_fn'): trans_fn = loss_param.transform_fn return listwise_rank_loss( - label, - pred, - session, - temperature=temp, - label_is_logits=label_is_logits, - transform_fn=trans_fn, - scale_logits=scale, - weights=loss_weight) + label, + pred, + session, + temperature=temp, + label_is_logits=label_is_logits, + transform_fn=trans_fn, + scale_logits=scale, + weights=loss_weight + ) elif loss_type == LossType.LISTWISE_DISTILL_LOSS: session = kwargs.get('session_ids', None) trans_fn, temp, label_clip_max_value, scale = None, 1.0, 512.0, False @@ -182,40 +190,44 @@ def build(loss_type, if loss_param.HasField('transform_fn'): trans_fn = loss_param.transform_fn return listwise_distill_loss( - label, - pred, - session, - temperature=temp, - label_clip_max_value=label_clip_max_value, - transform_fn=trans_fn, - scale_logits=scale, - weights=loss_weight) + label, + pred, + session, + temperature=temp, + label_clip_max_value=label_clip_max_value, + transform_fn=trans_fn, + scale_logits=scale, + weights=loss_weight + ) elif loss_type == LossType.F1_REWEIGHTED_LOSS: f1_beta_square = 1.0 if loss_param is None else loss_param.f1_beta_square label_smoothing = 0 if loss_param is None else loss_param.label_smoothing return f1_reweight_sigmoid_cross_entropy( - label, - pred, - f1_beta_square, - weights=loss_weight, - label_smoothing=label_smoothing) + label, + pred, + f1_beta_square, + weights=loss_weight, + label_smoothing=label_smoothing + ) elif loss_type == LossType.BINARY_FOCAL_LOSS: if loss_param is None: return sigmoid_focal_loss_with_logits( - label, pred, sample_weights=loss_weight, name=loss_name) + label, pred, sample_weights=loss_weight, name=loss_name + ) gamma = loss_param.gamma alpha = None if loss_param.HasField('alpha'): alpha = loss_param.alpha return sigmoid_focal_loss_with_logits( - label, - pred, - gamma=gamma, - alpha=alpha, - ohem_ratio=loss_param.ohem_ratio, - sample_weights=loss_weight, - label_smoothing=loss_param.label_smoothing, - name=loss_name) + label, + pred, + gamma=gamma, + alpha=alpha, + ohem_ratio=loss_param.ohem_ratio, + sample_weights=loss_weight, + label_smoothing=loss_param.label_smoothing, + name=loss_name + ) else: raise ValueError('unsupported loss type: %s' % LossType.Name(loss_type)) @@ -244,14 +256,18 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): loss_name += '_' + kd.soft_label_name.replace('/', '_') loss_weight = kd.loss_weight - if kd.HasField('task_space_indicator_name') and kd.HasField( - 'task_space_indicator_value'): + if kd.HasField('task_space_indicator_name' + ) and kd.HasField('task_space_indicator_value'): in_task_space = tf.to_float( - tf.equal(feature_dict[kd.task_space_indicator_name], - kd.task_space_indicator_value)) + tf.equal( + feature_dict[kd.task_space_indicator_name], + kd.task_space_indicator_value + ) + ) loss_weight = loss_weight * ( - kd.in_task_space_weight * in_task_space + kd.out_task_space_weight * - (1 - in_task_space)) + kd.in_task_space_weight * in_task_space + kd.out_task_space_weight * + (1 - in_task_space) + ) label = label_dict[kd.soft_label_name] pred = prediction_dict[kd.pred_name] @@ -318,19 +334,22 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): labels = label preds = pred losses = tf.keras.losses.KLD(labels, preds) - loss_dict[loss_name] = tf.reduce_mean( - losses, name=loss_name) * loss_weight + loss_dict[loss_name + ] = tf.reduce_mean(losses, name=loss_name) * loss_weight elif kd.loss_type == LossType.BINARY_CROSS_ENTROPY_LOSS: losses = tf.keras.backend.binary_crossentropy( - label, pred, from_logits=True) - loss_dict[loss_name] = tf.reduce_mean( - losses, name=loss_name) * loss_weight + label, pred, from_logits=True + ) + loss_dict[loss_name + ] = tf.reduce_mean(losses, name=loss_name) * loss_weight elif kd.loss_type == LossType.CROSS_ENTROPY_LOSS: loss_dict[loss_name] = tf.losses.log_loss( - label, pred, weights=loss_weight) + label, pred, weights=loss_weight + ) elif kd.loss_type == LossType.L2_LOSS: loss_dict[loss_name] = tf.losses.mean_squared_error( - labels=label, predictions=pred, weights=loss_weight) + labels=label, predictions=pred, weights=loss_weight + ) else: loss_param = kd.WhichOneof('loss_param') kwargs = {} @@ -339,10 +358,11 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): if hasattr(loss_param, 'session_name'): kwargs['session_ids'] = feature_dict[loss_param.session_name] loss_dict[loss_name] = build( - kd.loss_type, - label, - pred, - loss_weight=loss_weight, - loss_param=loss_param, - **kwargs) + kd.loss_type, + label, + pred, + loss_weight=loss_weight, + loss_param=loss_param, + **kwargs + ) return loss_dict diff --git a/easy_rec/python/builders/optimizer_builder.py b/easy_rec/python/builders/optimizer_builder.py index 8b9a8251b..de9b5efb4 100644 --- a/easy_rec/python/builders/optimizer_builder.py +++ b/easy_rec/python/builders/optimizer_builder.py @@ -15,7 +15,6 @@ # ============================================================================== """Functions to build training optimizers.""" import logging - import tensorflow as tf from easy_rec.python.compat import weight_decay_optimizers @@ -46,24 +45,27 @@ def build(optimizer_config): learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) optimizer = tf.train.RMSPropOptimizer( - learning_rate, - decay=config.decay, - momentum=config.momentum_optimizer_value, - epsilon=config.epsilon) + learning_rate, + decay=config.decay, + momentum=config.momentum_optimizer_value, + epsilon=config.epsilon + ) if optimizer_type == 'momentum_optimizer': config = optimizer_config.momentum_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) optimizer = tf.train.MomentumOptimizer( - learning_rate, momentum=config.momentum_optimizer_value) + learning_rate, momentum=config.momentum_optimizer_value + ) if optimizer_type == 'adam_optimizer': config = optimizer_config.adam_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) optimizer = tf.train.AdamOptimizer( - learning_rate, beta1=config.beta1, beta2=config.beta2) + learning_rate, beta1=config.beta1, beta2=config.beta2 + ) if optimizer_type == 'adamw_optimizer': config = optimizer_config.adamw_optimizer @@ -71,22 +73,25 @@ def build(optimizer_config): summary_vars.append(learning_rate) logging.info('adamw_optimizer weight_decay = %.8f' % config.weight_decay) optimizer = weight_decay_optimizers.AdamWOptimizer( - weight_decay=config.weight_decay, - learning_rate=learning_rate, - beta1=config.beta1, - beta2=config.beta2) + weight_decay=config.weight_decay, + learning_rate=learning_rate, + beta1=config.beta1, + beta2=config.beta2 + ) if optimizer_type == 'adam_asyncw_optimizer': config = optimizer_config.adam_asyncw_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - logging.info('adam_asyncw_optimizer weight_decay = %.8f' % - config.weight_decay) + logging.info( + 'adam_asyncw_optimizer weight_decay = %.8f' % config.weight_decay + ) optimizer = weight_decay_optimizers.AdamAsyncWOptimizer( - weight_decay=config.weight_decay, - learning_rate=learning_rate, - beta1=config.beta1, - beta2=config.beta2) + weight_decay=config.weight_decay, + learning_rate=learning_rate, + beta1=config.beta1, + beta2=config.beta2 + ) if optimizer_type == 'lazy_adam_optimizer': config = optimizer_config.lazy_adam_optimizer @@ -94,52 +99,59 @@ def build(optimizer_config): summary_vars.append(learning_rate) from easy_rec.python.compat.adam_s import AdamOptimizerS optimizer = AdamOptimizerS( - learning_rate=learning_rate, beta1=config.beta1, beta2=config.beta2) + learning_rate=learning_rate, beta1=config.beta1, beta2=config.beta2 + ) if optimizer_type == 'momentumw_optimizer': config = optimizer_config.momentumw_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - logging.info('momentumw_optimizer weight_decay = %.8f' % - config.weight_decay) + logging.info( + 'momentumw_optimizer weight_decay = %.8f' % config.weight_decay + ) optimizer = weight_decay_optimizers.MomentumWOptimizer( - weight_decay=config.weight_decay, - learning_rate=learning_rate, - momentum=config.momentum_optimizer_value) + weight_decay=config.weight_decay, + learning_rate=learning_rate, + momentum=config.momentum_optimizer_value + ) if optimizer_type == 'adagrad_optimizer': config = optimizer_config.adagrad_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) optimizer = tf.train.AdagradOptimizer( - learning_rate, - initial_accumulator_value=config.initial_accumulator_value) + learning_rate, + initial_accumulator_value=config.initial_accumulator_value + ) if optimizer_type == 'adam_async_optimizer': config = optimizer_config.adam_async_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) optimizer = tf.train.AdamAsyncOptimizer( - learning_rate, beta1=config.beta1, beta2=config.beta2) + learning_rate, beta1=config.beta1, beta2=config.beta2 + ) if optimizer_type == 'ftrl_optimizer': config = optimizer_config.ftrl_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) optimizer = tf.train.FtrlOptimizer( - learning_rate=learning_rate, - learning_rate_power=config.learning_rate_power, - initial_accumulator_value=config.initial_accumulator_value, - l1_regularization_strength=config.l1_reg, - l2_regularization_strength=config.l2_reg, - l2_shrinkage_regularization_strength=config.l2_shrinkage_reg) + learning_rate=learning_rate, + learning_rate_power=config.learning_rate_power, + initial_accumulator_value=config.initial_accumulator_value, + l1_regularization_strength=config.l1_reg, + l2_regularization_strength=config.l2_reg, + l2_shrinkage_regularization_strength=config.l2_shrinkage_reg + ) if optimizer is None: raise ValueError('Optimizer %s not supported.' % optimizer_type) if optimizer_config.use_moving_average: optimizer = tf.contrib.opt.MovingAverageOptimizer( - optimizer, average_decay=optimizer_config.moving_average_decay) + optimizer, average_decay=optimizer_config.moving_average_decay + ) return optimizer, summary_vars @@ -161,19 +173,21 @@ def _create_learning_rate(learning_rate_config): if learning_rate_type == 'constant_learning_rate': config = learning_rate_config.constant_learning_rate learning_rate = tf.constant( - config.learning_rate, dtype=tf.float32, name='learning_rate') + config.learning_rate, dtype=tf.float32, name='learning_rate' + ) if learning_rate_type == 'exponential_decay_learning_rate': config = learning_rate_config.exponential_decay_learning_rate learning_rate = learning_schedules.exponential_decay_with_burnin( - tf.train.get_or_create_global_step(), - config.initial_learning_rate, - config.decay_steps, - config.decay_factor, - burnin_learning_rate=config.burnin_learning_rate, - burnin_steps=config.burnin_steps, - min_learning_rate=config.min_learning_rate, - staircase=config.staircase) + tf.train.get_or_create_global_step(), + config.initial_learning_rate, + config.decay_steps, + config.decay_factor, + burnin_learning_rate=config.burnin_learning_rate, + burnin_steps=config.burnin_steps, + min_learning_rate=config.min_learning_rate, + staircase=config.staircase + ) if learning_rate_type == 'manual_step_learning_rate': config = learning_rate_config.manual_step_learning_rate @@ -183,27 +197,31 @@ def _create_learning_rate(learning_rate_config): learning_rate_sequence = [config.initial_learning_rate] learning_rate_sequence += [x.learning_rate for x in config.schedule] learning_rate = learning_schedules.manual_stepping( - tf.train.get_or_create_global_step(), learning_rate_step_boundaries, - learning_rate_sequence, config.warmup) + tf.train.get_or_create_global_step(), learning_rate_step_boundaries, + learning_rate_sequence, config.warmup + ) if learning_rate_type == 'cosine_decay_learning_rate': config = learning_rate_config.cosine_decay_learning_rate learning_rate = learning_schedules.cosine_decay_with_warmup( - tf.train.get_or_create_global_step(), config.learning_rate_base, - config.total_steps, config.warmup_learning_rate, config.warmup_steps, - config.hold_base_rate_steps) + tf.train.get_or_create_global_step(), config.learning_rate_base, + config.total_steps, config.warmup_learning_rate, config.warmup_steps, + config.hold_base_rate_steps + ) if learning_rate_type == 'poly_decay_learning_rate': config = learning_rate_config.poly_decay_learning_rate learning_rate = tf.train.polynomial_decay( - config.learning_rate_base, tf.train.get_or_create_global_step(), - config.total_steps, config.end_learning_rate, config.power) + config.learning_rate_base, tf.train.get_or_create_global_step(), + config.total_steps, config.end_learning_rate, config.power + ) if learning_rate_type == 'transformer_learning_rate': config = learning_rate_config.transformer_learning_rate learning_rate = learning_schedules.transformer_policy( - tf.train.get_or_create_global_step(), config.learning_rate_base, - config.hidden_size, config.warmup_steps, config.step_scaling_rate) + tf.train.get_or_create_global_step(), config.learning_rate_base, + config.hidden_size, config.warmup_steps, config.step_scaling_rate + ) if learning_rate is None: raise ValueError('Learning_rate %s not supported.' % learning_rate_type) diff --git a/easy_rec/python/builders/strategy_builder.py b/easy_rec/python/builders/strategy_builder.py index a0c758318..be26c2e9f 100644 --- a/easy_rec/python/builders/strategy_builder.py +++ b/easy_rec/python/builders/strategy_builder.py @@ -2,8 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.protos.train_pb2 import TrainConfig +from easy_rec.python.protos.train_pb2 import DistributionStrategy, TrainConfig def build(train_config): @@ -25,16 +24,18 @@ def build(train_config): elif train_config.train_distribute == DistributionStrategy.ExascaleStrategy: import pai distribution = pai.distribute.ExascaleStrategy( - max_splits=10, - issorted=True, - optimize_clip_by_global_norm=False, - enable_sparse_allreduce=False, - enable_hierarchical_allreduce=True) + max_splits=10, + issorted=True, + optimize_clip_by_global_norm=False, + enable_sparse_allreduce=False, + enable_hierarchical_allreduce=True + ) # the older version of MultiWorkerMirroredStrategy # works under tf1.12 to tf1.15 elif train_config.train_distribute == DistributionStrategy.CollectiveAllReduceStrategy: distribution = tf.contrib.distribute.CollectiveAllReduceStrategy( - num_gpus_per_worker=train_config.num_gpus_per_worker) + num_gpus_per_worker=train_config.num_gpus_per_worker + ) # works under tf1.15 and tf2.x elif train_config.train_distribute == DistributionStrategy.PSStrategy: if tf.__version__ <= '1.15': diff --git a/easy_rec/python/compat/adam_s.py b/easy_rec/python/compat/adam_s.py index a0ef60b80..3fa067732 100644 --- a/easy_rec/python/compat/adam_s.py +++ b/easy_rec/python/compat/adam_s.py @@ -13,19 +13,12 @@ # limitations under the License. # ============================================================================== """Adam for TensorFlow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function from tensorflow.python.eager import context from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import optimizer -from tensorflow.python.training import training_ops +from tensorflow.python.ops import array_ops, control_flow_ops, math_ops, resource_variable_ops, state_ops # NOQA +from tensorflow.python.training import optimizer, training_ops class AdamOptimizerS(optimizer.Optimizer): @@ -37,13 +30,15 @@ class AdamOptimizerS(optimizer.Optimizer): ([pdf](https://arxiv.org/pdf/1412.6980.pdf)) """ - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - use_locking=False, - name='Adam'): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + use_locking=False, + name='Adam' + ): r"""Construct a new Adam optimizer. Initialization: @@ -118,8 +113,10 @@ def _get_beta_accumulators(self): graph = None else: graph = ops.get_default_graph() - return (self._get_non_slot_variable('beta1_power', graph=graph), - self._get_non_slot_variable('beta2_power', graph=graph)) + return ( + self._get_non_slot_variable('beta1_power', graph=graph), + self._get_non_slot_variable('beta2_power', graph=graph) + ) def _create_slots(self, var_list): # Create the beta1 and beta2 accumulators on the same device as the first @@ -128,9 +125,11 @@ def _create_slots(self, var_list): # silently ignored). first_var = min(var_list, key=lambda x: x.name) self._create_non_slot_variable( - initial_value=self._beta1, name='beta1_power', colocate_with=first_var) + initial_value=self._beta1, name='beta1_power', colocate_with=first_var + ) self._create_non_slot_variable( - initial_value=self._beta2, name='beta2_power', colocate_with=first_var) + initial_value=self._beta2, name='beta2_power', colocate_with=first_var + ) # Create slots for the first and second moments. for v in var_list: @@ -153,34 +152,36 @@ def _apply_dense(self, grad, var): v = self.get_slot(var, 'v') beta1_power, beta2_power = self._get_beta_accumulators() return training_ops.apply_adam( - var, - m, - v, - math_ops.cast(beta1_power, var.dtype.base_dtype), - math_ops.cast(beta2_power, var.dtype.base_dtype), - math_ops.cast(self._lr_t, var.dtype.base_dtype), - math_ops.cast(self._beta1_t, var.dtype.base_dtype), - math_ops.cast(self._beta2_t, var.dtype.base_dtype), - math_ops.cast(self._epsilon_t, var.dtype.base_dtype), - grad, - use_locking=self._use_locking).op + var, + m, + v, + math_ops.cast(beta1_power, var.dtype.base_dtype), + math_ops.cast(beta2_power, var.dtype.base_dtype), + math_ops.cast(self._lr_t, var.dtype.base_dtype), + math_ops.cast(self._beta1_t, var.dtype.base_dtype), + math_ops.cast(self._beta2_t, var.dtype.base_dtype), + math_ops.cast(self._epsilon_t, var.dtype.base_dtype), + grad, + use_locking=self._use_locking + ).op def _resource_apply_dense(self, grad, var): m = self.get_slot(var, 'm') v = self.get_slot(var, 'v') beta1_power, beta2_power = self._get_beta_accumulators() return training_ops.resource_apply_adam( - var.handle, - m.handle, - v.handle, - math_ops.cast(beta1_power, grad.dtype.base_dtype), - math_ops.cast(beta2_power, grad.dtype.base_dtype), - math_ops.cast(self._lr_t, grad.dtype.base_dtype), - math_ops.cast(self._beta1_t, grad.dtype.base_dtype), - math_ops.cast(self._beta2_t, grad.dtype.base_dtype), - math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), - grad, - use_locking=self._use_locking) + var.handle, + m.handle, + v.handle, + math_ops.cast(beta1_power, grad.dtype.base_dtype), + math_ops.cast(beta2_power, grad.dtype.base_dtype), + math_ops.cast(self._lr_t, grad.dtype.base_dtype), + math_ops.cast(self._beta1_t, grad.dtype.base_dtype), + math_ops.cast(self._beta2_t, grad.dtype.base_dtype), + math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), + grad, + use_locking=self._use_locking + ) def _apply_sparse_shared(self, grad, var, indices, scatter_add): beta1_power, beta2_power = self._get_beta_accumulators() @@ -208,8 +209,9 @@ def _apply_sparse_shared(self, grad, var, indices, scatter_add): # var_update = state_ops.assign_sub( # var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking) v_part_sqrt = math_ops.sqrt(v_part_n) - var_update = scatter_add(var, indices, - -lr * m_part_n / (v_part_sqrt + epsilon_t)) + var_update = scatter_add( + var, indices, -lr * m_part_n / (v_part_sqrt + epsilon_t) + ) return control_flow_ops.group(*[var_update, m_t, v_t]) def _apply_sparse(self, grad, var): @@ -225,12 +227,14 @@ def _apply_sparse(self, grad, var): def _resource_scatter_add(self, x, i, v): with ops.control_dependencies( - [resource_variable_ops.resource_scatter_add(x.handle, i, v)]): + [resource_variable_ops.resource_scatter_add(x.handle, i, v)] + ): return x.value() def _resource_apply_sparse(self, grad, var, indices): - return self._apply_sparse_shared(grad, var, indices, - self._resource_scatter_add) + return self._apply_sparse_shared( + grad, var, indices, self._resource_scatter_add + ) def _finish(self, update_ops, name_scope): # Update the power accumulators. @@ -238,8 +242,11 @@ def _finish(self, update_ops, name_scope): beta1_power, beta2_power = self._get_beta_accumulators() with ops.colocate_with(beta1_power): update_beta1 = beta1_power.assign( - beta1_power * self._beta1_t, use_locking=self._use_locking) + beta1_power * self._beta1_t, use_locking=self._use_locking + ) update_beta2 = beta2_power.assign( - beta2_power * self._beta2_t, use_locking=self._use_locking) + beta2_power * self._beta2_t, use_locking=self._use_locking + ) return control_flow_ops.group( - *update_ops + [update_beta1, update_beta2], name=name_scope) + *update_ops + [update_beta1, update_beta2], name=name_scope + ) diff --git a/easy_rec/python/compat/array_ops.py b/easy_rec/python/compat/array_ops.py index d788bc8c1..44ca091ba 100644 --- a/easy_rec/python/compat/array_ops.py +++ b/easy_rec/python/compat/array_ops.py @@ -1,8 +1,6 @@ import numpy as np import tensorflow as tf -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import constant_op, ops, sparse_tensor from tensorflow.python.ops import gen_math_ops @@ -12,8 +10,9 @@ def convert_to_int_tensor(tensor, name, dtype=tf.int32): if tensor.dtype.is_integer: tensor = gen_math_ops.cast(tensor, dtype) else: - raise TypeError('%s must be an integer tensor; dtype=%s' % - (name, tensor.dtype)) + raise TypeError( + '%s must be an integer tensor; dtype=%s' % (name, tensor.dtype) + ) return tensor @@ -58,8 +57,9 @@ def get_positive_axis(axis, ndims): elif -ndims <= axis < 0: return axis + ndims else: - raise ValueError('axis=%s out of bounds: expected %s<=axis<%s' % - (axis, -ndims, ndims)) + raise ValueError( + 'axis=%s out of bounds: expected %s<=axis<%s' % (axis, -ndims, ndims) + ) elif axis < 0: raise ValueError('axis may only be negative if ndims is statically known.') return axis @@ -74,7 +74,8 @@ def tile_one_dimension(data, axis, multiple): else: ones_value = tf.ones(tf.rank(data), tf.int32) multiples = tf.concat( - [ones_value[:axis], [multiple], ones_value[axis + 1:]], axis=0) + [ones_value[:axis], [multiple], ones_value[axis + 1:]], axis=0 + ) return tf.tile(data, multiples) @@ -83,8 +84,10 @@ def _all_dimensions(x): # Fast path: avoid creating Rank and Range ops if ndims is known. if isinstance(x, ops.Tensor) and x.get_shape().ndims is not None: return constant_op.constant(np.arange(x.get_shape().ndims), dtype=tf.int32) - if (isinstance(x, sparse_tensor.SparseTensor) and - x.dense_shape.get_shape().is_fully_defined()): + if ( + isinstance(x, sparse_tensor.SparseTensor) + and x.dense_shape.get_shape().is_fully_defined() + ): r = x.dense_shape.get_shape().dims[0].value # sparse.dense_shape is 1-D. return constant_op.constant(np.arange(r), dtype=tf.int32) @@ -147,8 +150,9 @@ def repeat_with_axis(data, repeats, axis, name=None): if repeats.shape.ndims == 0: expanded = tf.expand_dims(data, axis + 1) tiled = tile_one_dimension(expanded, axis + 1, repeats) - result_shape = tf.concat([data_shape[:axis], [-1], data_shape[axis + 1:]], - axis=0) + result_shape = tf.concat( + [data_shape[:axis], [-1], data_shape[axis + 1:]], axis=0 + ) return tf.reshape(tiled, result_shape) # Broadcast the `repeats` tensor so rank(repeats) == axis + 1. @@ -156,7 +160,8 @@ def repeat_with_axis(data, repeats, axis, name=None): repeats_shape = tf.shape(repeats) repeats_ndims = tf.rank(repeats) broadcast_shape = tf.concat( - [data_shape[:axis + 1 - repeats_ndims], repeats_shape], axis=0) + [data_shape[:axis + 1 - repeats_ndims], repeats_shape], axis=0 + ) repeats = tf.broadcast_to(repeats, broadcast_shape) repeats.set_shape([None] * (axis + 1)) @@ -164,7 +169,8 @@ def repeat_with_axis(data, repeats, axis, name=None): # contain one `True` value for each repetition. E.g., if # `repeats = [3, 1, 2]`, then `mask = [[1, 1, 1], [1, 0, 0], [1, 1, 0]]`. max_repeat = gen_math_ops.maximum( - 0, gen_math_ops._max(repeats, _all_dimensions(repeats))) + 0, gen_math_ops._max(repeats, _all_dimensions(repeats)) + ) mask = tf.sequence_mask(repeats, max_repeat) # Add a new dimension around each value that needs to be repeated, and @@ -180,15 +186,18 @@ def repeat_with_axis(data, repeats, axis, name=None): if axis == 0: result = masked else: - result_shape = tf.concat([data_shape[:axis], [-1], data_shape[axis + 1:]], - axis=0) + result_shape = tf.concat( + [data_shape[:axis], [-1], data_shape[axis + 1:]], axis=0 + ) result = tf.reshape(masked, result_shape) # Preserve shape information. if data.shape.ndims is not None: new_axis_size = 0 if repeats.shape[0] == 0 else None - result.set_shape(data.shape[:axis].concatenate( - [new_axis_size]).concatenate(data.shape[axis + 1:])) + result.set_shape( + data.shape[:axis].concatenate([new_axis_size] + ).concatenate(data.shape[axis + 1:]) + ) return result diff --git a/easy_rec/python/compat/dynamic_variable.py b/easy_rec/python/compat/dynamic_variable.py index 83414331c..9bd7f43dd 100644 --- a/easy_rec/python/compat/dynamic_variable.py +++ b/easy_rec/python/compat/dynamic_variable.py @@ -15,7 +15,6 @@ # import json - import tensorflow as tf from sparse_operation_kit.experiment import raw_ops as dynamic_variable_ops from sparse_operation_kit.experiment.communication import num_gpus @@ -23,8 +22,7 @@ from tensorflow.python.framework import ops # from tensorflow.python.ops import array_ops from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops.resource_variable_ops import ResourceVariable -from tensorflow.python.ops.resource_variable_ops import variable_accessed +from tensorflow.python.ops.resource_variable_ops import ResourceVariable, variable_accessed # NOQA # from tensorflow.python.util import object_identity @@ -84,28 +82,33 @@ class DynamicVariable(ResourceVariable): print("v.size:", v.size) """ - def __init__(self, - dimension, - initializer=None, - var_type=None, - name=None, - constraint=None, - trainable=True, - key_type=None, - dtype=None, - mode=None, - variable_def=None, - import_scope=None, - **kwargs): + def __init__( + self, + dimension, + initializer=None, + var_type=None, + name=None, + constraint=None, + trainable=True, + key_type=None, + dtype=None, + mode=None, + variable_def=None, + import_scope=None, + **kwargs + ): self._indices = None if variable_def is not None: super(DynamicVariable, self)._init_from_proto( - variable_def, import_scope=import_scope, validate_shape=False) + variable_def, import_scope=import_scope, validate_shape=False + ) g = ops.get_default_graph() handle = g.as_graph_element( - ops.prepend_name_scope( - variable_def.variable_name, import_scope=import_scope), - allow_operation=False) + ops.prepend_name_scope( + variable_def.variable_name, import_scope=import_scope + ), + allow_operation=False + ) self._dimension = handle.op.get_attr('shape').dim[-1].size self._key_type = handle.op.get_attr('key_type') self._handle_type = handle.op.get_attr('dtype') @@ -134,7 +137,8 @@ def __init__(self, self._config_dict = kwargs if var_type == 'hybrid' and self._key_type != tf.int64: raise NotImplementedError( - 'only key_type tf.int64 is supported in HKV backend') + 'only key_type tf.int64 is supported in HKV backend' + ) if name is None: global dynamic_variable_count name = 'sok_dynamic_Variable_' + str(dynamic_variable_count) @@ -143,15 +147,15 @@ def __init__(self, self._var_type = var_type self._base = super(DynamicVariable, self) self._base.__init__( - initial_value=[[0.0] * dimension], - trainable=trainable, - name=name + '/proxy', - dtype=self._handle_dtype, - constraint=constraint, - distribute_strategy=None, - synchronization=None, - aggregation=None, - shape=[None, dimension], + initial_value=[[0.0] * dimension], + trainable=trainable, + name=name + '/proxy', + dtype=self._handle_dtype, + constraint=constraint, + distribute_strategy=None, + synchronization=None, + aggregation=None, + shape=[None, dimension], ) with ops.init_scope(): @@ -165,33 +169,33 @@ def __init__(self, initializer = '' if initializer is None else initializer self._initializer = initializer handle = dynamic_variable_ops.dummy_var_handle( - container='DummyVariableContainer', - shared_name=self._dummy_name, - key_type=self._key_type, - dtype=self._handle_dtype, - shape=shape, + container='DummyVariableContainer', + shared_name=self._dummy_name, + key_type=self._key_type, + dtype=self._handle_dtype, + shape=shape, ) if type(initializer) is str: init_op = dynamic_variable_ops.dummy_var_initialize( - handle, - initializer=initializer, - var_type=var_type, - unique_name=self._dummy_name, - key_type=self._key_type, - dtype=self._handle_dtype, - config=self._config, + handle, + initializer=initializer, + var_type=var_type, + unique_name=self._dummy_name, + key_type=self._key_type, + dtype=self._handle_dtype, + config=self._config, ) else: with tf.control_dependencies([initializer._initializer_op]): initial_val = initializer.read_value() init_op = dynamic_variable_ops.dummy_var_initialize( - handle, - initializer=initial_val, - var_type=var_type, - unique_name=self._dummy_name, - key_type=self._key_type, - dtype=self._handle_dtype, - config=self._config, + handle, + initializer=initial_val, + var_type=var_type, + unique_name=self._dummy_name, + key_type=self._key_type, + dtype=self._handle_dtype, + config=self._config, ) # TODO: Add is_initialized_op # is_initialized_op = ops.convert_to_tensor(True) @@ -204,17 +208,21 @@ def __init__(self, # self._is_initialized_op = tf.group([self._is_initialized_op, is_initialized_op]) handle_data = ( - resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult - .HandleData()) + resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult. + HandleData() + ) handle_data.is_set = True handle_data.shape_and_type.append( - resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult - .HandleShapeAndType( - shape=self.shape.as_proto(), dtype=self.dtype.as_datatype_enum)) + resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult. + HandleShapeAndType( + shape=self.shape.as_proto(), dtype=self.dtype.as_datatype_enum + ) + ) resource_variable_ops._set_handle_shapes_and_types( - self._handle, - handle_data, - graph_mode=False if context.executing_eagerly() else True) + self._handle, + handle_data, + graph_mode=False if context.executing_eagerly() else True + ) def is_static(self): return self._handle is self._tf_handle @@ -246,15 +254,16 @@ def __repr__(self): if self.is_static(): return self._base.__repr__() return "" % ( - self._dummy_name, - self.shape, - self.dtype.name, + self._dummy_name, + self.shape, + self.dtype.name, ) @property def size(self): return dynamic_variable_ops.dummy_var_shape( - self._dummy_handle, key_type=self._key_type, dtype=self._handle_dtype) + self._dummy_handle, key_type=self._key_type, dtype=self._handle_dtype + ) @property def indices(self): @@ -310,10 +319,11 @@ def sparse_read(self, indices, name=None, lookup_only=False): if indices.dtype == tf.int32: indices = tf.cast(indices, tf.int64) return dynamic_variable_ops.dummy_var_sparse_read( - self._dummy_handle, - indices, - dtype=self._handle_dtype, - lookup_only=lookup_only) + self._dummy_handle, + indices, + dtype=self._handle_dtype, + lookup_only=lookup_only + ) def scatter_sub(self, sparse_delta, use_locking=False, name=None): if self.is_static(): @@ -321,9 +331,9 @@ def scatter_sub(self, sparse_delta, use_locking=False, name=None): if not isinstance(sparse_delta, ops.IndexedSlices): raise TypeError('sparse_delta is not IndexedSlices: %s' % sparse_delta) return dynamic_variable_ops.dummy_var_scatter_add( - self._dummy_handle, - sparse_delta.indices, - ops.convert_to_tensor(-sparse_delta.values, self.dtype), + self._dummy_handle, + sparse_delta.indices, + ops.convert_to_tensor(-sparse_delta.values, self.dtype), ) def scatter_add(self, sparse_delta, use_locking=False, name=None): @@ -332,9 +342,9 @@ def scatter_add(self, sparse_delta, use_locking=False, name=None): if not isinstance(sparse_delta, ops.IndexedSlices): raise TypeError('sparse_delta is not IndexedSlices: %s' % sparse_delta) return dynamic_variable_ops.dummy_var_scatter_add( - self._dummy_handle, - sparse_delta.indices, - ops.convert_to_tensor(sparse_delta.values, self.dtype), + self._dummy_handle, + sparse_delta.indices, + ops.convert_to_tensor(sparse_delta.values, self.dtype), ) def scatter_update(self, sparse_delta, use_locking=False, name=None): @@ -343,9 +353,9 @@ def scatter_update(self, sparse_delta, use_locking=False, name=None): if not isinstance(sparse_delta, ops.IndexedSlices): raise TypeError('sparse_delta is not IndexedSlices: %s' % sparse_delta) return dynamic_variable_ops.dummy_var_scatter_update( - self._dummy_handle, - sparse_delta.indices, - ops.convert_to_tensor(sparse_delta.values, self.dtype), + self._dummy_handle, + sparse_delta.indices, + ops.convert_to_tensor(sparse_delta.values, self.dtype), ) # ------------------------------------------------------------------------- @@ -366,7 +376,8 @@ def to_proto(self, *args, **kwargs): def from_proto(variable_def, import_scope=None): if '/DummyVarHandle' in variable_def.variable_name: return DynamicVariable( - dimension=0, variable_def=variable_def, import_scope=import_scope) + dimension=0, variable_def=variable_def, import_scope=import_scope + ) else: return _resource_var_from_proto(variable_def, import_scope) # raise NotImplementedError("from_proto() is not supported.") @@ -383,13 +394,15 @@ def is_initialized(self, name): if self.is_static(): return self._base.is_initialized(name) raise NotImplementedError( - 'is_initialized() is not supported in dynamic mode.') + 'is_initialized() is not supported in dynamic mode.' + ) def _read_variable_op(self): if self.is_static(): return self._base._read_variable_op() raise NotImplementedError( - '_read_variable_op() is not supported in dynamic mode.') + '_read_variable_op() is not supported in dynamic mode.' + ) def value(self): if self.is_static(): @@ -400,13 +413,15 @@ def _dense_var_to_tensor(self, *args, **kwargs): if self.is_static(): return self._base._dense_var_to_tensor(*args, **kwargs) raise NotImplementedError( - '_dense_var_to_tensor() is not supported in dynamic mode.') + '_dense_var_to_tensor() is not supported in dynamic mode.' + ) def _gather_saveables_for_checkpoint(self): if self.is_static(): return self._base._gather_saveables_for_checkpoint() raise NotImplementedError( - '_gather_saveables_for_checkpoint() is not supported in dynamic mode.') + '_gather_saveables_for_checkpoint() is not supported in dynamic mode.' + ) def gather_nd(self, *args, **kwargs): if self.is_static(): @@ -426,46 +441,58 @@ def assign(self, *args, **kwargs): def scatter_max(self, *args, **kwargs): if self.is_static(): return self._base.scatter_max(*args, **kwargs) - raise NotImplementedError('scatter_max() is not supported in dynamic mode.') + raise NotImplementedError( + 'scatter_max() is not supported in dynamic mode.' + ) def scatter_min(self, *args, **kwargs): if self.is_static(): return self._base.scatter_min(*args, **kwargs) - raise NotImplementedError('scatter_min() is not supported in dynamic mode.') + raise NotImplementedError( + 'scatter_min() is not supported in dynamic mode.' + ) def scatter_mul(self, *args, **kwargs): if self.is_static(): return self._base.scatter_mul(*args, **kwargs) - raise NotImplementedError('scatter_mul() is not supported in dynamic mode.') + raise NotImplementedError( + 'scatter_mul() is not supported in dynamic mode.' + ) def scatter_dim(self, *args, **kwargs): if self.is_static(): return self._base.scatter_dim(*args, **kwargs) - raise NotImplementedError('scatter_dim() is not supported in dynamic mode.') + raise NotImplementedError( + 'scatter_dim() is not supported in dynamic mode.' + ) def batch_scatter_update(self, *args, **kwargs): if self.is_static(): return self._base.batch_scatter_update(*args, **kwargs) raise NotImplementedError( - 'batch_scatter_update() is not supported in dynamic mode.') + 'batch_scatter_update() is not supported in dynamic mode.' + ) def scatter_nd_sub(self, *args, **kwargs): if self.is_static(): return self._base.scatter_nd_sub(*args, **kwargs) raise NotImplementedError( - 'scatter_nd_sub() is not supported in dynamic mode.') + 'scatter_nd_sub() is not supported in dynamic mode.' + ) def scatter_nd_update(self, *args, **kwargs): if self.is_static(): return self._base.scatter_nd_update(*args, **kwargs) raise NotImplementedError( - 'scatter_nd_update() is not supported in dynamic mode.') + 'scatter_nd_update() is not supported in dynamic mode.' + ) def _strided_slice_assign(self, *args, **kwargs): if self.is_static(): return self._base._strided_slice_assign(*args, **kwargs) raise NotImplementedError( - '_strided_slice_assign() is not supported in dynamic mode.') + '_strided_slice_assign() is not supported in dynamic mode.' + ) def __int__(self, *args, **kwargs): if self.is_static(): @@ -510,7 +537,8 @@ def export(var): """ if isinstance(var, DynamicVariable): indices, values = dynamic_variable_ops.dummy_var_export( - var.handle, key_type=var.key_type, dtype=var.handle_dtype) + var.handle, key_type=var.key_type, dtype=var.handle_dtype + ) with tf.device('CPU'): indices = tf.identity(indices) values = tf.identity(values) diff --git a/easy_rec/python/compat/early_stopping.py b/easy_rec/python/compat/early_stopping.py index fc850fb62..7d9bde4b0 100644 --- a/easy_rec/python/compat/early_stopping.py +++ b/easy_rec/python/compat/early_stopping.py @@ -19,28 +19,21 @@ import logging import operator import os +import tensorflow as tf import threading import time - -import tensorflow as tf from distutils.version import LooseVersion -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import gfile -from tensorflow.python.platform import tf_logging +from tensorflow.python.framework import dtypes, ops +from tensorflow.python.ops import init_ops, state_ops, variable_scope +from tensorflow.python.platform import gfile, tf_logging from tensorflow.python.summary import summary_iterator -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import session_run_hook -from tensorflow.python.training import training_util +from tensorflow.python.training import basic_session_run_hooks, session_run_hook, training_util # NOQA from easy_rec.python.utils.config_util import parse_time from easy_rec.python.utils.load_class import load_by_path if LooseVersion(tf.__version__) >= LooseVersion('2.12.0'): - from tensorflow_estimator.python.estimator.estimator_export import estimator_export + from tensorflow_estimator.python.estimator.estimator_export import estimator_export # NOQA else: from tensorflow.python.util.tf_export import estimator_export @@ -59,10 +52,9 @@ def find_early_stop_var(var_list): @estimator_export('estimator.experimental.make_early_stopping_hook') -def make_early_stopping_hook(estimator, - should_stop_fn, - run_every_secs=60, - run_every_steps=None): +def make_early_stopping_hook( + estimator, should_stop_fn, run_every_secs=60, run_every_steps=None +): """Creates early-stopping hook. Returns a `SessionRunHook` that stops training when `should_stop_fn` returns `True`. Usage example: @@ -101,23 +93,29 @@ def make_early_stopping_hook(estimator, ValueError: If both `run_every_secs` and `run_every_steps` are set. """ if run_every_secs is not None and run_every_steps is not None: - raise ValueError('Only one of `run_every_secs` and `run_every_steps` must ' - 'be set.') + raise ValueError( + 'Only one of `run_every_secs` and `run_every_steps` must ' + 'be set.' + ) if estimator.config.is_chief: - return _StopOnPredicateHook(should_stop_fn, run_every_secs, run_every_steps) + return _StopOnPredicateHook( + should_stop_fn, run_every_secs, run_every_steps + ) else: return _CheckForStoppingHook() @estimator_export('estimator.experimental.stop_if_higher_hook') -def stop_if_higher_hook(estimator, - metric_name, - threshold, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None): +def stop_if_higher_hook( + estimator, + metric_name, + threshold, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None +): """Creates hook to stop if the given metric is higher than the threshold. Usage example: @@ -157,24 +155,27 @@ def stop_if_higher_hook(estimator, early stopping if true. """ return _stop_if_threshold_crossed_hook( - estimator=estimator, - metric_name=metric_name, - threshold=threshold, - higher_is_better=True, - eval_dir=eval_dir, - min_steps=min_steps, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + estimator=estimator, + metric_name=metric_name, + threshold=threshold, + higher_is_better=True, + eval_dir=eval_dir, + min_steps=min_steps, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps + ) @estimator_export('estimator.experimental.stop_if_lower_hook') -def stop_if_lower_hook(estimator, - metric_name, - threshold, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None): +def stop_if_lower_hook( + estimator, + metric_name, + threshold, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None +): """Creates hook to stop if the given metric is lower than the threshold. Usage example: @@ -214,24 +215,27 @@ def stop_if_lower_hook(estimator, early stopping if true. """ return _stop_if_threshold_crossed_hook( - estimator=estimator, - metric_name=metric_name, - threshold=threshold, - higher_is_better=False, - eval_dir=eval_dir, - min_steps=min_steps, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + estimator=estimator, + metric_name=metric_name, + threshold=threshold, + higher_is_better=False, + eval_dir=eval_dir, + min_steps=min_steps, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps + ) @estimator_export('estimator.experimental.stop_if_no_increase_hook') -def stop_if_no_increase_hook(estimator, - metric_name, - max_steps_without_increase, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None): +def stop_if_no_increase_hook( + estimator, + metric_name, + max_steps_without_increase, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None +): """Creates hook to stop if metric does not increase within given max steps. Usage example: @@ -272,22 +276,25 @@ def stop_if_no_increase_hook(estimator, training steps, and initiates early stopping if true. """ return _stop_if_no_metric_improvement_hook( - estimator=estimator, - metric_name=metric_name, - max_steps_without_improvement=max_steps_without_increase, - higher_is_better=True, - eval_dir=eval_dir, - min_steps=min_steps, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps) - - -def custom_early_stop_hook(estimator, - eval_dir, - custom_stop_func, - custom_stop_func_params, - run_every_secs=60, - run_every_steps=None): + estimator=estimator, + metric_name=metric_name, + max_steps_without_improvement=max_steps_without_increase, + higher_is_better=True, + eval_dir=eval_dir, + min_steps=min_steps, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps + ) + + +def custom_early_stop_hook( + estimator, + eval_dir, + custom_stop_func, + custom_stop_func_params, + run_every_secs=60, + run_every_steps=None +): """Custom early stop hook for complex early stop conditions. Args: @@ -310,8 +317,8 @@ def custom_early_stop_hook(estimator, if eval_dir is None: eval_dir = estimator.eval_dir() - if isinstance(custom_stop_func, str) or isinstance(custom_stop_func, - type(u'')): + if isinstance(custom_stop_func, + str) or isinstance(custom_stop_func, type(u'')): custom_stop_func = load_by_path(custom_stop_func) def _custom_stop_fn(): @@ -319,20 +326,23 @@ def _custom_stop_fn(): return custom_stop_func(eval_results, custom_stop_func_params) return make_early_stopping_hook( - estimator=estimator, - should_stop_fn=_custom_stop_fn, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + estimator=estimator, + should_stop_fn=_custom_stop_fn, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps + ) @estimator_export('estimator.experimental.stop_if_no_decrease_hook') -def stop_if_no_decrease_hook(estimator, - metric_name, - max_steps_without_decrease, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None): +def stop_if_no_decrease_hook( + estimator, + metric_name, + max_steps_without_decrease, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None +): """Creates hook to stop if metric does not decrease within given max steps. Usage example: @@ -373,14 +383,15 @@ def stop_if_no_decrease_hook(estimator, training steps, and initiates early stopping if true. """ return _stop_if_no_metric_improvement_hook( - estimator=estimator, - metric_name=metric_name, - max_steps_without_improvement=max_steps_without_decrease, - higher_is_better=False, - eval_dir=eval_dir, - min_steps=min_steps, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + estimator=estimator, + metric_name=metric_name, + max_steps_without_improvement=max_steps_without_decrease, + higher_is_better=False, + eval_dir=eval_dir, + min_steps=min_steps, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps + ) def read_eval_metrics(eval_dir): @@ -403,12 +414,14 @@ def read_eval_metrics(eval_dir): if metrics: eval_metrics_dict[event.step].update(metrics) return collections.OrderedDict( - sorted(eval_metrics_dict.items(), key=lambda t: t[0])) + sorted(eval_metrics_dict.items(), key=lambda t: t[0]) + ) -def _stop_if_threshold_crossed_hook(estimator, metric_name, threshold, - higher_is_better, eval_dir, min_steps, - run_every_secs, run_every_steps): +def _stop_if_threshold_crossed_hook( + estimator, metric_name, threshold, higher_is_better, eval_dir, min_steps, + run_every_secs, run_every_steps +): """Creates early-stopping hook to stop training if threshold is crossed.""" if eval_dir is None: eval_dir = estimator.eval_dir() @@ -426,23 +439,25 @@ def stop_if_threshold_crossed_fn(): val = metrics[metric_name] if is_lhs_better(val, threshold): tf_logging.info( - 'At step %s, metric "%s" has value %s which is %s the configured ' - 'threshold (%s) for early stopping.', step, metric_name, val, - greater_or_lesser, threshold) + 'At step %s, metric "%s" has value %s which is %s the configured ' + 'threshold (%s) for early stopping.', step, metric_name, val, + greater_or_lesser, threshold + ) return True return False return make_early_stopping_hook( - estimator=estimator, - should_stop_fn=stop_if_threshold_crossed_fn, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + estimator=estimator, + should_stop_fn=stop_if_threshold_crossed_fn, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps + ) -def _stop_if_no_metric_improvement_hook(estimator, metric_name, - max_steps_without_improvement, - higher_is_better, eval_dir, min_steps, - run_every_secs, run_every_steps): +def _stop_if_no_metric_improvement_hook( + estimator, metric_name, max_steps_without_improvement, higher_is_better, + eval_dir, min_steps, run_every_secs, run_every_steps +): """Returns hook to stop training if given metric shows no improvement.""" if eval_dir is None: eval_dir = estimator.eval_dir() @@ -465,18 +480,20 @@ def stop_if_no_metric_improvement_fn(): best_val_step = step if step - best_val_step >= max_steps_without_improvement: tf_logging.info( - 'No %s in metric "%s" for %s steps, which is greater than or equal ' - 'to max steps (%s) configured for early stopping.', - increase_or_decrease, metric_name, step - best_val_step, - max_steps_without_improvement) + 'No %s in metric "%s" for %s steps, which is greater than or equal ' + 'to max steps (%s) configured for early stopping.', + increase_or_decrease, metric_name, step - best_val_step, + max_steps_without_improvement + ) return True return False return make_early_stopping_hook( - estimator=estimator, - should_stop_fn=stop_if_no_metric_improvement_fn, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + estimator=estimator, + should_stop_fn=stop_if_no_metric_improvement_fn, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps + ) def _summaries(eval_dir): @@ -490,23 +507,26 @@ def _summaries(eval_dir): """ if gfile.Exists(eval_dir): for event_file in gfile.Glob( - os.path.join(eval_dir, _EVENT_FILE_GLOB_PATTERN)): + os.path.join(eval_dir, _EVENT_FILE_GLOB_PATTERN) + ): for event in summary_iterator.summary_iterator(event_file): yield event def _get_or_create_stop_var(): with variable_scope.variable_scope( - name_or_scope=EARLY_STOP_SIG_SCOPE, - values=[], - reuse=variable_scope.AUTO_REUSE): + name_or_scope=EARLY_STOP_SIG_SCOPE, + values=[], + reuse=variable_scope.AUTO_REUSE + ): return variable_scope.get_variable( - name=EARLY_STOP_SIG_NAME, - shape=[], - dtype=dtypes.bool, - initializer=init_ops.constant_initializer(False), - collections=[ops.GraphKeys.GLOBAL_VARIABLES], - trainable=False) + name=EARLY_STOP_SIG_NAME, + shape=[], + dtype=dtypes.bool, + initializer=init_ops.constant_initializer(False), + collections=[ops.GraphKeys.GLOBAL_VARIABLES], + trainable=False + ) class _StopOnPredicateHook(session_run_hook.SessionRunHook): @@ -518,7 +538,8 @@ def __init__(self, should_stop_fn, run_every_secs=60, run_every_steps=None): self._should_stop_fn = should_stop_fn self._timer = basic_session_run_hooks.SecondOrStepTimer( - every_secs=run_every_secs, every_steps=run_every_steps) + every_secs=run_every_secs, every_steps=run_every_steps + ) self._global_step_tensor = None self._stop_var = _get_or_create_stop_var() self._stop_op = None @@ -536,8 +557,9 @@ def after_run(self, run_context, run_values): if self._timer.should_trigger_for_step(global_step): self._timer.update_last_triggered_step(global_step) if self._should_stop_fn(): - tf_logging.info('Requesting early stopping at global step %d', - global_step) + tf_logging.info( + 'Requesting early stopping at global step %d', global_step + ) run_context.session.run(self._stop_op) run_context.request_stop() @@ -569,7 +591,8 @@ def __init__(self, model_dir, run_every_secs=10, run_every_steps=None): self._stop = False self._check_run = True self._timer = basic_session_run_hooks.SecondOrStepTimer( - every_secs=run_every_secs, every_steps=run_every_steps) + every_secs=run_every_secs, every_steps=run_every_steps + ) sleep_time = run_every_secs if run_every_secs is not None else 1 self._curr_step = 0 @@ -579,8 +602,9 @@ def _check_stop(): self._timer.update_last_triggered_step(self._curr_step) if gfile.Exists(self._stop_sig_file): self._stop = True - logging.info('OssStopSignalHook: stop on signal %s' % - self._stop_sig_file) + logging.info( + 'OssStopSignalHook: stop on signal %s' % self._stop_sig_file + ) break else: time.sleep(sleep_time) @@ -617,9 +641,10 @@ def oss_stop_hook(estimator, run_every_secs=10, run_every_steps=None): """ if estimator.config.is_chief: return OssStopSignalHook( - estimator.model_dir, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + estimator.model_dir, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps + ) else: return _CheckForStoppingHook() diff --git a/easy_rec/python/compat/embedding_ops.py b/easy_rec/python/compat/embedding_ops.py index 2ad3af7f9..09215f3b8 100644 --- a/easy_rec/python/compat/embedding_ops.py +++ b/easy_rec/python/compat/embedding_ops.py @@ -2,14 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Add embedding column for EmbeddingVariable which is only available on pai.""" -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import sparse_ops +from tensorflow.python.framework import dtypes, ops, sparse_tensor, tensor_shape # NOQA +from tensorflow.python.ops import array_ops, embedding_ops, math_ops, sparse_ops # NOQA def _prune_invalid_ids(sparse_ids, sparse_weights): @@ -17,8 +11,9 @@ def _prune_invalid_ids(sparse_ids, sparse_weights): is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) if sparse_weights is not None: is_id_valid = math_ops.logical_and( - is_id_valid, - array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) + is_id_valid, + array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool) + ) sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) if sparse_weights is not None: sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) @@ -34,14 +29,16 @@ def _prune_invalid_weights(sparse_ids, sparse_weights): return sparse_ids, sparse_weights -def safe_embedding_lookup_sparse(embedding_weights, - sparse_ids, - sparse_weights=None, - combiner='mean', - default_id=None, - name=None, - partition_strategy='div', - max_norm=None): +def safe_embedding_lookup_sparse( + embedding_weights, + sparse_ids, + sparse_weights=None, + combiner='mean', + default_id=None, + name=None, + partition_strategy='div', + max_norm=None +): """Lookup embedding results, accounting for invalid IDs and empty features. Fixed so that could be used with Pai EmbeddingVariables. @@ -91,72 +88,90 @@ def safe_embedding_lookup_sparse(embedding_weights, raise ValueError('Missing embedding_weights %s.' % embedding_weights) embed_tensors = [ops.convert_to_tensor(embedding_weights)] - with ops.name_scope(name, 'embedding_lookup', - embed_tensors + [sparse_ids, sparse_weights]) as scope: + with ops.name_scope( + name, 'embedding_lookup', embed_tensors + [sparse_ids, sparse_weights] + ) as scope: # Reshape higher-rank sparse ids and weights to linear segment ids. original_shape = sparse_ids.dense_shape original_rank_dim = sparse_ids.dense_shape.get_shape()[0] original_rank = ( - array_ops.size(original_shape) - if original_rank_dim.value is None else original_rank_dim.value) - sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ + array_ops.size(original_shape) + if original_rank_dim.value is None else original_rank_dim.value + ) + sparse_ids = sparse_ops.sparse_reshape( + sparse_ids, [ math_ops.reduce_prod( - array_ops.slice(original_shape, [0], [original_rank - 1])), + array_ops.slice(original_shape, [0], [original_rank - 1]) + ), array_ops.gather(original_shape, original_rank - 1) - ]) + ] + ) if sparse_weights is not None: - sparse_weights = sparse_tensor.SparseTensor(sparse_ids.indices, - sparse_weights.values, - sparse_ids.dense_shape) + sparse_weights = sparse_tensor.SparseTensor( + sparse_ids.indices, sparse_weights.values, sparse_ids.dense_shape + ) # Prune invalid ids and weights. sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) if combiner != 'sum': sparse_ids, sparse_weights = _prune_invalid_weights( - sparse_ids, sparse_weights) + sparse_ids, sparse_weights + ) # Fill in dummy values for empty features, if necessary. sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows( - sparse_ids, default_id or 0) + sparse_ids, default_id or 0 + ) if sparse_weights is not None: - sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) + sparse_weights, _ = sparse_ops.sparse_fill_empty_rows( + sparse_weights, 1.0 + ) indices = sparse_ids.indices values = sparse_ids.values if values.dtype != dtypes.int64: values = math_ops.to_int64(values) sparse_ids = sparse_tensor.SparseTensor( - indices=indices, values=values, dense_shape=sparse_ids.dense_shape) + indices=indices, values=values, dense_shape=sparse_ids.dense_shape + ) result = embedding_ops.embedding_lookup_sparse( - embedding_weights, - sparse_ids, - sparse_weights, - combiner=combiner, - partition_strategy=partition_strategy, - name=None if default_id is None else scope, - max_norm=max_norm) + embedding_weights, + sparse_ids, + sparse_weights, + combiner=combiner, + partition_strategy=partition_strategy, + name=None if default_id is None else scope, + max_norm=max_norm + ) if default_id is None: # Broadcast is_row_empty to the same shape as embedding_lookup_result, # for use in Select. is_row_empty = array_ops.tile( - array_ops.reshape(is_row_empty, [-1, 1]), - array_ops.stack([1, array_ops.shape(result)[1]])) + array_ops.reshape(is_row_empty, [-1, 1]), + array_ops.stack([1, array_ops.shape(result)[1]]) + ) result = array_ops.where( - is_row_empty, array_ops.zeros_like(result), result, name=scope) + is_row_empty, array_ops.zeros_like(result), result, name=scope + ) # Reshape back from linear ids back into higher-dimensional dense result. final_result = array_ops.reshape( - result, - array_ops.concat([ - array_ops.slice( - math_ops.cast(original_shape, dtypes.int32), [0], - [original_rank - 1]), - array_ops.slice(array_ops.shape(result), [1], [-1]) - ], 0)) + result, + array_ops.concat( + [ + array_ops.slice( + math_ops.cast(original_shape, dtypes.int32), [0], + [original_rank - 1] + ), + array_ops.slice(array_ops.shape(result), [1], [-1]) + ], 0 + ) + ) final_result.set_shape( - tensor_shape.unknown_shape( - (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) + tensor_shape.unknown_shape((original_rank_dim - 1).value + ).concatenate(result.get_shape()[1:]) + ) return final_result diff --git a/easy_rec/python/compat/embedding_parallel_saver.py b/easy_rec/python/compat/embedding_parallel_saver.py index 96c08c592..406a641c1 100644 --- a/easy_rec/python/compat/embedding_parallel_saver.py +++ b/easy_rec/python/compat/embedding_parallel_saver.py @@ -1,18 +1,13 @@ # -*- encoding:utf-8 -*- import logging -import os - import numpy as np +import os from tensorflow.core.protobuf import saver_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops +from tensorflow.python.framework import dtypes, ops # from tensorflow.python.ops import math_ops # from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import script_ops -from tensorflow.python.ops import state_ops +from tensorflow.python.ops import array_ops, control_flow_ops, script_ops, state_ops # NOQA from tensorflow.python.platform import gfile from tensorflow.python.training import saver @@ -21,6 +16,7 @@ try: import horovod.tensorflow as hvd from sparse_operation_kit.experiment import raw_ops as dynamic_variable_ops + from easy_rec.python.compat import dynamic_variable except Exception: dynamic_variable_ops = None @@ -28,6 +24,7 @@ try: from tensorflow.python.framework.load_library import load_op_library + import easy_rec load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) @@ -45,67 +42,76 @@ def _get_embed_part_id(embed_file): class EmbeddingParallelSaver(saver.Saver): - def __init__(self, - var_list=None, - reshape=False, - sharded=False, - max_to_keep=5, - keep_checkpoint_every_n_hours=10000.0, - name=None, - restore_sequentially=False, - saver_def=None, - builder=None, - defer_build=False, - allow_empty=False, - write_version=saver_pb2.SaverDef.V2, - pad_step_number=False, - save_relative_paths=False, - filename=None): + def __init__( + self, + var_list=None, + reshape=False, + sharded=False, + max_to_keep=5, + keep_checkpoint_every_n_hours=10000.0, + name=None, + restore_sequentially=False, + saver_def=None, + builder=None, + defer_build=False, + allow_empty=False, + write_version=saver_pb2.SaverDef.V2, + pad_step_number=False, + save_relative_paths=False, + filename=None + ): self._kv_vars = [] self._embed_vars = [] tf_vars = [] embed_para_vars = ops.get_collection(constant.EmbeddingParallel) for var in var_list: if dynamic_variable is not None and isinstance( - var, dynamic_variable.DynamicVariable): + var, dynamic_variable.DynamicVariable + ): self._kv_vars.append(var) elif var.name in embed_para_vars: - logging.info('save shard embedding %s part_id=%d part_shape=%s' % - (var.name, hvd.rank(), var.get_shape())) + logging.info( + 'save shard embedding %s part_id=%d part_shape=%s' % + (var.name, hvd.rank(), var.get_shape()) + ) self._embed_vars.append(var) else: tf_vars.append(var) super(EmbeddingParallelSaver, self).__init__( - tf_vars, - reshape=reshape, - sharded=sharded, - max_to_keep=max_to_keep, - keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours, - name=name, - restore_sequentially=restore_sequentially, - saver_def=saver_def, - builder=builder, - defer_build=defer_build, - allow_empty=allow_empty, - write_version=write_version, - pad_step_number=pad_step_number, - save_relative_paths=save_relative_paths, - filename=filename) + tf_vars, + reshape=reshape, + sharded=sharded, + max_to_keep=max_to_keep, + keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours, + name=name, + restore_sequentially=restore_sequentially, + saver_def=saver_def, + builder=builder, + defer_build=defer_build, + allow_empty=allow_empty, + write_version=write_version, + pad_step_number=pad_step_number, + save_relative_paths=save_relative_paths, + filename=filename + ) self._is_build = False def _has_embed_vars(self): return (len(self._kv_vars) + len(self._embed_vars)) > 0 def _save_dense_embedding(self, embed_var): - logging.info('task[%d] save_dense_embed: %s' % (hvd.rank(), embed_var.name)) + logging.info( + 'task[%d] save_dense_embed: %s' % (hvd.rank(), embed_var.name) + ) def _save_embed(embed, filename, var_name): task_id = hvd.rank() filename = filename.decode('utf-8') var_name = var_name.decode('utf-8').replace('/', '__') embed_dir = filename + '-embedding/' - logging.info('task[%d] save_dense_embed: %s to %s' % - (task_id, var_name, embed_dir)) + logging.info( + 'task[%d] save_dense_embed: %s to %s' % (task_id, var_name, embed_dir) + ) if not gfile.Exists(embed_dir): gfile.MakeDirs(embed_dir) embed_file = filename + '-embedding/embed-' + var_name + '-part-%d.bin' % task_id @@ -123,20 +129,23 @@ def _save_embed(embed, filename, var_name): return np.asarray([embed_file], order='C', dtype=np.object) file_name = ops.get_default_graph().get_tensor_by_name( - self.saver_def.filename_tensor_name) - save_paths = script_ops.py_func(_save_embed, - [embed_var, file_name, embed_var.name], - dtypes.string) + self.saver_def.filename_tensor_name + ) + save_paths = script_ops.py_func( + _save_embed, [embed_var, file_name, embed_var.name], dtypes.string + ) return save_paths def _load_dense_embedding(self, embed_var): file_name = ops.get_default_graph().get_tensor_by_name( - self.saver_def.filename_tensor_name) + self.saver_def.filename_tensor_name + ) embed_dim = embed_var.get_shape()[-1] embed_part_size = embed_var.get_shape()[0] - def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, - filename, var_name): + def _load_embed( + embed, embed_dim, embed_part_size, part_id, part_num, filename, var_name + ): filename = filename.decode('utf-8') var_name = var_name.decode('utf-8').replace('/', '__') embed_pattern = filename + '-embedding/embed-' + var_name + '-part-*.bin' @@ -144,10 +153,14 @@ def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, embed_files.sort(key=_get_embed_part_id) - logging.info('task[%d] embed_files=%s embed_dim=%d embed_part_size=%d' % - (part_id, ','.join(embed_files), embed_dim, embed_part_size)) + logging.info( + 'task[%d] embed_files=%s embed_dim=%d embed_part_size=%d' % + (part_id, ','.join(embed_files), embed_dim, embed_part_size) + ) - part_embed_vals = np.zeros([embed_part_size, embed_dim], dtype=np.float32) + part_embed_vals = np.zeros( + [embed_part_size, embed_dim], dtype=np.float32 + ) part_update_cnt = 0 for embed_file in embed_files: part_id_o = _get_embed_part_id(embed_file) @@ -157,8 +170,11 @@ def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, embed_ids_o = np.arange(len(embed_val)) embed_ids_o = part_id_o + embed_ids_o * len(embed_files) sel_ids = np.where( - np.logical_and((embed_ids_o % part_num) == part_id, - embed_ids_o < embed_part_size * part_num))[0] + np.logical_and( + (embed_ids_o % part_num) == part_id, embed_ids_o + < embed_part_size * part_num + ) + )[0] part_update_cnt += len(sel_ids) embed_ids = embed_ids_o[sel_ids] embed_ids_n = np.array(embed_ids / part_num, dtype=np.int64) @@ -169,26 +185,31 @@ def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, with ops.control_dependencies([embed_var._initializer_op]): if load_embed_lib is not None: embed_val = load_embed_lib.load_embed( - task_index=hvd.rank(), - task_num=hvd.size(), - embed_dim=embed_dim, - embed_part_size=embed_part_size, - var_name='embed-' + embed_var.name.replace('/', '__'), - ckpt_path=file_name) + task_index=hvd.rank(), + task_num=hvd.size(), + embed_dim=embed_dim, + embed_part_size=embed_part_size, + var_name='embed-' + embed_var.name.replace('/', '__'), + ckpt_path=file_name + ) else: - embed_val = script_ops.py_func(_load_embed, [ + embed_val = script_ops.py_func( + _load_embed, [ embed_var, embed_dim, embed_part_size, hvd.rank(), hvd.size(), file_name, embed_var.name - ], dtypes.float32) + ], dtypes.float32 + ) embed_val.set_shape(embed_var.get_shape()) return state_ops.assign(embed_var, embed_val) def _save_kv_embedding(self, sok_var): indices, values = dynamic_variable_ops.dummy_var_export( - sok_var.handle, key_type=sok_var.key_type, dtype=sok_var.handle_dtype) + sok_var.handle, key_type=sok_var.key_type, dtype=sok_var.handle_dtype + ) file_name = ops.get_default_graph().get_tensor_by_name( - self.saver_def.filename_tensor_name) + self.saver_def.filename_tensor_name + ) def _save_key_vals(indices, values, filename, var_name): var_name = var_name.decode('utf-8').replace('/', '__') @@ -217,9 +238,9 @@ def _save_key_vals(indices, values, filename, var_name): return np.asarray([key_file, val_file], order='C', dtype=np.object) - save_paths = script_ops.py_func(_save_key_vals, - [indices, values, file_name, sok_var.name], - dtypes.string) + save_paths = script_ops.py_func( + _save_key_vals, [indices, values, file_name, sok_var.name], dtypes.string + ) return save_paths def _load_kv_embedding(self, sok_var): @@ -228,11 +249,14 @@ def _load_key_vals(filename, var_name): var_name = var_name.decode('utf-8').replace('/', '__') filename = filename.decode('utf-8') key_file_pattern = filename + '-embedding/embed-' + var_name + '-part-*.key' - logging.info('key_file_pattern=%s filename=%s var_name=%s var=%s' % - (key_file_pattern, filename, var_name, str(sok_var))) + logging.info( + 'key_file_pattern=%s filename=%s var_name=%s var=%s' % + (key_file_pattern, filename, var_name, str(sok_var)) + ) key_files = gfile.Glob(key_file_pattern) - logging.info('key_file_pattern=%s file_num=%d' % - (key_file_pattern, len(key_files))) + logging.info( + 'key_file_pattern=%s file_num=%d' % (key_file_pattern, len(key_files)) + ) all_keys = [] all_vals = [] for key_file in key_files: @@ -243,16 +267,21 @@ def _load_key_vals(filename, var_name): if len(tmp_ids) == 0: break all_keys.append(tmp_keys.take(tmp_ids, axis=0)) - logging.info('part_keys.shape=%s %s %s' % (str( - tmp_keys.shape), str(tmp_ids.shape), str(all_keys[-1].shape))) + logging.info( + 'part_keys.shape=%s %s %s' % + (str(tmp_keys.shape), str(tmp_ids.shape), str(all_keys[-1].shape)) + ) val_file = key_file[:-4] + 'vals' with gfile.GFile(val_file, 'rb') as fin: - tmp_vals = np.frombuffer( - fin.read(), dtype=np.float32).reshape([-1, sok_var._dimension]) + tmp_vals = np.frombuffer(fin.read(), dtype=np.float32).reshape( + [-1, sok_var._dimension] + ) all_vals.append(tmp_vals.take(tmp_ids, axis=0)) - logging.info('part_vals.shape=%s %s %s' % (str( - tmp_vals.shape), str(tmp_ids.shape), str(all_vals[-1].shape))) + logging.info( + 'part_vals.shape=%s %s %s' % + (str(tmp_vals.shape), str(tmp_ids.shape), str(all_vals[-1].shape)) + ) all_keys = np.concatenate(all_keys, axis=0) all_vals = np.concatenate(all_vals, axis=0) @@ -264,18 +293,24 @@ def _load_key_vals(filename, var_name): return all_keys, all_vals file_name = ops.get_default_graph().get_tensor_by_name( - self.saver_def.filename_tensor_name) + self.saver_def.filename_tensor_name + ) if load_embed_lib is not None: keys, vals = load_embed_lib.load_kv_embed( - task_index=hvd.rank(), - task_num=hvd.size(), - embed_dim=sok_var._dimension, - var_name='embed-' + sok_var.name.replace('/', '__'), - ckpt_path=file_name) + task_index=hvd.rank(), + task_num=hvd.size(), + embed_dim=sok_var._dimension, + var_name='embed-' + sok_var.name.replace('/', '__'), + ckpt_path=file_name + ) else: - logging.warning('libload_embed.so not loaded, will use python script_ops') - keys, vals = script_ops.py_func(_load_key_vals, [file_name, sok_var.name], - (dtypes.int64, dtypes.float32)) + logging.warning( + 'libload_embed.so not loaded, will use python script_ops' + ) + keys, vals = script_ops.py_func( + _load_key_vals, [file_name, sok_var.name], + (dtypes.int64, dtypes.float32) + ) with ops.control_dependencies([sok_var._initializer_op]): return dynamic_variable_ops.dummy_var_assign(sok_var.handle, keys, vals) @@ -291,14 +326,16 @@ def build(self): for embed_var in self._embed_vars: restore_ops.append(self._load_dense_embedding(embed_var)) old_restore_op = ops.get_default_graph().get_operation_by_name( - self.saver_def.restore_op_name) + self.saver_def.restore_op_name + ) restore_ops.append(old_restore_op) restore_op_n = control_flow_ops.group(restore_ops) self.saver_def.restore_op_name = restore_op_n.name if self.saver_def.save_tensor_name and self._has_embed_vars(): file_name = ops.get_default_graph().get_tensor_by_name( - self.saver_def.filename_tensor_name) + self.saver_def.filename_tensor_name + ) save_part_ops = [] for sok_var in self._kv_vars: save_part_op = self._save_kv_embedding(sok_var) @@ -307,7 +344,8 @@ def build(self): save_part_op = self._save_dense_embedding(embed_var) save_part_ops.append(save_part_op) old_save_op = ops.get_default_graph().get_tensor_by_name( - self.saver_def.save_tensor_name) + self.saver_def.save_tensor_name + ) # only the first worker needs to save non embedding variables if hvd.rank() == 0: save_part_ops.append(old_save_op) diff --git a/easy_rec/python/compat/estimator_train.py b/easy_rec/python/compat/estimator_train.py index 1ec71e491..72ffec9ef 100644 --- a/easy_rec/python/compat/estimator_train.py +++ b/easy_rec/python/compat/estimator_train.py @@ -2,19 +2,15 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os - import tensorflow as tf +from tensorflow.python.distribute import estimator_training as distribute_coordinator_training # NOQA from tensorflow.python.estimator import run_config as run_config_lib -from tensorflow.python.estimator.training import _assert_eval_spec -from tensorflow.python.estimator.training import _ContinuousEvalListener -from tensorflow.python.estimator.training import _TrainingExecutor +from tensorflow.python.estimator.training import _assert_eval_spec, _ContinuousEvalListener, _TrainingExecutor # NOQA from tensorflow.python.util import compat from easy_rec.python.compat.exporter import FinalExporter from easy_rec.python.utils import estimator_utils -from tensorflow.python.distribute import estimator_training as distribute_coordinator_training # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 gfile = tf.gfile @@ -25,8 +21,9 @@ class TrainDoneListener(_ContinuousEvalListener): def __init__(self, estimator): self._model_dir = estimator.model_dir - self._train_done_file = os.path.join(self._model_dir, - 'ESTIMATOR_TRAIN_DONE') + self._train_done_file = os.path.join( + self._model_dir, 'ESTIMATOR_TRAIN_DONE' + ) @property def train_done_file(self): @@ -47,8 +44,9 @@ def after_eval(self, eval_result): latest_ckpt_path = estimator_utils.latest_checkpoint(model_dir) if latest_ckpt_path != last_ckpt_path: logging.info( - 'TrainDoneListener: latest_ckpt_path[%s] != last_ckpt_path[%s]' % - (latest_ckpt_path, last_ckpt_path)) + 'TrainDoneListener: latest_ckpt_path[%s] != last_ckpt_path[%s]' % + (latest_ckpt_path, last_ckpt_path) + ) # there are more checkpoints wait to be evaluated return True return not gfile.Exists(self._train_done_file) @@ -60,26 +58,30 @@ def train_and_evaluate(estimator, train_spec, eval_spec): train_done_listener = TrainDoneListener(estimator) executor = _TrainingExecutor( - estimator=estimator, - train_spec=train_spec, - eval_spec=eval_spec, - continuous_eval_listener=train_done_listener) + estimator=estimator, + train_spec=train_spec, + eval_spec=eval_spec, + continuous_eval_listener=train_done_listener + ) config = estimator.config # If `distribute_coordinator_mode` is set and running in distributed # environment, we run `train_and_evaluate` via distribute coordinator. if distribute_coordinator_training.should_run_distribute_coordinator(config): logging.info('Running `train_and_evaluate` with Distribute Coordinator.') - distribute_coordinator_training.train_and_evaluate(estimator, train_spec, - eval_spec, - _TrainingExecutor) + distribute_coordinator_training.train_and_evaluate( + estimator, train_spec, eval_spec, _TrainingExecutor + ) return - if (config.task_type == run_config_lib.TaskType.EVALUATOR and - config.task_id > 0): + if ( + config.task_type == run_config_lib.TaskType.EVALUATOR + and config.task_id > 0 + ): raise ValueError( - 'For distributed training, there can only be one `evaluator` task ' - '(with task id 0). Given task id {}'.format(config.task_id)) + 'For distributed training, there can only be one `evaluator` task ' + '(with task id 0). Given task id {}'.format(config.task_id) + ) result = executor.run() @@ -87,22 +89,25 @@ def train_and_evaluate(estimator, train_spec, eval_spec): # before num_steps is reached or num_steps is set to infinite if estimator_utils.is_evaluator(): export_dir_base = os.path.join( - compat.as_str_any(estimator.model_dir), compat.as_str_any('export')) + compat.as_str_any(estimator.model_dir), compat.as_str_any('export') + ) for exporter in eval_spec.exporters: if isinstance(exporter, FinalExporter): export_path = os.path.join( - compat.as_str_any(export_dir_base), - compat.as_str_any(exporter.name)) + compat.as_str_any(export_dir_base), compat.as_str_any(exporter.name) + ) # avoid duplicate export if gfile.IsDirectory(export_path + '/'): continue exporter.export( - estimator=estimator, - export_path=export_path, - checkpoint_path=estimator_utils.latest_checkpoint( - estimator.model_dir), - eval_result=None, - is_the_final_export=True) + estimator=estimator, + export_path=export_path, + checkpoint_path=estimator_utils.latest_checkpoint( + estimator.model_dir + ), + eval_result=None, + is_the_final_export=True + ) if estimator_utils.is_chief(): with gfile.GFile(train_done_listener.train_done_file, 'w') as fout: diff --git a/easy_rec/python/compat/exporter.py b/easy_rec/python/compat/exporter.py index d8e3ed418..3bbd4af62 100644 --- a/easy_rec/python/compat/exporter.py +++ b/easy_rec/python/compat/exporter.py @@ -15,20 +15,14 @@ # ============================================================================== """`Exporter` class represents different flavors of model export.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os - -from tensorflow.python.estimator import gc -from tensorflow.python.estimator import util +from tensorflow.python.estimator import gc, util from tensorflow.python.estimator.canned import metric_keys -from tensorflow.python.estimator.exporter import Exporter -from tensorflow.python.estimator.exporter import _SavedModelExporter +from tensorflow.python.estimator.exporter import Exporter, _SavedModelExporter from tensorflow.python.framework import errors_impl -from tensorflow.python.platform import gfile -from tensorflow.python.platform import tf_logging +from tensorflow.python.platform import gfile, tf_logging from tensorflow.python.summary import summary_iterator from easy_rec.python.utils import io_util @@ -53,11 +47,13 @@ def _loss_smaller(best_eval_result, current_eval_result): default_key = metric_keys.MetricKeys.LOSS if not best_eval_result or default_key not in best_eval_result: raise ValueError( - 'best_eval_result cannot be empty or no loss is found in it.') + 'best_eval_result cannot be empty or no loss is found in it.' + ) if not current_eval_result or default_key not in current_eval_result: raise ValueError( - 'current_eval_result cannot be empty or no loss is found in it.') + 'current_eval_result cannot be empty or no loss is found in it.' + ) return best_eval_result[default_key] > current_eval_result[default_key] @@ -66,16 +62,21 @@ def _verify_compare_fn_args(compare_fn): """Verifies compare_fn arguments.""" args = set(util.fn_args(compare_fn)) if 'best_eval_result' not in args: - raise ValueError('compare_fn (%s) must include best_eval_result argument.' % - compare_fn) + raise ValueError( + 'compare_fn (%s) must include best_eval_result argument.' % compare_fn + ) if 'current_eval_result' not in args: raise ValueError( - 'compare_fn (%s) must include current_eval_result argument.' % - compare_fn) - non_valid_args = list(args - set(['best_eval_result', 'current_eval_result'])) + 'compare_fn (%s) must include current_eval_result argument.' % compare_fn + ) + non_valid_args = list( + args - set(['best_eval_result', 'current_eval_result']) + ) if non_valid_args: - raise ValueError('compare_fn (%s) has following not expected args: %s' % - (compare_fn, non_valid_args)) + raise ValueError( + 'compare_fn (%s) has following not expected args: %s' % + (compare_fn, non_valid_args) + ) def _get_ckpt_version(path): @@ -92,14 +93,16 @@ class BestExporter(Exporter): existing model. """ - def __init__(self, - name='best_exporter', - serving_input_receiver_fn=None, - event_file_pattern='eval_val/*.tfevents.*', - compare_fn=_loss_smaller, - assets_extra=None, - as_text=False, - exports_to_keep=5): + def __init__( + self, + name='best_exporter', + serving_input_receiver_fn=None, + event_file_pattern='eval_val/*.tfevents.*', + compare_fn=_loss_smaller, + assets_extra=None, + as_text=False, + exports_to_keep=5 + ): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. Example of creating a BestExporter for training and evaluation: @@ -180,9 +183,9 @@ def make_train_and_eval_fn(): raise ValueError('`compare_fn` must not be None.') _verify_compare_fn_args(self._compare_fn) - self._saved_model_exporter = _SavedModelExporter(name, - serving_input_receiver_fn, - assets_extra, as_text) + self._saved_model_exporter = _SavedModelExporter( + name, serving_input_receiver_fn, assets_extra, as_text + ) self._event_file_pattern = event_file_pattern self._model_dir = None @@ -191,15 +194,18 @@ def make_train_and_eval_fn(): self._exports_to_keep = exports_to_keep if exports_to_keep is not None and exports_to_keep <= 0: raise ValueError( - '`exports_to_keep`, if provided, must be a positive number. Got %s' % - exports_to_keep) + '`exports_to_keep`, if provided, must be a positive number. Got %s' % + exports_to_keep + ) @property def name(self): return self._saved_model_exporter.name - def export(self, estimator, export_path, checkpoint_path, eval_result, - is_the_final_export): + def export( + self, estimator, export_path, checkpoint_path, eval_result, + is_the_final_export + ): export_result = None if self._model_dir != estimator.model_dir and self._event_file_pattern: @@ -207,27 +213,30 @@ def export(self, estimator, export_path, checkpoint_path, eval_result, tf_logging.info('Loading best metric from event files.') self._model_dir = estimator.model_dir - full_event_file_pattern = os.path.join(self._model_dir, - self._event_file_pattern) + full_event_file_pattern = os.path.join( + self._model_dir, self._event_file_pattern + ) self._best_eval_result = self._get_best_eval_result( - full_event_file_pattern, eval_result) + full_event_file_pattern, eval_result + ) if self._best_eval_result is None or self._compare_fn( - best_eval_result=self._best_eval_result, - current_eval_result=eval_result): + best_eval_result=self._best_eval_result, current_eval_result=eval_result + ): tf_logging.info('Performing best model export.') self._best_eval_result = eval_result - export_result = self._saved_model_exporter.export(estimator, export_path, - checkpoint_path, - eval_result, - is_the_final_export) + export_result = self._saved_model_exporter.export( + estimator, export_path, checkpoint_path, eval_result, + is_the_final_export + ) self._garbage_collect_exports(export_path) # cp best checkpoints to best folder model_dir, _ = os.path.split(checkpoint_path) # add / is to be compatiable with oss best_dir = os.path.join(model_dir, 'best_ckpt/') - tf_logging.info('Copy best checkpoint %s to %s' % - (checkpoint_path, best_dir)) + tf_logging.info( + 'Copy best checkpoint %s to %s' % (checkpoint_path, best_dir) + ) if not gfile.Exists(best_dir): gfile.MakeDirs(best_dir) for tmp_file in gfile.Glob(checkpoint_path + '.*'): @@ -240,8 +249,9 @@ def export(self, estimator, export_path, checkpoint_path, eval_result, try: gfile.Copy(tmp_file, dst_path) except Exception as ex: - tf_logging.warn('Copy file %s to %s failed: %s' % - (tmp_file, dst_path, str(ex))) + tf_logging.warn( + 'Copy file %s to %s failed: %s' % (tmp_file, dst_path, str(ex)) + ) self._garbage_collect_ckpts(best_dir) return export_result @@ -264,8 +274,9 @@ def _garbage_collect_ckpts(self, best_dir): tmp_steps = sorted(tmp_steps) drop_num = len(tmp_steps) - self._exports_to_keep tf_logging.info( - 'garbage_collect_ckpts: steps: %s export_to_keep: %d drop num: %d' % - (str(tmp_steps), self._exports_to_keep, drop_num)) + 'garbage_collect_ckpts: steps: %s export_to_keep: %d drop num: %d' % + (str(tmp_steps), self._exports_to_keep, drop_num) + ) for ver in tmp_steps[:drop_num]: tmp_prefix = os.path.join(best_dir, 'model.ckpt-%d.*' % ver) for tmp_file in gfile.Glob(tmp_prefix): @@ -296,7 +307,8 @@ def _export_version_parser(path): keep_filter = gc._largest_export_versions(self._exports_to_keep) delete_filter = gc._negation(keep_filter) for p in delete_filter( - gc._get_paths(export_dir_base, parser=_export_version_parser)): + gc._get_paths(export_dir_base, parser=_export_version_parser) + ): try: gfile.DeleteRecursively(io_util.fix_oss_dir(p.path)) except errors_impl.NotFoundError as e: @@ -328,7 +340,8 @@ def _get_best_eval_result(self, event_files, curr_eval_result): event_eval_result[value.tag] = value.simple_value if len(event_eval_result) >= 2: if best_eval_result is None or self._compare_fn( - best_eval_result, event_eval_result): + best_eval_result, event_eval_result + ): best_eval_result = event_eval_result return best_eval_result @@ -339,11 +352,9 @@ class FinalExporter(Exporter): This class performs a single export at the end of training. """ - def __init__(self, - name, - serving_input_receiver_fn, - assets_extra=None, - as_text=False): + def __init__( + self, name, serving_input_receiver_fn, assets_extra=None, as_text=False + ): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. Args: @@ -364,24 +375,26 @@ def __init__(self, Raises: ValueError: if any arguments is invalid. """ - self._saved_model_exporter = _SavedModelExporter(name, - serving_input_receiver_fn, - assets_extra, as_text) + self._saved_model_exporter = _SavedModelExporter( + name, serving_input_receiver_fn, assets_extra, as_text + ) @property def name(self): return self._saved_model_exporter.name - def export(self, estimator, export_path, checkpoint_path, eval_result, - is_the_final_export): + def export( + self, estimator, export_path, checkpoint_path, eval_result, + is_the_final_export + ): if not is_the_final_export: return None tf_logging.info('Performing the final export in the end of training.') - return self._saved_model_exporter.export(estimator, export_path, - checkpoint_path, eval_result, - is_the_final_export) + return self._saved_model_exporter.export( + estimator, export_path, checkpoint_path, eval_result, is_the_final_export + ) class LatestExporter(Exporter): @@ -390,12 +403,14 @@ class LatestExporter(Exporter): In addition to exporting, this class also garbage collects stale exports. """ - def __init__(self, - name, - serving_input_receiver_fn, - assets_extra=None, - as_text=False, - exports_to_keep=5): + def __init__( + self, + name, + serving_input_receiver_fn, + assets_extra=None, + as_text=False, + exports_to_keep=5 + ): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. Args: @@ -419,24 +434,26 @@ def __init__(self, Raises: ValueError: if any arguments is invalid. """ - self._saved_model_exporter = _SavedModelExporter(name, - serving_input_receiver_fn, - assets_extra, as_text) + self._saved_model_exporter = _SavedModelExporter( + name, serving_input_receiver_fn, assets_extra, as_text + ) self._exports_to_keep = exports_to_keep if exports_to_keep is not None and exports_to_keep <= 0: raise ValueError( - '`exports_to_keep`, if provided, must be positive number') + '`exports_to_keep`, if provided, must be positive number' + ) @property def name(self): return self._saved_model_exporter.name - def export(self, estimator, export_path, checkpoint_path, eval_result, - is_the_final_export): - export_result = self._saved_model_exporter.export(estimator, export_path, - checkpoint_path, - eval_result, - is_the_final_export) + def export( + self, estimator, export_path, checkpoint_path, eval_result, + is_the_final_export + ): + export_result = self._saved_model_exporter.export( + estimator, export_path, checkpoint_path, eval_result, is_the_final_export + ) self._garbage_collect_exports(export_path) return export_result @@ -465,7 +482,8 @@ def _export_version_parser(path): keep_filter = gc._largest_export_versions(self._exports_to_keep) delete_filter = gc._negation(keep_filter) for p in delete_filter( - gc._get_paths(export_dir_base, parser=_export_version_parser)): + gc._get_paths(export_dir_base, parser=_export_version_parser) + ): try: gfile.DeleteRecursively(io_util.fix_oss_dir(p.path)) except errors_impl.NotFoundError as e: diff --git a/easy_rec/python/compat/feature_column/feature_column.py b/easy_rec/python/compat/feature_column/feature_column.py index 1981ac4a3..b2acd612d 100644 --- a/easy_rec/python/compat/feature_column/feature_column.py +++ b/easy_rec/python/compat/feature_column/feature_column.py @@ -128,41 +128,22 @@ in both places. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import abc import collections import math -import os - import numpy as np +import os import six from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops +from tensorflow.python.framework import dtypes, ops from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib from tensorflow.python.framework import tensor_shape from tensorflow.python.keras.engine import training from tensorflow.python.layers import base # from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.ops import template -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables +from tensorflow.python.ops import array_ops, check_ops, control_flow_ops, data_flow_ops, embedding_ops, init_ops, lookup_ops, math_ops, nn_ops, parsing_ops, resource_variable_ops, sparse_ops, string_ops, template, variable_scope, variables # NOQA # from tensorflow.python.ops.ragged import ragged_tensor # from tensorflow.python.ops.ragged import ragged_util from tensorflow.python.platform import gfile @@ -171,9 +152,7 @@ from tensorflow.python.util import nest from easy_rec.python.compat.feature_column import utils as fc_utils -from easy_rec.python.utils import conditional -from easy_rec.python.utils import constant -from easy_rec.python.utils import embedding_utils +from easy_rec.python.utils import conditional, constant, embedding_utils try: from easy_rec.python.compat import dynamic_variable @@ -186,19 +165,22 @@ hvd = None -def embedding_lookup_ragged(embedding_weights, - ragged_ids, - ragged_weights, - combiner, - max_norm=None, - name=None): +def embedding_lookup_ragged( + embedding_weights, + ragged_ids, + ragged_weights, + combiner, + max_norm=None, + name=None +): segment_ids = ragged_ids.value_rowids() if segment_ids.dtype != dtypes.int32: segment_ids = math_ops.cast(segment_ids, dtypes.int32) ids = ragged_ids.flat_values ids, idx = array_ops.unique(ids) embeddings = embedding_ops.embedding_lookup( - embedding_weights, ids, partition_strategy='mod', max_norm=max_norm) + embedding_weights, ids, partition_strategy='mod', max_norm=max_norm + ) if ragged_weights is not None: weights = ragged_weights.flat_values embeddings = array_ops.gather(embeddings, idx) @@ -232,61 +214,73 @@ def embedding_lookup_ragged(embedding_weights, assert idx is not None if combiner == 'sum': embeddings = math_ops.sparse_segment_sum( - embeddings, idx, segment_ids, name=name) + embeddings, idx, segment_ids, name=name + ) elif combiner == 'mean': embeddings = math_ops.sparse_segment_mean( - embeddings, idx, segment_ids, name=name) + embeddings, idx, segment_ids, name=name + ) elif combiner == 'sqrtn': embeddings = math_ops.sparse_segment_sqrt_n( - embeddings, idx, segment_ids, name=name) + embeddings, idx, segment_ids, name=name + ) else: assert False, 'Unrecognized combiner' return embeddings # model parallel embedding lookup -def embedding_parallel_lookup(embedding, - lookup_indices, - output_ids, - is_training, - output_tensors=None, - batch_size=None): +def embedding_parallel_lookup( + embedding, + lookup_indices, + output_ids, + is_training, + output_tensors=None, + batch_size=None +): N = len(output_ids) if batch_size is None: num_segments = None else: num_segments = N * batch_size # first concat all the ids and unique - if isinstance(lookup_indices, dict) and 'sparse_fea' in lookup_indices.keys(): + if isinstance(lookup_indices, + dict) and 'sparse_fea' in lookup_indices.keys(): # all_uniq_ids, uniq_idx, segment_lens = features['sparse_fea'] all_ids, segment_lens = lookup_indices['sparse_fea'] all_uniq_ids, uniq_idx = array_ops.unique(all_ids) cumsum_lens = math_ops.cumsum(segment_lens) segment_ids = array_ops.searchsorted( - cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') - elif isinstance(lookup_indices, dict) and 'ragged_ids' in lookup_indices.keys( - ) and 'ragged_lens' in lookup_indices.keys(): + cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right' + ) + elif isinstance(lookup_indices, + dict) and 'ragged_ids' in lookup_indices.keys( + ) and 'ragged_lens' in lookup_indices.keys(): all_ids, segment_lens = lookup_indices['ragged_ids'], lookup_indices[ - 'ragged_lens'] + 'ragged_lens'] all_uniq_ids, uniq_idx = array_ops.unique(all_ids) cumsum_lens = math_ops.cumsum(segment_lens) segment_ids = array_ops.searchsorted( - cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') + cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right' + ) elif isinstance(lookup_indices[0], sparse_tensor_lib.SparseTensor): with ops.device('/cpu:0'): all_ids = array_ops.concat([x.values for x in lookup_indices], axis=0) - segment_ids = array_ops.concat([x.indices[:, 0] for x in lookup_indices], - axis=0) + segment_ids = array_ops.concat( + [x.indices[:, 0] for x in lookup_indices], axis=0 + ) all_uniq_ids, uniq_idx = array_ops.unique(all_ids) elif 'RaggedTensor' in str(type(lookup_indices[0])): with ops.device('/cpu:0'): all_ids = array_ops.concat([x.values for x in lookup_indices], axis=0) - segment_lens = array_ops.concat([x.row_lengths() for x in lookup_indices], - axis=0) + segment_lens = array_ops.concat( + [x.row_lengths() for x in lookup_indices], axis=0 + ) all_uniq_ids, uniq_idx = array_ops.unique(all_ids) cumsum_lens = math_ops.cumsum(segment_lens) segment_ids = array_ops.searchsorted( - cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') + cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right' + ) else: assert False, 'invalid indices type: %s' % str(type(lookup_indices[0])) @@ -294,22 +288,25 @@ def embedding_parallel_lookup(embedding, if num_parts > 1: # dynamic partition p_assignments = math_ops.cast(all_uniq_ids % num_parts, dtypes.int32) - gather_ids = data_flow_ops.dynamic_partition(all_uniq_ids, p_assignments, - num_parts) + gather_ids = data_flow_ops.dynamic_partition( + all_uniq_ids, p_assignments, num_parts + ) original_ids = math_ops.range(array_ops.size(all_uniq_ids)) - original_part_ids = data_flow_ops.dynamic_partition(original_ids, - p_assignments, - num_parts) + original_part_ids = data_flow_ops.dynamic_partition( + original_ids, p_assignments, num_parts + ) # all2all - split_sizes = array_ops.concat([array_ops.shape(x) for x in gather_ids], - axis=0) + split_sizes = array_ops.concat( + [array_ops.shape(x) for x in gather_ids], axis=0 + ) send_ids = array_ops.concat(gather_ids, axis=0) recv_ids, recv_lens = hvd.alltoall(send_ids, split_sizes) # read embedding from dynamic variable if isinstance(embedding, dynamic_variable.DynamicVariable): send_embed = embedding.sparse_read( - recv_ids, lookup_only=(not is_training)) + recv_ids, lookup_only=(not is_training) + ) else: # find in subarray position # 0 2 4 6 8 10 ... @@ -320,27 +317,32 @@ def embedding_parallel_lookup(embedding, # all2all recv_embeddings, _ = hvd.alltoall(send_embed, recv_lens) recv_embeddings = array_ops.split( - recv_embeddings, num_or_size_splits=split_sizes) + recv_embeddings, num_or_size_splits=split_sizes + ) recv_embeddings = data_flow_ops.parallel_dynamic_stitch( - original_part_ids, recv_embeddings, name='parallel_dynamic_stitch') + original_part_ids, recv_embeddings, name='parallel_dynamic_stitch' + ) embeddings = math_ops.sparse_segment_sum( - recv_embeddings, - uniq_idx, - segment_ids, - num_segments=num_segments, - name='sparse_segment_sum') + recv_embeddings, + uniq_idx, + segment_ids, + num_segments=num_segments, + name='sparse_segment_sum' + ) else: if isinstance(embedding, dynamic_variable.DynamicVariable): recv_embeddings = embedding.sparse_read( - all_uniq_ids, lookup_only=(not is_training)) + all_uniq_ids, lookup_only=(not is_training) + ) else: recv_embeddings = array_ops.gather(embedding, all_uniq_ids) embeddings = math_ops.sparse_segment_sum( - recv_embeddings, - uniq_idx, - segment_ids, - num_segments=num_segments, - name='sparse_segment_sum') + recv_embeddings, + uniq_idx, + segment_ids, + num_segments=num_segments, + name='sparse_segment_sum' + ) embed_dim = embedding.get_shape()[-1] output_tensor = array_ops.reshape(embeddings, [N, -1, embed_dim]) @@ -353,28 +355,32 @@ def embedding_parallel_lookup(embedding, if batch_size is None: batch_size = -1 return array_ops.reshape( - array_ops.transpose(output_tensor, perm=[1, 0, 2]), - [batch_size, N * embed_dim]) - - -def _internal_input_layer(features, - feature_columns, - weight_collections=None, - trainable=True, - cols_to_vars=None, - scope=None, - cols_to_output_tensors=None, - from_template=False, - feature_name_to_output_tensors=None, - is_training=True): + array_ops.transpose(output_tensor, perm=[1, 0, 2]), + [batch_size, N * embed_dim] + ) + + +def _internal_input_layer( + features, + feature_columns, + weight_collections=None, + trainable=True, + cols_to_vars=None, + scope=None, + cols_to_output_tensors=None, + from_template=False, + feature_name_to_output_tensors=None, + is_training=True +): """See input_layer, `scope` is a name or variable scope to use.""" feature_columns = _normalize_feature_columns(feature_columns) for column in feature_columns: if not isinstance(column, _DenseColumn): raise ValueError( - 'Items of feature_columns must be a _DenseColumn. ' - 'You can wrap a categorical column with an ' - 'embedding_column or indicator_column. Given: {}'.format(column)) + 'Items of feature_columns must be a _DenseColumn. ' + 'You can wrap a categorical column with an ' + 'embedding_column or indicator_column. Given: {}'.format(column) + ) weight_collections = list(weight_collections or []) if ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections: weight_collections.append(ops.GraphKeys.GLOBAL_VARIABLES) @@ -391,7 +397,8 @@ def _get_logits(): # pylint: disable=missing-docstring tmp_cols = sorted(tmp_cols, key=lambda x: x.name) for column in tmp_cols: with variable_scope.variable_scope( - None, default_name=column._var_scope_name): # pylint: disable=protected-access + None, default_name=column._var_scope_name + ): # pylint: disable=protected-access tensor = column._get_dense_tensor( # pylint: disable=protected-access builder, weight_collections=weight_collections, @@ -399,14 +406,16 @@ def _get_logits(): # pylint: disable=missing-docstring num_elements = column._variable_shape.num_elements() # pylint: disable=protected-access batch_size = array_ops.shape(tensor)[0] output_tensor = array_ops.reshape( - tensor, shape=(batch_size, num_elements)) + tensor, shape=(batch_size, num_elements) + ) output_tensors.append(output_tensor) if cols_to_vars is not None: # Retrieve any variables created (some _DenseColumn's don't create # variables, in which case an empty list is returned). cols_to_vars[column] = ops.get_collection( - ops.GraphKeys.GLOBAL_VARIABLES, - scope=variable_scope.get_variable_scope().name) + ops.GraphKeys.GLOBAL_VARIABLES, + scope=variable_scope.get_variable_scope().name + ) if cols_to_output_tensors is not None: cols_to_output_tensors[column] = output_tensor if feature_name_to_output_tensors is not None: @@ -448,7 +457,8 @@ def _get_var_type(column): for column in feature_columns: ordered_columns.append(column) with variable_scope.variable_scope( - None, default_name=column._var_scope_name): # pylint: disable=protected-access + None, default_name=column._var_scope_name + ): # pylint: disable=protected-access # for features which does not require embedding if 'Embedding' not in str(type(column)): dense_cols.append(column) @@ -470,29 +480,6 @@ def _get_var_type(column): if column.ev_params is not None: assert dynamic_variable is not None, 'sok is not installed' embedding_weights = dynamic_variable.DynamicVariable( - name='embedding_weights', - dimension=column.dimension, - initializer='random {"stddev":0.0025}', # column.initializer, - var_type=_get_var_type(column), - trainable=column.trainable and trainable, - dtype=dtypes.float32, - init_capacity=column.ev_params.init_capacity, - max_capacity=column.ev_params.max_capacity) - else: - embedding_weights = variable_scope.get_variable( - name='embedding_weights', - shape=embedding_shape, - dtype=dtypes.float32, - initializer=column.initializer, - trainable=column.trainable and trainable, - partitioner=None, - collections=weight_collections) - shared_weights[shared_name] = embedding_weights - else: - with ops.device(embedding_device): - if column.ev_params is not None: - assert dynamic_variable is not None, 'sok is not installed' - embedding_weights = dynamic_variable.DynamicVariable( name='embedding_weights', dimension=column.dimension, initializer='random {"stddev":0.0025}', # column.initializer, @@ -500,16 +487,43 @@ def _get_var_type(column): trainable=column.trainable and trainable, dtype=dtypes.float32, init_capacity=column.ev_params.init_capacity, - max_capacity=column.ev_params.max_capacity) - else: - embedding_weights = variable_scope.get_variable( + max_capacity=column.ev_params.max_capacity + ) + else: + embedding_weights = variable_scope.get_variable( name='embedding_weights', shape=embedding_shape, dtype=dtypes.float32, initializer=column.initializer, trainable=column.trainable and trainable, partitioner=None, - collections=weight_collections) + collections=weight_collections + ) + shared_weights[shared_name] = embedding_weights + else: + with ops.device(embedding_device): + if column.ev_params is not None: + assert dynamic_variable is not None, 'sok is not installed' + embedding_weights = dynamic_variable.DynamicVariable( + name='embedding_weights', + dimension=column.dimension, + initializer='random {"stddev":0.0025}', # column.initializer, + var_type=_get_var_type(column), + trainable=column.trainable and trainable, + dtype=dtypes.float32, + init_capacity=column.ev_params.init_capacity, + max_capacity=column.ev_params.max_capacity + ) + else: + embedding_weights = variable_scope.get_variable( + name='embedding_weights', + shape=embedding_shape, + dtype=dtypes.float32, + initializer=column.initializer, + trainable=column.trainable and trainable, + partitioner=None, + collections=weight_collections + ) lookup_embeddings.append(embedding_weights) output_id = len(output_tensors) output_tensors.append(None) @@ -526,8 +540,8 @@ def _get_var_type(column): elif 'ragged_ids' in features.keys(): if lookup_indices is None: lookup_indices = { - 'ragged_ids': features['ragged_ids'], - 'ragged_lens': features['ragged_lens'] + 'ragged_ids': features['ragged_ids'], + 'ragged_lens': features['ragged_lens'] } if 'ragged_wgts' in features: lookup_indices['ragged_wgts'] = features['ragged_wgts'] @@ -536,24 +550,26 @@ def _get_var_type(column): lookup_indices = [] with ops.device('/cpu:0'): sparse_tensors = column.categorical_column._get_sparse_tensors( - builder, - weight_collections=weight_collections, - trainable=trainable) + builder, + weight_collections=weight_collections, + trainable=trainable + ) lookup_indices.append(sparse_tensors.id_tensor) if sparse_tensors.weight_tensor is not None: lookup_wgts.append(sparse_tensors.weight_tensor) if cols_to_vars is not None: cols_to_vars[column] = ops.get_collection( - ops.GraphKeys.GLOBAL_VARIABLES, - scope=variable_scope.get_variable_scope().name) + ops.GraphKeys.GLOBAL_VARIABLES, + scope=variable_scope.get_variable_scope().name + ) if dense_cnt > 0: if 'dense_fea' in features: fea_dim_s = 0 for dense_output_id, dense_col in zip(dense_output_ids, dense_cols): fea_dim_e = fea_dim_s + dense_col.shape[0] - output_tensors[dense_output_id] = features[ - 'dense_fea'][:, fea_dim_s:fea_dim_e] + output_tensors[dense_output_id] = features['dense_fea' + ][:, fea_dim_s:fea_dim_e] fea_dim_s = fea_dim_e batch_sizes.append(array_ops.shape(features['dense_fea'])[0]) else: @@ -574,10 +590,10 @@ def _get_var_type(column): if packed_input: uniq_embed_cnt = len(set(lookup_embeddings)) assert uniq_embed_cnt == 1, 'only one uniq embed is support for packed inputs' - outputs = embedding_parallel_lookup(lookup_embeddings[0], - lookup_indices, lookup_output_ids, - is_training, output_tensors, - batch_size) + outputs = embedding_parallel_lookup( + lookup_embeddings[0], lookup_indices, lookup_output_ids, is_training, + output_tensors, batch_size + ) else: if batch_size is None: all_indices = [] @@ -587,13 +603,13 @@ def _get_var_type(column): batch_size = math_ops.reduce_max(all_indices) + 1 # group lookup_embeddings grouped_inputs = {} - for embedding, lookup_indice, output_id in zip(lookup_embeddings, - lookup_indices, - lookup_output_ids): + for embedding, lookup_indice, output_id in zip( + lookup_embeddings, lookup_indices, lookup_output_ids + ): if embedding not in grouped_inputs: grouped_inputs[embedding] = { - 'lookup_indice': [lookup_indice], - 'output_id': [output_id] + 'lookup_indice': [lookup_indice], + 'output_id': [output_id] } else: grouped_inputs[embedding]['lookup_indice'].append(lookup_indice) @@ -602,9 +618,10 @@ def _get_var_type(column): for embedding in grouped_inputs: lookup_indices = grouped_inputs[embedding]['lookup_indice'] output_ids = grouped_inputs[embedding]['output_id'] - outputs = embedding_parallel_lookup(embedding, lookup_indices, - output_ids, is_training, - output_tensors, batch_size) + outputs = embedding_parallel_lookup( + embedding, lookup_indices, output_ids, is_training, output_tensors, + batch_size + ) for output_tensor, col in zip(output_tensors, feature_columns): if feature_name_to_output_tensors is not None: @@ -631,23 +648,27 @@ def _get_var_type(column): return _get_logits() else: with variable_scope.variable_scope( - scope, default_name='input_layer', values=features.values()): + scope, default_name='input_layer', values=features.values() + ): if embedding_utils.is_embedding_parallel(): return _get_logits_embedding_parallel() else: - with conditional(embedding_utils.embedding_on_cpu(), - ops.device('/cpu:0')): + with conditional( + embedding_utils.embedding_on_cpu(), ops.device('/cpu:0') + ): return _get_logits() -def input_layer(features, - feature_columns, - weight_collections=None, - trainable=True, - cols_to_vars=None, - cols_to_output_tensors=None, - feature_name_to_output_tensors=None, - is_training=True): +def input_layer( + features, + feature_columns, + weight_collections=None, + trainable=True, + cols_to_vars=None, + cols_to_output_tensors=None, + feature_name_to_output_tensors=None, + is_training=True +): """Returns a dense `Tensor` as input layer based on given `feature_columns`. Generally a single example in training data is described with FeatureColumns. @@ -705,14 +726,15 @@ def input_layer(features, ValueError: if an item in `feature_columns` is not a `_DenseColumn`. """ return _internal_input_layer( - features, - feature_columns, - weight_collections=weight_collections, - trainable=trainable, - cols_to_vars=cols_to_vars, - cols_to_output_tensors=cols_to_output_tensors, - feature_name_to_output_tensors=feature_name_to_output_tensors, - is_training=is_training) + features, + feature_columns, + weight_collections=weight_collections, + trainable=trainable, + cols_to_vars=cols_to_vars, + cols_to_output_tensors=cols_to_output_tensors, + feature_name_to_output_tensors=feature_name_to_output_tensors, + is_training=is_training + ) # TODO(akshayka): InputLayer should be a subclass of Layer, and it @@ -722,13 +744,15 @@ def input_layer(features, class InputLayer(object): """An object-oriented version of `input_layer` that reuses variables.""" - def __init__(self, - feature_columns, - weight_collections=None, - trainable=True, - cols_to_vars=None, - name='feature_column_input_layer', - create_scope_now=True): + def __init__( + self, + feature_columns, + weight_collections=None, + trainable=True, + cols_to_vars=None, + name='feature_column_input_layer', + create_scope_now=True + ): """See `input_layer`.""" self._feature_columns = feature_columns self._weight_collections = weight_collections @@ -736,17 +760,19 @@ def __init__(self, self._cols_to_vars = cols_to_vars self._name = name self._input_layer_template = template.make_template( - self._name, _internal_input_layer, create_scope_now_=create_scope_now) + self._name, _internal_input_layer, create_scope_now_=create_scope_now + ) self._scope = self._input_layer_template.variable_scope def __call__(self, features): return self._input_layer_template( - features=features, - feature_columns=self._feature_columns, - weight_collections=self._weight_collections, - trainable=self._trainable, - cols_to_vars=None, - from_template=True) + features=features, + feature_columns=self._feature_columns, + weight_collections=self._weight_collections, + trainable=self._trainable, + cols_to_vars=None, + from_template=True + ) @property def name(self): @@ -777,13 +803,15 @@ def weights(self): return self._input_layer_template.weights -def linear_model(features, - feature_columns, - units=1, - sparse_combiner='sum', - weight_collections=None, - trainable=True, - cols_to_vars=None): +def linear_model( + features, + feature_columns, + units=1, + sparse_combiner='sum', + weight_collections=None, + trainable=True, + cols_to_vars=None +): """Returns a linear prediction `Tensor` based on given `feature_columns`. This function generates a weighted sum based on output dimension `units`. @@ -902,12 +930,13 @@ def linear_model(features, with variable_scope.variable_scope(None, 'linear_model') as vs: model_name = _strip_leading_slashes(vs.name) linear_model_layer = _LinearModel( - feature_columns=feature_columns, - units=units, - sparse_combiner=sparse_combiner, - weight_collections=weight_collections, - trainable=trainable, - name=model_name) + feature_columns=feature_columns, + units=units, + sparse_combiner=sparse_combiner, + weight_collections=weight_collections, + trainable=trainable, + name=model_name + ) retval = linear_model_layer(features) # pylint: disable=not-callable if cols_to_vars is not None: cols_to_vars.update(linear_model_layer.cols_to_vars()) @@ -942,16 +971,18 @@ class _FCLinearWrapper(base.Layer): See `linear_model` above. """ - def __init__(self, - feature_column, - units=1, - sparse_combiner='sum', - weight_collections=None, - trainable=True, - name=None, - **kwargs): - super(_FCLinearWrapper, self).__init__( - trainable=trainable, name=name, **kwargs) + def __init__( + self, + feature_column, + units=1, + sparse_combiner='sum', + weight_collections=None, + trainable=True, + name=None, + **kwargs + ): + super(_FCLinearWrapper, + self).__init__(trainable=trainable, name=name, **kwargs) self._feature_column = feature_column self._units = units self._sparse_combiner = sparse_combiner @@ -960,52 +991,58 @@ def __init__(self, def build(self, _): if isinstance(self._feature_column, _CategoricalColumn): weight = self.add_variable( - name='weights', - shape=(self._feature_column._num_buckets, self._units), # pylint: disable=protected-access - initializer=init_ops.zeros_initializer(), - trainable=self.trainable) + name='weights', + shape=(self._feature_column._num_buckets, self._units), # pylint: disable=protected-access + initializer=init_ops.zeros_initializer(), + trainable=self.trainable + ) else: num_elements = self._feature_column._variable_shape.num_elements() # pylint: disable=protected-access weight = self.add_variable( - name='weights', - shape=[num_elements, self._units], - initializer=init_ops.zeros_initializer(), - trainable=self.trainable) + name='weights', + shape=[num_elements, self._units], + initializer=init_ops.zeros_initializer(), + trainable=self.trainable + ) _add_to_collections(weight, self._weight_collections) self._weight_var = weight self.built = True def call(self, builder): weighted_sum = _create_weighted_sum( - column=self._feature_column, - builder=builder, - units=self._units, - sparse_combiner=self._sparse_combiner, - weight_collections=self._weight_collections, - trainable=self.trainable, - weight_var=self._weight_var) + column=self._feature_column, + builder=builder, + units=self._units, + sparse_combiner=self._sparse_combiner, + weight_collections=self._weight_collections, + trainable=self.trainable, + weight_var=self._weight_var + ) return weighted_sum class _BiasLayer(base.Layer): """A layer for the bias term.""" - def __init__(self, - units=1, - trainable=True, - weight_collections=None, - name=None, - **kwargs): + def __init__( + self, + units=1, + trainable=True, + weight_collections=None, + name=None, + **kwargs + ): super(_BiasLayer, self).__init__(trainable=trainable, name=name, **kwargs) self._units = units self._weight_collections = weight_collections def build(self, _): self._bias_variable = self.add_variable( - 'bias_weights', - shape=[self._units], - initializer=init_ops.zeros_initializer(), - trainable=self.trainable) + 'bias_weights', + shape=[self._units], + initializer=init_ops.zeros_initializer(), + trainable=self.trainable + ) _add_to_collections(self._bias_variable, self._weight_collections) self.built = True @@ -1014,8 +1051,10 @@ def call(self, _): def _get_expanded_variable_list(variable): - if (isinstance(variable, variables.Variable) or - resource_variable_ops.is_resource_variable(variable)): + if ( + isinstance(variable, variables.Variable) + or resource_variable_ops.is_resource_variable(variable) + ): return [variable] # Single variable case. else: # Must be a PartitionedVariable, so convert into a list. return list(variable) @@ -1031,14 +1070,16 @@ class _LinearModel(training.Model): See `linear_model` for details. """ - def __init__(self, - feature_columns, - units=1, - sparse_combiner='sum', - weight_collections=None, - trainable=True, - name=None, - **kwargs): + def __init__( + self, + feature_columns, + units=1, + sparse_combiner='sum', + weight_collections=None, + trainable=True, + name=None, + **kwargs + ): super(_LinearModel, self).__init__(name=name, **kwargs) self._feature_columns = _normalize_feature_columns(feature_columns) self._weight_collections = list(weight_collections or []) @@ -1050,22 +1091,25 @@ def __init__(self, column_layers = {} for column in sorted(self._feature_columns, key=lambda x: x.name): with variable_scope.variable_scope( - None, default_name=column._var_scope_name) as vs: # pylint: disable=protected-access + None, default_name=column._var_scope_name + ) as vs: # pylint: disable=protected-access # Having the fully expressed variable scope name ends up doubly # expressing the outer scope (scope with which this method was called) # in the name of the variable that would get created. column_name = _strip_leading_slashes(vs.name) - column_layer = _FCLinearWrapper(column, units, sparse_combiner, - self._weight_collections, trainable, - column_name, **kwargs) + column_layer = _FCLinearWrapper( + column, units, sparse_combiner, self._weight_collections, trainable, + column_name, **kwargs + ) column_layers[column_name] = column_layer self._column_layers = self._add_layers(column_layers) self._bias_layer = _BiasLayer( - units=units, - trainable=trainable, - weight_collections=self._weight_collections, - name='bias_layer', - **kwargs) + units=units, + trainable=trainable, + weight_collections=self._weight_collections, + name='bias_layer', + **kwargs + ) self._cols_to_vars = {} def cols_to_vars(self): @@ -1081,8 +1125,9 @@ def call(self, features): for column in self._feature_columns: if not isinstance(column, (_DenseColumn, _CategoricalColumn)): raise ValueError( - 'Items of feature_columns must be either a ' - '_DenseColumn or _CategoricalColumn. Given: {}'.format(column)) + 'Items of feature_columns must be either a ' + '_DenseColumn or _CategoricalColumn. Given: {}'.format(column) + ) weighted_sums = [] ordered_columns = [] builder = _LazyBuilder(features) @@ -1092,11 +1137,13 @@ def call(self, features): weighted_sum = layer(builder) weighted_sums.append(weighted_sum) self._cols_to_vars[column] = ops.get_collection( - ops.GraphKeys.GLOBAL_VARIABLES, scope=layer.scope_name) + ops.GraphKeys.GLOBAL_VARIABLES, scope=layer.scope_name + ) _verify_static_batch_size_equality(weighted_sums, ordered_columns) predictions_no_bias = math_ops.add_n( - weighted_sums, name='weighted_sum_no_bias') + weighted_sums, name='weighted_sum_no_bias' + ) predictions = nn_ops.bias_add( predictions_no_bias, self._bias_layer( # pylint: disable=not-callable @@ -1152,7 +1199,8 @@ def _transform_features(features, feature_columns): feature_columns = _normalize_feature_columns(feature_columns) outputs = {} with ops.name_scope( - None, default_name='transform_features', values=features.values()): + None, default_name='transform_features', values=features.values() + ): builder = _LazyBuilder(features) for column in sorted(feature_columns, key=lambda x: x.name): with ops.name_scope(None, default_name=column.name): @@ -1208,25 +1256,31 @@ def make_parse_example_spec(feature_columns): result = {} for column in feature_columns: if not isinstance(column, _FeatureColumn): - raise ValueError('All feature_columns must be _FeatureColumn instances. ' - 'Given: {}'.format(column)) + raise ValueError( + 'All feature_columns must be _FeatureColumn instances. ' + 'Given: {}'.format(column) + ) config = column._parse_example_spec # pylint: disable=protected-access for key, value in six.iteritems(config): if key in result and value != result[key]: - raise ValueError('feature_columns contain different parse_spec for key ' - '{}. Given {} and {}'.format(key, value, result[key])) + raise ValueError( + 'feature_columns contain different parse_spec for key ' + '{}. Given {} and {}'.format(key, value, result[key]) + ) result.update(config) return result -def _embedding_column(categorical_column, - dimension, - combiner='mean', - initializer=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True): +def _embedding_column( + categorical_column, + dimension, + combiner='mean', + initializer=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True +): """`_DenseColumn` that converts from sparse, categorical input. Use this when your inputs are sparse, but you want to convert them to a dense @@ -1301,44 +1355,52 @@ def model_fn(features, ...): if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError('Must specify both `ckpt_to_load_from` and ' - '`tensor_name_in_ckpt` or none of them.') + raise ValueError( + 'Must specify both `ckpt_to_load_from` and ' + '`tensor_name_in_ckpt` or none of them.' + ) if (initializer is not None) and (not callable(initializer)): - raise ValueError('initializer must be callable if specified. ' - 'Embedding of column_name: {}'.format( - categorical_column.name)) + raise ValueError( + 'initializer must be callable if specified. ' + 'Embedding of column_name: {}'.format(categorical_column.name) + ) if initializer is None: initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=0.01 / math.sqrt(dimension)) + mean=0.0, stddev=0.01 / math.sqrt(dimension) + ) embedding_shape = categorical_column._num_buckets, dimension # pylint: disable=protected-access def _creator(weight_collections, scope): embedding_column_layer = _EmbeddingColumnLayer( - embedding_shape=embedding_shape, - initializer=initializer, - weight_collections=weight_collections, - trainable=trainable, - name='embedding_column_layer') + embedding_shape=embedding_shape, + initializer=initializer, + weight_collections=weight_collections, + trainable=trainable, + name='embedding_column_layer' + ) return embedding_column_layer(None, scope=scope) # pylint: disable=not-callable return _EmbeddingColumn( - categorical_column=categorical_column, - dimension=dimension, - combiner=combiner, - layer_creator=_creator, - ckpt_to_load_from=ckpt_to_load_from, - tensor_name_in_ckpt=tensor_name_in_ckpt, - max_norm=max_norm, - trainable=trainable) - - -def _numeric_column(key, - shape=(1,), - default_value=None, - dtype=dtypes.float32, - normalizer_fn=None): + categorical_column=categorical_column, + dimension=dimension, + combiner=combiner, + layer_creator=_creator, + ckpt_to_load_from=ckpt_to_load_from, + tensor_name_in_ckpt=tensor_name_in_ckpt, + max_norm=max_norm, + trainable=trainable + ) + + +def _numeric_column( + key, + shape=(1, ), + default_value=None, + dtype=dtypes.float32, + normalizer_fn=None +): """Represents real valued or numerical features. Example: @@ -1392,21 +1454,27 @@ def _numeric_column(key, """ shape = _check_shape(shape, key) if not (dtype.is_integer or dtype.is_floating): - raise ValueError('dtype must be convertible to float. ' - 'dtype: {}, key: {}'.format(dtype, key)) - default_value = fc_utils.check_default_value(shape, default_value, dtype, key) + raise ValueError( + 'dtype must be convertible to float. ' + 'dtype: {}, key: {}'.format(dtype, key) + ) + default_value = fc_utils.check_default_value( + shape, default_value, dtype, key + ) if normalizer_fn is not None and not callable(normalizer_fn): raise TypeError( - 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) + 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn) + ) fc_utils.assert_key_is_string(key) return _NumericColumn( - key, - shape=shape, - default_value=default_value, - dtype=dtype, - normalizer_fn=normalizer_fn) + key, + shape=shape, + default_value=default_value, + dtype=dtype, + normalizer_fn=normalizer_fn + ) def _bucketized_column(source_column, boundaries): @@ -1477,13 +1545,18 @@ def _bucketized_column(source_column, boundaries): """ if not isinstance(source_column, _NumericColumn): raise ValueError( - 'source_column must be a column generated with numeric_column(). ' - 'Given: {}'.format(source_column)) + 'source_column must be a column generated with numeric_column(). ' + 'Given: {}'.format(source_column) + ) if len(source_column.shape) > 1: - raise ValueError('source_column must be one-dimensional column. ' - 'Given: {}'.format(source_column)) - if (not boundaries or - not (isinstance(boundaries, list) or isinstance(boundaries, tuple))): + raise ValueError( + 'source_column must be one-dimensional column. ' + 'Given: {}'.format(source_column) + ) + if ( + not boundaries + or not (isinstance(boundaries, list) or isinstance(boundaries, tuple)) + ): raise ValueError('boundaries must be a sorted list.') for i in range(len(boundaries) - 1): if boundaries[i] >= boundaries[i + 1]: @@ -1491,9 +1564,9 @@ def _bucketized_column(source_column, boundaries): return _BucketizedColumn(source_column, tuple(boundaries)) -def _categorical_column_with_hash_bucket(key, - hash_bucket_size, - dtype=dtypes.string): +def _categorical_column_with_hash_bucket( + key, hash_bucket_size, dtype=dtypes.string +): """Represents sparse feature where ids are set by hashing. Use this when your sparse features are in string or integer format, and you @@ -1536,12 +1609,14 @@ def _categorical_column_with_hash_bucket(key, ValueError: `dtype` is neither string nor integer. """ if hash_bucket_size is None: - raise ValueError('hash_bucket_size must be set. ' 'key: {}'.format(key)) + raise ValueError('hash_bucket_size must be set. ' + 'key: {}'.format(key)) if hash_bucket_size < 1: - raise ValueError('hash_bucket_size must be at least 1. ' - 'hash_bucket_size: {}, key: {}'.format( - hash_bucket_size, key)) + raise ValueError( + 'hash_bucket_size must be at least 1. ' + 'hash_bucket_size: {}, key: {}'.format(hash_bucket_size, key) + ) fc_utils.assert_key_is_string(key) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) @@ -1549,12 +1624,14 @@ def _categorical_column_with_hash_bucket(key, return _HashedCategoricalColumn(key, hash_bucket_size, dtype) -def _categorical_column_with_vocabulary_file(key, - vocabulary_file, - vocabulary_size=None, - num_oov_buckets=0, - default_value=None, - dtype=dtypes.string): +def _categorical_column_with_vocabulary_file( + key, + vocabulary_file, + vocabulary_size=None, + num_oov_buckets=0, + default_value=None, + dtype=dtypes.string +): """A `_CategoricalColumn` with a vocabulary file. Use this when your inputs are in string or integer format, and you have a @@ -1643,8 +1720,9 @@ def _categorical_column_with_vocabulary_file(key, with gfile.GFile(vocabulary_file) as f: vocabulary_size = sum(1 for _ in f) logging.info( - 'vocabulary_size = %d in %s is inferred from the number of elements ' - 'in the vocabulary_file %s.', vocabulary_size, key, vocabulary_file) + 'vocabulary_size = %d in %s is inferred from the number of elements ' + 'in the vocabulary_file %s.', vocabulary_size, key, vocabulary_file + ) # `vocabulary_size` isn't required for lookup, but it is for `_num_buckets`. if vocabulary_size < 1: @@ -1652,27 +1730,28 @@ def _categorical_column_with_vocabulary_file(key, if num_oov_buckets: if default_value is not None: raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'.format( - key)) + 'Can\'t specify both num_oov_buckets and default_value in {}.'. + format(key) + ) if num_oov_buckets < 0: - raise ValueError('Invalid num_oov_buckets {} in {}.'.format( - num_oov_buckets, key)) + raise ValueError( + 'Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key) + ) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) return _VocabularyFileCategoricalColumn( - key=key, - vocabulary_file=vocabulary_file, - vocabulary_size=vocabulary_size, - num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, - default_value=-1 if default_value is None else default_value, - dtype=dtype) - - -def _categorical_column_with_vocabulary_list(key, - vocabulary_list, - dtype=None, - default_value=-1, - num_oov_buckets=0): + key=key, + vocabulary_file=vocabulary_file, + vocabulary_size=vocabulary_size, + num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, + default_value=-1 if default_value is None else default_value, + dtype=dtype + ) + + +def _categorical_column_with_vocabulary_list( + key, vocabulary_list, dtype=None, default_value=-1, num_oov_buckets=0 +): """A `_CategoricalColumn` with in-memory vocabulary. Use this when your inputs are in string or integer format, and you have an @@ -1750,38 +1829,48 @@ def _categorical_column_with_vocabulary_list(key, """ if (vocabulary_list is None) or (len(vocabulary_list) < 1): raise ValueError( - 'vocabulary_list {} must be non-empty, column_name: {}'.format( - vocabulary_list, key)) + 'vocabulary_list {} must be non-empty, column_name: {}'.format( + vocabulary_list, key + ) + ) if len(set(vocabulary_list)) != len(vocabulary_list): raise ValueError( - 'Duplicate keys in vocabulary_list {}, column_name: {}'.format( - vocabulary_list, key)) + 'Duplicate keys in vocabulary_list {}, column_name: {}'.format( + vocabulary_list, key + ) + ) vocabulary_dtype = dtypes.as_dtype(np.array(vocabulary_list).dtype) if num_oov_buckets: if default_value != -1: raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'.format( - key)) + 'Can\'t specify both num_oov_buckets and default_value in {}.'. + format(key) + ) if num_oov_buckets < 0: - raise ValueError('Invalid num_oov_buckets {} in {}.'.format( - num_oov_buckets, key)) + raise ValueError( + 'Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key) + ) fc_utils.assert_string_or_int( - vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key)) + vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key) + ) if dtype is None: dtype = vocabulary_dtype elif dtype.is_integer != vocabulary_dtype.is_integer: raise ValueError( - 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format( - dtype, vocabulary_dtype, key)) + 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format( + dtype, vocabulary_dtype, key + ) + ) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) return _VocabularyListCategoricalColumn( - key=key, - vocabulary_list=tuple(vocabulary_list), - dtype=dtype, - default_value=default_value, - num_oov_buckets=num_oov_buckets) + key=key, + vocabulary_list=tuple(vocabulary_list), + dtype=dtype, + default_value=default_value, + num_oov_buckets=num_oov_buckets + ) def _categorical_column_with_identity(key, num_buckets, default_value=None): @@ -1839,16 +1928,20 @@ def _categorical_column_with_identity(key, num_buckets, default_value=None): ValueError: if `default_value` is not in range `[0, num_buckets)`. """ if num_buckets < 1: - raise ValueError('num_buckets {} < 1, column_name {}'.format( - num_buckets, key)) - if (default_value is not None) and ((default_value < 0) or - (default_value >= num_buckets)): raise ValueError( - 'default_value {} not in range [0, {}), column_name {}'.format( - default_value, num_buckets, key)) + 'num_buckets {} < 1, column_name {}'.format(num_buckets, key) + ) + if (default_value is not None + ) and ((default_value < 0) or (default_value >= num_buckets)): + raise ValueError( + 'default_value {} not in range [0, {}), column_name {}'.format( + default_value, num_buckets, key + ) + ) fc_utils.assert_key_is_string(key) return _IdentityCategoricalColumn( - key=key, num_buckets=num_buckets, default_value=default_value) + key=key, num_buckets=num_buckets, default_value=default_value + ) def _indicator_column(categorical_column): @@ -1885,9 +1978,9 @@ def _indicator_column(categorical_column): return _IndicatorColumn(categorical_column) -def _weighted_categorical_column(categorical_column, - weight_feature_key, - dtype=dtypes.float32): +def _weighted_categorical_column( + categorical_column, weight_feature_key, dtype=dtypes.float32 +): """Applies weight values to a `_CategoricalColumn`. Use this when each of your sparse inputs has both an ID and a value. For @@ -1955,9 +2048,10 @@ def _weighted_categorical_column(categorical_column, if (dtype is None) or not (dtype.is_integer or dtype.is_floating): raise ValueError('dtype {} is not convertible to float.'.format(dtype)) return _WeightedCategoricalColumn( - categorical_column=categorical_column, - weight_feature_key=weight_feature_key, - dtype=dtype) + categorical_column=categorical_column, + weight_feature_key=weight_feature_key, + dtype=dtype + ) def _crossed_column(keys, hash_bucket_size, hash_key=None): @@ -2065,38 +2159,48 @@ def _crossed_column(keys, hash_bucket_size, hash_key=None): ValueError: If `hash_bucket_size < 1`. """ if not hash_bucket_size or hash_bucket_size < 1: - raise ValueError('hash_bucket_size must be > 1. ' - 'hash_bucket_size: {}'.format(hash_bucket_size)) + raise ValueError( + 'hash_bucket_size must be > 1. ' + 'hash_bucket_size: {}'.format(hash_bucket_size) + ) if not keys or len(keys) < 2: raise ValueError( - 'keys must be a list with length > 1. Given: {}'.format(keys)) + 'keys must be a list with length > 1. Given: {}'.format(keys) + ) for key in keys: - if (not isinstance(key, six.string_types) and - not isinstance(key, _CategoricalColumn)): + if ( + not isinstance(key, six.string_types) + and not isinstance(key, _CategoricalColumn) + ): raise ValueError( - 'Unsupported key type. All keys must be either string, or ' - 'categorical column except _HashedCategoricalColumn. ' - 'Given: {}'.format(key)) + 'Unsupported key type. All keys must be either string, or ' + 'categorical column except _HashedCategoricalColumn. ' + 'Given: {}'.format(key) + ) if isinstance(key, _HashedCategoricalColumn): raise ValueError( - 'categorical_column_with_hash_bucket is not supported for crossing. ' - 'Hashing before crossing will increase probability of collision. ' - 'Instead, use the feature name as a string. Given: {}'.format(key)) + 'categorical_column_with_hash_bucket is not supported for crossing. ' + 'Hashing before crossing will increase probability of collision. ' + 'Instead, use the feature name as a string. Given: {}'.format(key) + ) return _CrossedColumn( - keys=tuple(keys), hash_bucket_size=hash_bucket_size, hash_key=hash_key) + keys=tuple(keys), hash_bucket_size=hash_bucket_size, hash_key=hash_key + ) # TODO(rohanj): Clearly define semantics of this layer. class _EmbeddingColumnLayer(base.Layer): """A layer that stores all the state required for a embedding column.""" - def __init__(self, - embedding_shape, - initializer, - weight_collections=None, - trainable=True, - name=None, - **kwargs): + def __init__( + self, + embedding_shape, + initializer, + weight_collections=None, + trainable=True, + name=None, + **kwargs + ): """Constructor. Args: @@ -2111,8 +2215,8 @@ def __init__(self, name: Name of the layer **kwargs: keyword named properties. """ - super(_EmbeddingColumnLayer, self).__init__( - trainable=trainable, name=name, **kwargs) + super(_EmbeddingColumnLayer, + self).__init__(trainable=trainable, name=name, **kwargs) self._embedding_shape = embedding_shape self._initializer = initializer self._weight_collections = weight_collections @@ -2128,11 +2232,12 @@ def set_weight_collections(self, weight_collections): def build(self, _): self._embedding_weight_var = self.add_variable( - name='embedding_weights', - shape=self._embedding_shape, - dtype=dtypes.float32, - initializer=self._initializer, - trainable=self.trainable) + name='embedding_weights', + shape=self._embedding_shape, + dtype=dtypes.float32, + initializer=self._initializer, + trainable=self.trainable + ) if self._weight_collections and not context.executing_eagerly(): _add_to_collections(self._embedding_weight_var, self._weight_collections) self.built = True @@ -2268,39 +2373,40 @@ def input_layer(features, feature_columns, ...): pass -def _create_weighted_sum(column, - builder, - units, - sparse_combiner, - weight_collections, - trainable, - weight_var=None): +def _create_weighted_sum( + column, + builder, + units, + sparse_combiner, + weight_collections, + trainable, + weight_var=None +): """Creates a weighted sum for a dense/categorical column for linear_model.""" if isinstance(column, _CategoricalColumn): return _create_categorical_column_weighted_sum( - column=column, - builder=builder, - units=units, - sparse_combiner=sparse_combiner, - weight_collections=weight_collections, - trainable=trainable, - weight_var=weight_var) + column=column, + builder=builder, + units=units, + sparse_combiner=sparse_combiner, + weight_collections=weight_collections, + trainable=trainable, + weight_var=weight_var + ) else: return _create_dense_column_weighted_sum( - column=column, - builder=builder, - units=units, - weight_collections=weight_collections, - trainable=trainable, - weight_var=weight_var) + column=column, + builder=builder, + units=units, + weight_collections=weight_collections, + trainable=trainable, + weight_var=weight_var + ) -def _create_dense_column_weighted_sum(column, - builder, - units, - weight_collections, - trainable, - weight_var=None): +def _create_dense_column_weighted_sum( + column, builder, units, weight_collections, trainable, weight_var=None +): """Create a weighted sum of a dense column for linear_model.""" tensor = column._get_dense_tensor( # pylint: disable=protected-access builder, @@ -2313,11 +2419,12 @@ def _create_dense_column_weighted_sum(column, weight = weight_var else: weight = variable_scope.get_variable( - name='weights', - shape=[num_elements, units], - initializer=init_ops.zeros_initializer(), - trainable=trainable, - collections=weight_collections) + name='weights', + shape=[num_elements, units], + initializer=init_ops.zeros_initializer(), + trainable=trainable, + collections=weight_collections + ) return math_ops.matmul(tensor, weight, name='weighted_sum') @@ -2339,10 +2446,9 @@ def _num_buckets(self): pass @abc.abstractmethod - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): """Returns an IdWeightPair. `IdWeightPair` is a pair of `SparseTensor`s which represents ids and @@ -2366,13 +2472,15 @@ def _get_sparse_tensors(self, pass -def _create_categorical_column_weighted_sum(column, - builder, - units, - sparse_combiner, - weight_collections, - trainable, - weight_var=None): +def _create_categorical_column_weighted_sum( + column, + builder, + units, + sparse_combiner, + weight_collections, + trainable, + weight_var=None +): # pylint: disable=g-doc-return-or-yield,g-doc-args """Create a weighted sum of a categorical column for linear_model. @@ -2405,28 +2513,32 @@ def _create_categorical_column_weighted_sum(column, weight_collections=weight_collections, trainable=trainable) id_tensor = sparse_ops.sparse_reshape( - sparse_tensors.id_tensor, - [array_ops.shape(sparse_tensors.id_tensor)[0], -1]) + sparse_tensors.id_tensor, + [array_ops.shape(sparse_tensors.id_tensor)[0], -1] + ) weight_tensor = sparse_tensors.weight_tensor if weight_tensor is not None: weight_tensor = sparse_ops.sparse_reshape( - weight_tensor, [array_ops.shape(weight_tensor)[0], -1]) + weight_tensor, [array_ops.shape(weight_tensor)[0], -1] + ) if weight_var is not None: weight = weight_var else: weight = variable_scope.get_variable( - name='weights', - shape=(column._num_buckets, units), # pylint: disable=protected-access - initializer=init_ops.zeros_initializer(), - trainable=trainable, - collections=weight_collections) + name='weights', + shape=(column._num_buckets, units), # pylint: disable=protected-access + initializer=init_ops.zeros_initializer(), + trainable=trainable, + collections=weight_collections + ) return embedding_ops.safe_embedding_lookup_sparse( - weight, - id_tensor, - sparse_weights=weight_tensor, - combiner=sparse_combiner, - name='weighted_sum') + weight, + id_tensor, + sparse_weights=weight_tensor, + combiner=sparse_combiner, + name='weighted_sum' + ) class _SequenceDenseColumn(_FeatureColumn): @@ -2436,10 +2548,9 @@ class _SequenceDenseColumn(_FeatureColumn): 'TensorSequenceLengthPair', ['dense_tensor', 'sequence_length']) @abc.abstractmethod - def _get_sequence_dense_tensor(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sequence_dense_tensor( + self, inputs, weight_collections=None, trainable=None + ): """Returns a `TensorSequenceLengthPair`.""" pass @@ -2518,8 +2629,10 @@ def get(self, key): raise ValueError('Feature {} is not in features dictionary.'.format(key)) if not isinstance(key, _FeatureColumn): - raise TypeError('"key" must be either a "str" or "_FeatureColumn". ' - 'Provided: {}'.format(key)) + raise TypeError( + '"key" must be either a "str" or "_FeatureColumn". ' + 'Provided: {}'.format(key) + ) column = key logging.debug('Transforming feature_column %s.', column) @@ -2552,13 +2665,15 @@ def _get_raw_feature_as_tensor(self, key): return raw_feature feature_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - raw_feature) + raw_feature + ) def expand_dims(input_tensor): # Input_tensor must have rank 1. if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): - return sparse_ops.sparse_reshape(input_tensor, - [array_ops.shape(input_tensor)[0], 1]) + return sparse_ops.sparse_reshape( + input_tensor, [array_ops.shape(input_tensor)[0], 1] + ) else: return array_ops.expand_dims(input_tensor, -1) @@ -2566,20 +2681,27 @@ def expand_dims(input_tensor): if rank is not None: if rank == 0: raise ValueError( - 'Feature (key: {}) cannot have rank 0. Give: {}'.format( - key, feature_tensor)) + 'Feature (key: {}) cannot have rank 0. Give: {}'.format( + key, feature_tensor + ) + ) return feature_tensor if rank != 1 else expand_dims(feature_tensor) # Handle dynamic rank. - with ops.control_dependencies([ + with ops.control_dependencies( + [ check_ops.assert_positive( - array_ops.rank(feature_tensor), - message='Feature (key: {}) cannot have rank 0. Given: {}'.format( - key, feature_tensor)) - ]): + array_ops.rank(feature_tensor), + message='Feature (key: {}) cannot have rank 0. Given: {}'.format( + key, feature_tensor + ) + ) + ] + ): return control_flow_ops.cond( - math_ops.equal(1, array_ops.rank(feature_tensor)), - lambda: expand_dims(feature_tensor), lambda: feature_tensor) + math_ops.equal(1, array_ops.rank(feature_tensor)), + lambda: expand_dims(feature_tensor), lambda: feature_tensor + ) # TODO(ptucker): Move to third_party/tensorflow/python/ops/sparse_ops.py @@ -2614,13 +2736,16 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): ValueError: when `input_tensor`'s rank is `None`. """ input_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - input_tensor) + input_tensor + ) if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): return input_tensor - with ops.name_scope(None, 'to_sparse_input', ( + with ops.name_scope( + None, 'to_sparse_input', ( input_tensor, ignore_value, - )): + ) + ): if ignore_value is None: if input_tensor.dtype == dtypes.string: # Exception due to TF strings are converted to numpy objects by default. @@ -2633,14 +2758,18 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): # default value for that type. ignore_value = input_tensor.dtype.as_numpy_dtype() ignore_value = math_ops.cast( - ignore_value, input_tensor.dtype, name='ignore_value') + ignore_value, input_tensor.dtype, name='ignore_value' + ) indices = array_ops.where( - math_ops.not_equal(input_tensor, ignore_value), name='indices') + math_ops.not_equal(input_tensor, ignore_value), name='indices' + ) return sparse_tensor_lib.SparseTensor( - indices=indices, - values=array_ops.gather_nd(input_tensor, indices, name='values'), - dense_shape=array_ops.shape( - input_tensor, out_type=dtypes.int64, name='dense_shape')) + indices=indices, + values=array_ops.gather_nd(input_tensor, indices, name='values'), + dense_shape=array_ops.shape( + input_tensor, out_type=dtypes.int64, name='dense_shape' + ) + ) def _normalize_feature_columns(feature_columns): @@ -2670,29 +2799,34 @@ def _normalize_feature_columns(feature_columns): for column in feature_columns: if not isinstance(column, _FeatureColumn): - raise ValueError('Items of feature_columns must be a _FeatureColumn. ' - 'Given (type {}): {}.'.format(type(column), column)) + raise ValueError( + 'Items of feature_columns must be a _FeatureColumn. ' + 'Given (type {}): {}.'.format(type(column), column) + ) if not feature_columns: raise ValueError('feature_columns must not be empty.') name_to_column = {} for column in feature_columns: if column.name in name_to_column: - raise ValueError('Duplicate feature column name found for columns: {} ' - 'and {}. This usually means that these columns refer to ' - 'same base feature. Either one must be discarded or a ' - 'duplicated but renamed item must be inserted in ' - 'features dict.'.format(column, - name_to_column[column.name])) + raise ValueError( + 'Duplicate feature column name found for columns: {} ' + 'and {}. This usually means that these columns refer to ' + 'same base feature. Either one must be discarded or a ' + 'duplicated but renamed item must be inserted in ' + 'features dict.'.format(column, name_to_column[column.name]) + ) name_to_column[column.name] = column return feature_columns class _NumericColumn( - _DenseColumn, - collections.namedtuple( - '_NumericColumn', - ['key', 'shape', 'default_value', 'dtype', 'normalizer_fn'])): + _DenseColumn, + collections.namedtuple( + '_NumericColumn', + ['key', 'shape', 'default_value', 'dtype', 'normalizer_fn'] + ) +): """See `numeric_column`.""" @property @@ -2702,17 +2836,17 @@ def name(self): @property def _parse_example_spec(self): return { - self.key: - parsing_ops.FixedLenFeature(self.shape, self.dtype, - self.default_value) + self.key: + parsing_ops.FixedLenFeature(self.shape, self.dtype, self.default_value) } def _transform_feature(self, inputs): input_tensor = inputs.get(self.key) if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): raise ValueError( - 'The corresponding Tensor of numerical column must be a Tensor. ' - 'SparseTensor is not supported. key: {}'.format(self.key)) + 'The corresponding Tensor of numerical column must be a Tensor. ' + 'SparseTensor is not supported. key: {}'.format(self.key) + ) if self.normalizer_fn is not None: input_tensor = self.normalizer_fn(input_tensor) return math_ops.cast(input_tensor, dtypes.float32) @@ -2743,10 +2877,10 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): return inputs.get(self) -class _BucketizedColumn(_DenseColumn, _CategoricalColumn, - collections.namedtuple('_BucketizedColumn', - ['source_column', 'boundaries']) - ): +class _BucketizedColumn( + _DenseColumn, _CategoricalColumn, + collections.namedtuple('_BucketizedColumn', ['source_column', 'boundaries']) +): """See `bucketized_column`.""" @property @@ -2770,27 +2904,28 @@ def _transform_feature(self, inputs): @property def _variable_shape(self): return tensor_shape.TensorShape( - tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) + tuple(self.source_column.shape) + (len(self.boundaries) + 1, ) + ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable input_tensor = inputs.get(self) return array_ops.one_hot( - indices=math_ops.cast(input_tensor, dtypes.int64), - depth=len(self.boundaries) + 1, - on_value=1., - off_value=0.) + indices=math_ops.cast(input_tensor, dtypes.int64), + depth=len(self.boundaries) + 1, + on_value=1., + off_value=0. + ) @property def _num_buckets(self): # By construction, source_column is always one-dimensional. return (len(self.boundaries) + 1) * self.source_column.shape[0] - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): """Converts dense inputs to SparseTensor so downstream code can use it.""" input_tensor = inputs.get(self) batch_size = array_ops.shape(input_tensor)[0] @@ -2798,31 +2933,39 @@ def _get_sparse_tensors(self, source_dimension = self.source_column.shape[0] i1 = array_ops.reshape( - array_ops.tile( - array_ops.expand_dims(math_ops.range(0, batch_size), 1), - [1, source_dimension]), (-1,)) + array_ops.tile( + array_ops.expand_dims(math_ops.range(0, batch_size), 1), + [1, source_dimension] + ), (-1, ) + ) i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size]) # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets bucket_indices = ( - array_ops.reshape(input_tensor, - (-1,)) + (len(self.boundaries) + 1) * i2) + array_ops.reshape(input_tensor, (-1, )) + (len(self.boundaries) + 1) * i2 + ) indices = math_ops.cast( - array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64) + array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64 + ) dense_shape = math_ops.cast( - array_ops.stack([batch_size, source_dimension]), dtypes.int64) + array_ops.stack([batch_size, source_dimension]), dtypes.int64 + ) sparse_tensor = sparse_tensor_lib.SparseTensor( - indices=indices, values=bucket_indices, dense_shape=dense_shape) + indices=indices, values=bucket_indices, dense_shape=dense_shape + ) return _CategoricalColumn.IdWeightPair(sparse_tensor, None) class _EmbeddingColumn( - _DenseColumn, _SequenceDenseColumn, - collections.namedtuple( - '_EmbeddingColumn', - ('categorical_column', 'dimension', 'combiner', 'layer_creator', - 'ckpt_to_load_from', 'tensor_name_in_ckpt', 'max_norm', 'trainable'))): + _DenseColumn, _SequenceDenseColumn, + collections.namedtuple( + '_EmbeddingColumn', ( + 'categorical_column', 'dimension', 'combiner', 'layer_creator', + 'ckpt_to_load_from', 'tensor_name_in_ckpt', 'max_norm', 'trainable' + ) + ) +): """See `embedding_column`.""" @property @@ -2844,10 +2987,9 @@ def _variable_shape(self): self._shape = tensor_shape.TensorShape([self.dimension]) return self._shape - def _get_dense_tensor_internal(self, - inputs, - weight_collections=None, - trainable=None): + def _get_dense_tensor_internal( + self, inputs, weight_collections=None, trainable=None + ): """Private method that follows the signature of _get_dense_tensor.""" # Get sparse IDs and weights. sparse_tensors = self.categorical_column._get_sparse_tensors( # pylint: disable=protected-access @@ -2858,53 +3000,60 @@ def _get_dense_tensor_internal(self, sparse_weights = sparse_tensors.weight_tensor embedding_weights = self.layer_creator( - weight_collections=weight_collections, - scope=variable_scope.get_variable_scope()) + weight_collections=weight_collections, + scope=variable_scope.get_variable_scope() + ) if self.ckpt_to_load_from is not None: to_restore = embedding_weights if isinstance(to_restore, variables.PartitionedVariable): to_restore = to_restore._get_variable_list() # pylint: disable=protected-access checkpoint_utils.init_from_checkpoint( - self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) + self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore} + ) # Return embedding lookup result. return embedding_ops.safe_embedding_lookup_sparse( - embedding_weights=embedding_weights, - sparse_ids=sparse_ids, - sparse_weights=sparse_weights, - combiner=self.combiner, - name='%s_weights' % self.name, - max_norm=self.max_norm) + embedding_weights=embedding_weights, + sparse_ids=sparse_ids, + sparse_weights=sparse_weights, + combiner=self.combiner, + name='%s_weights' % self.name, + max_norm=self.max_norm + ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): if isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must not be of type _SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use input_layer, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'sequence_input_layer instead of input_layer. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In embedding_column: {}. ' + 'categorical_column must not be of type _SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use input_layer, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'sequence_input_layer instead of input_layer. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) return self._get_dense_tensor_internal( - inputs=inputs, - weight_collections=weight_collections, - trainable=trainable) + inputs=inputs, + weight_collections=weight_collections, + trainable=trainable + ) - def _get_sequence_dense_tensor(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sequence_dense_tensor( + self, inputs, weight_collections=None, trainable=None + ): if not isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must be of type _SequenceCategoricalColumn ' - 'to use sequence_input_layer. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In embedding_column: {}. ' + 'categorical_column must be of type _SequenceCategoricalColumn ' + 'to use sequence_input_layer. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) dense_tensor = self._get_dense_tensor_internal( # pylint: disable=protected-access inputs=inputs, weight_collections=weight_collections, @@ -2912,9 +3061,11 @@ def _get_sequence_dense_tensor(self, sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor) + sparse_tensors.id_tensor + ) return _SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length) + dense_tensor=dense_tensor, sequence_length=sequence_length + ) def _get_graph_for_variable(var): @@ -2925,13 +3076,16 @@ def _get_graph_for_variable(var): class _SharedEmbeddingColumn( - _DenseColumn, _SequenceDenseColumn, - collections.namedtuple( - '_SharedEmbeddingColumn', - ('categorical_column', 'dimension', 'combiner', 'initializer', - 'shared_embedding_collection_name', 'ckpt_to_load_from', - 'tensor_name_in_ckpt', 'max_norm', 'trainable', 'partitioner', - 'ev_params'))): + _DenseColumn, _SequenceDenseColumn, + collections.namedtuple( + '_SharedEmbeddingColumn', ( + 'categorical_column', 'dimension', 'combiner', 'initializer', + 'shared_embedding_collection_name', 'ckpt_to_load_from', + 'tensor_name_in_ckpt', 'max_norm', 'trainable', 'partitioner', + 'ev_params' + ) + ) +): """See `embedding_column`.""" @property @@ -2961,10 +3115,9 @@ def _variable_shape(self): self._shape = tensor_shape.TensorShape([self.dimension]) return self._shape - def _get_dense_tensor_internal(self, - inputs, - weight_collections=None, - trainable=None): + def _get_dense_tensor_internal( + self, inputs, weight_collections=None, trainable=None + ): """Private method that follows the signature of _get_dense_tensor.""" # This method is called from a variable_scope with name _var_scope_name, # which is shared among all shared embeddings. Open a name_scope here, so @@ -2980,37 +3133,42 @@ def _get_dense_tensor_internal(self, embedding_shape = (self.categorical_column._num_buckets, self.dimension) # pylint: disable=protected-access shared_embedding_collection = ops.get_collection( - self.shared_embedding_collection_name) + self.shared_embedding_collection_name + ) if shared_embedding_collection: if len(shared_embedding_collection) > 1: raise ValueError( - 'Collection {} can only contain one variable. ' - 'Suggested fix A: Choose a unique name for this collection. ' - 'Suggested fix B: Do not add any variables to this collection. ' - 'The feature_column library already adds a variable under the ' - 'hood.'.format(shared_embedding_collection)) + 'Collection {} can only contain one variable. ' + 'Suggested fix A: Choose a unique name for this collection. ' + 'Suggested fix B: Do not add any variables to this collection. ' + 'The feature_column library already adds a variable under the ' + 'hood.'.format(shared_embedding_collection) + ) embedding_weights = shared_embedding_collection[0] if embedding_weights.get_shape( ) != embedding_shape and not self.ev_params is not None: # noqa : E714 raise ValueError( - 'Shared embedding collection {} contains variable {} of ' - 'unexpected shape {}. Expected shape is {}. ' - 'Suggested fix A: Choose a unique name for this collection. ' - 'Suggested fix B: Do not add any variables to this collection. ' - 'The feature_column library already adds a variable under the ' - 'hood.'.format(self.shared_embedding_collection_name, - embedding_weights.name, - embedding_weights.get_shape(), embedding_shape)) + 'Shared embedding collection {} contains variable {} of ' + 'unexpected shape {}. Expected shape is {}. ' + 'Suggested fix A: Choose a unique name for this collection. ' + 'Suggested fix B: Do not add any variables to this collection. ' + 'The feature_column library already adds a variable under the ' + 'hood.'.format( + self.shared_embedding_collection_name, embedding_weights.name, + embedding_weights.get_shape(), embedding_shape + ) + ) else: if self.ev_params is None: embedding_weights = variable_scope.get_variable( - name='embedding_weights', - shape=embedding_shape, - dtype=dtypes.float32, - initializer=self.initializer, - trainable=self.trainable and trainable, - partitioner=self.partitioner, - collections=weight_collections) + name='embedding_weights', + shape=embedding_shape, + dtype=dtypes.float32, + initializer=self.initializer, + trainable=self.trainable and trainable, + partitioner=self.partitioner, + collections=weight_collections + ) else: # at eval or inference time, it is necessary to set # the initializers to zeros, so that new key will @@ -3024,86 +3182,99 @@ def _get_dense_tensor_internal(self, if 'EmbeddingVariableConfig' in dir(variables): ev_option = variables.EmbeddingVariableOption() ev_option.filter_strategy = variables.CounterFilter( - filter_freq=self.ev_params.filter_freq) + filter_freq=self.ev_params.filter_freq + ) extra_args['ev_option'] = ev_option else: extra_args['filter_options'] = variables.CounterFilterOptions( - self.ev_params.filter_freq) + self.ev_params.filter_freq + ) embedding_weights = variable_scope.get_embedding_variable( - name='embedding_weights', - embedding_dim=self.dimension, - initializer=initializer, - trainable=self.trainable and trainable, - partitioner=self.partitioner, - collections=weight_collections, - steps_to_live=self.ev_params.steps_to_live, - **extra_args) - - ops.add_to_collection(self.shared_embedding_collection_name, - embedding_weights) + name='embedding_weights', + embedding_dim=self.dimension, + initializer=initializer, + trainable=self.trainable and trainable, + partitioner=self.partitioner, + collections=weight_collections, + steps_to_live=self.ev_params.steps_to_live, + **extra_args + ) + + ops.add_to_collection( + self.shared_embedding_collection_name, embedding_weights + ) if self.ckpt_to_load_from is not None: to_restore = embedding_weights if isinstance(to_restore, variables.PartitionedVariable): to_restore = to_restore._get_variable_list() # pylint: disable=protected-access checkpoint_utils.init_from_checkpoint( - self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) + self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore} + ) if 'RaggedTensor' in str(type(sparse_ids)): assert sparse_weights is None return embedding_lookup_ragged( - embedding_weights=embedding_weights, - ragged_ids=sparse_ids, - ragged_weights=sparse_weights, - combiner=self.combiner, - max_norm=self.max_norm, - name='%s_weights' % self.name) + embedding_weights=embedding_weights, + ragged_ids=sparse_ids, + ragged_weights=sparse_weights, + combiner=self.combiner, + max_norm=self.max_norm, + name='%s_weights' % self.name + ) # Return embedding lookup result. return embedding_ops.safe_embedding_lookup_sparse( - embedding_weights=embedding_weights, - sparse_ids=sparse_ids, - sparse_weights=sparse_weights, - combiner=self.combiner, - name='%s_weights' % self.name, - max_norm=self.max_norm) + embedding_weights=embedding_weights, + sparse_ids=sparse_ids, + sparse_weights=sparse_weights, + combiner=self.combiner, + name='%s_weights' % self.name, + max_norm=self.max_norm + ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): if isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must not be of type _SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use input_layer, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'sequence_input_layer instead of input_layer. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In embedding_column: {}. ' + 'categorical_column must not be of type _SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use input_layer, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'sequence_input_layer instead of input_layer. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) return self._get_dense_tensor_internal( - inputs=inputs, - weight_collections=weight_collections, - trainable=trainable) + inputs=inputs, + weight_collections=weight_collections, + trainable=trainable + ) - def _get_sequence_dense_tensor(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sequence_dense_tensor( + self, inputs, weight_collections=None, trainable=None + ): if not isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must be of type _SequenceCategoricalColumn ' - 'to use sequence_input_layer. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In embedding_column: {}. ' + 'categorical_column must be of type _SequenceCategoricalColumn ' + 'to use sequence_input_layer. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) dense_tensor = self._get_dense_tensor_internal( # pylint: disable=protected-access inputs=inputs, weight_collections=weight_collections, trainable=trainable) sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor) + sparse_tensors.id_tensor + ) return _SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length) + dense_tensor=dense_tensor, sequence_length=sequence_length + ) def _check_shape(shape, key): @@ -3114,18 +3285,24 @@ def _check_shape(shape, key): shape = tuple(shape) for dimension in shape: if not isinstance(dimension, six.integer_types): - raise TypeError('shape dimensions must be integer. ' - 'shape: {}, key: {}'.format(shape, key)) + raise TypeError( + 'shape dimensions must be integer. ' + 'shape: {}, key: {}'.format(shape, key) + ) if dimension < 1: - raise ValueError('shape dimensions must be greater than 0. ' - 'shape: {}, key: {}'.format(shape, key)) + raise ValueError( + 'shape dimensions must be greater than 0. ' + 'shape: {}, key: {}'.format(shape, key) + ) return shape -class _HashedCategoricalColumn(_CategoricalColumn, - collections.namedtuple( - '_HashedCategoricalColumn', - ['key', 'hash_bucket_size', 'dtype'])): +class _HashedCategoricalColumn( + _CategoricalColumn, + collections.namedtuple( + '_HashedCategoricalColumn', ['key', 'hash_bucket_size', 'dtype'] + ) +): """See `categorical_column_with_hash_bucket`.""" @property @@ -3141,19 +3318,24 @@ def _parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + inputs.get(self.key) + ) if not isinstance(input_tensor, sparse_tensor_lib.SparseTensor): raise ValueError('SparseColumn input must be a SparseTensor.') fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key)) + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key) + ) if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype)) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype + ) + ) if self.dtype == dtypes.string: sparse_values = input_tensor.values @@ -3161,28 +3343,32 @@ def _transform_feature(self, inputs): sparse_values = string_ops.as_string(input_tensor.values) sparse_id_values = string_ops.string_to_hash_bucket_fast( - sparse_values, self.hash_bucket_size, name='lookup') - return sparse_tensor_lib.SparseTensor(input_tensor.indices, - sparse_id_values, - input_tensor.dense_shape) + sparse_values, self.hash_bucket_size, name='lookup' + ) + return sparse_tensor_lib.SparseTensor( + input_tensor.indices, sparse_id_values, input_tensor.dense_shape + ) @property def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.hash_bucket_size - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) class _VocabularyFileCategoricalColumn( - _CategoricalColumn, - collections.namedtuple('_VocabularyFileCategoricalColumn', - ('key', 'vocabulary_file', 'vocabulary_size', - 'num_oov_buckets', 'dtype', 'default_value'))): + _CategoricalColumn, + collections.namedtuple( + '_VocabularyFileCategoricalColumn', ( + 'key', 'vocabulary_file', 'vocabulary_size', 'num_oov_buckets', 'dtype', + 'default_value' + ) + ) +): """See `categorical_column_with_vocabulary_file`.""" @property @@ -3194,17 +3380,22 @@ def _parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + inputs.get(self.key) + ) if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype)) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype + ) + ) fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key)) + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key) + ) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -3213,30 +3404,31 @@ def _transform_feature(self, inputs): input_tensor = math_ops.cast(input_tensor, dtypes.int64) return lookup_ops.index_table_from_file( - vocabulary_file=self.vocabulary_file, - num_oov_buckets=self.num_oov_buckets, - vocab_size=self.vocabulary_size, - default_value=self.default_value, - key_dtype=key_dtype, - name='{}_lookup'.format(self.key)).lookup(input_tensor) + vocabulary_file=self.vocabulary_file, + num_oov_buckets=self.num_oov_buckets, + vocab_size=self.vocabulary_size, + default_value=self.default_value, + key_dtype=key_dtype, + name='{}_lookup'.format(self.key) + ).lookup(input_tensor) @property def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.vocabulary_size + self.num_oov_buckets - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) class _VocabularyListCategoricalColumn( - _CategoricalColumn, - collections.namedtuple( - '_VocabularyListCategoricalColumn', - ('key', 'vocabulary_list', 'dtype', 'default_value', 'num_oov_buckets')) + _CategoricalColumn, + collections.namedtuple( + '_VocabularyListCategoricalColumn', + ('key', 'vocabulary_list', 'dtype', 'default_value', 'num_oov_buckets') + ) ): """See `categorical_column_with_vocabulary_list`.""" @@ -3249,17 +3441,22 @@ def _parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + inputs.get(self.key) + ) if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype)) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype + ) + ) fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key)) + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key) + ) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -3268,28 +3465,30 @@ def _transform_feature(self, inputs): input_tensor = math_ops.cast(input_tensor, dtypes.int64) return lookup_ops.index_table_from_tensor( - vocabulary_list=tuple(self.vocabulary_list), - default_value=self.default_value, - num_oov_buckets=self.num_oov_buckets, - dtype=key_dtype, - name='{}_lookup'.format(self.key)).lookup(input_tensor) + vocabulary_list=tuple(self.vocabulary_list), + default_value=self.default_value, + num_oov_buckets=self.num_oov_buckets, + dtype=key_dtype, + name='{}_lookup'.format(self.key) + ).lookup(input_tensor) @property def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return len(self.vocabulary_list) + self.num_oov_buckets - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) -class _IdentityCategoricalColumn(_CategoricalColumn, - collections.namedtuple( - '_IdentityCategoricalColumn', - ('key', 'num_buckets', 'default_value'))): +class _IdentityCategoricalColumn( + _CategoricalColumn, + collections.namedtuple( + '_IdentityCategoricalColumn', ('key', 'num_buckets', 'default_value') + ) +): """See `categorical_column_with_identity`.""" @property @@ -3301,72 +3500,89 @@ def _parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(dtypes.int64)} def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + inputs.get(self.key) + ) if not input_tensor.dtype.is_integer: - raise ValueError('Invalid input, not integer. key: {} dtype: {}'.format( - self.key, input_tensor.dtype)) + raise ValueError( + 'Invalid input, not integer. key: {} dtype: {}'.format( + self.key, input_tensor.dtype + ) + ) values = math_ops.cast(input_tensor.values, dtypes.int64, name='values') num_buckets = math_ops.cast( - self.num_buckets, dtypes.int64, name='num_buckets') + self.num_buckets, dtypes.int64, name='num_buckets' + ) zero = math_ops.cast(0, dtypes.int64, name='zero') if self.default_value is None: # Fail if values are out-of-range. assert_less = check_ops.assert_less( - values, - num_buckets, - data=(values, num_buckets), - name='assert_less_than_num_buckets') + values, + num_buckets, + data=(values, num_buckets), + name='assert_less_than_num_buckets' + ) assert_greater = check_ops.assert_greater_equal( - values, zero, data=(values,), name='assert_greater_or_equal_0') + values, zero, data=(values, ), name='assert_greater_or_equal_0' + ) with ops.control_dependencies((assert_less, assert_greater)): values = array_ops.identity(values) else: # Assign default for out-of-range values. values = array_ops.where( - math_ops.logical_or( - values < zero, values >= num_buckets, name='out_of_range'), - array_ops.fill( - dims=array_ops.shape(values), - value=math_ops.cast(self.default_value, dtypes.int64), - name='default_values'), values) + math_ops.logical_or( + values < zero, values >= num_buckets, name='out_of_range' + ), + array_ops.fill( + dims=array_ops.shape(values), + value=math_ops.cast(self.default_value, dtypes.int64), + name='default_values' + ), values + ) return sparse_tensor_lib.SparseTensor( - indices=input_tensor.indices, - values=values, - dense_shape=input_tensor.dense_shape) + indices=input_tensor.indices, + values=values, + dense_shape=input_tensor.dense_shape + ) @property def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.num_buckets - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) class _WeightedCategoricalColumn( - _CategoricalColumn, - collections.namedtuple( - '_WeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype'))): + _CategoricalColumn, + collections.namedtuple( + '_WeightedCategoricalColumn', + ('categorical_column', 'weight_feature_key', 'dtype') + ) +): """See `weighted_categorical_column`.""" @property def name(self): - return '{}_weighted_by_{}'.format(self.categorical_column.name, - self.weight_feature_key) + return '{}_weighted_by_{}'.format( + self.categorical_column.name, self.weight_feature_key + ) @property def _parse_example_spec(self): config = self.categorical_column._parse_example_spec # pylint: disable=protected-access if self.weight_feature_key in config: - raise ValueError('Parse config {} already exists for {}.'.format( - config[self.weight_feature_key], self.weight_feature_key)) + raise ValueError( + 'Parse config {} already exists for {}.'.format( + config[self.weight_feature_key], self.weight_feature_key + ) + ) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @@ -3379,22 +3595,26 @@ def _transform_feature(self, inputs): if weight_tensor is None: raise ValueError('Missing weights {}.'.format(self.weight_feature_key)) weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - weight_tensor) + weight_tensor + ) if self.dtype != weight_tensor.dtype.base_dtype: - raise ValueError('Bad dtype, expected {}, but got {}.'.format( - self.dtype, weight_tensor.dtype)) + raise ValueError( + 'Bad dtype, expected {}, but got {}.'.format( + self.dtype, weight_tensor.dtype + ) + ) if not isinstance(weight_tensor, sparse_tensor_lib.SparseTensor): # The weight tensor can be a regular Tensor. In this case, sparsify it. weight_tensor = _to_sparse_input_and_drop_ignore_values( - weight_tensor, ignore_value=0.0) + weight_tensor, ignore_value=0.0 + ) if not weight_tensor.dtype.is_floating: weight_tensor = math_ops.cast(weight_tensor, dtypes.float32) return (inputs.get(self.categorical_column), weight_tensor) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): del weight_collections del trainable tensors = inputs.get(self) @@ -3402,9 +3622,11 @@ def _get_sparse_tensors(self, class _CrossedColumn( - _CategoricalColumn, - collections.namedtuple('_CrossedColumn', - ['keys', 'hash_bucket_size', 'hash_key'])): + _CategoricalColumn, + collections.namedtuple( + '_CrossedColumn', ['keys', 'hash_bucket_size', 'hash_key'] + ) +): """See `crossed_column`.""" @property @@ -3436,26 +3658,27 @@ def _transform_feature(self, inputs): ids_and_weights = key._get_sparse_tensors(inputs) # pylint: disable=protected-access if ids_and_weights.weight_tensor is not None: raise ValueError( - 'crossed_column does not support weight_tensor, but the given ' - 'column populates weight_tensor. ' - 'Given column: {}'.format(key.name)) + 'crossed_column does not support weight_tensor, but the given ' + 'column populates weight_tensor. ' + 'Given column: {}'.format(key.name) + ) feature_tensors.append(ids_and_weights.id_tensor) else: raise ValueError('Unsupported column type. Given: {}'.format(key)) return sparse_ops.sparse_cross_hashed( - inputs=feature_tensors, - num_buckets=self.hash_bucket_size, - hash_key=self.hash_key) + inputs=feature_tensors, + num_buckets=self.hash_bucket_size, + hash_key=self.hash_key + ) @property def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.hash_bucket_size - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -3477,9 +3700,10 @@ def _collect_leaf_level_keys(cross): return leaf_level_keys -class _IndicatorColumn(_DenseColumn, _SequenceDenseColumn, - collections.namedtuple('_IndicatorColumn', - ['categorical_column'])): +class _IndicatorColumn( + _DenseColumn, _SequenceDenseColumn, + collections.namedtuple('_IndicatorColumn', ['categorical_column']) +): """Represents a one-hot column for use in deep networks. Args: @@ -3510,28 +3734,33 @@ def _transform_feature(self, inputs): # If the underlying column is weighted, return the input as a dense tensor. if weight_tensor is not None: weighted_column = sparse_ops.sparse_merge( - sp_ids=id_tensor, - sp_values=weight_tensor, - vocab_size=int(self._variable_shape[-1])) + sp_ids=id_tensor, + sp_values=weight_tensor, + vocab_size=int(self._variable_shape[-1]) + ) # Remove (?, -1) index. - weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0], - weighted_column.dense_shape) + weighted_column = sparse_ops.sparse_slice( + weighted_column, [0, 0], weighted_column.dense_shape + ) # Use scatter_nd to merge duplicated indices if existed, # instead of sparse_tensor_to_dense. - return array_ops.scatter_nd(weighted_column.indices, - weighted_column.values, - weighted_column.dense_shape) + return array_ops.scatter_nd( + weighted_column.indices, weighted_column.values, + weighted_column.dense_shape + ) dense_id_tensor = sparse_ops.sparse_tensor_to_dense( - id_tensor, default_value=-1) + id_tensor, default_value=-1 + ) # One hot must be float for tf.concat reasons since all other inputs to # input_layer are float32. one_hot_id_tensor = array_ops.one_hot( - dense_id_tensor, - depth=self._variable_shape[-1], - on_value=1.0, - off_value=0.0) + dense_id_tensor, + depth=self._variable_shape[-1], + on_value=1.0, + off_value=0.0 + ) # Reduce to get a multi-hot per example. return math_ops.reduce_sum(one_hot_id_tensor, axis=[-2]) @@ -3567,42 +3796,47 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del trainable if isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must not be of type _SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use input_layer, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'sequence_input_layer instead of input_layer. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In indicator_column: {}. ' + 'categorical_column must not be of type _SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use input_layer, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'sequence_input_layer instead of input_layer. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) # Feature has been already transformed. Return the intermediate # representation created by _transform_feature. return inputs.get(self) - def _get_sequence_dense_tensor(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sequence_dense_tensor( + self, inputs, weight_collections=None, trainable=None + ): # Do nothing with weight_collections and trainable since no variables are # created in this function. del weight_collections del trainable if not isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must be of type _SequenceCategoricalColumn ' - 'to use sequence_input_layer. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In indicator_column: {}. ' + 'categorical_column must be of type _SequenceCategoricalColumn ' + 'to use sequence_input_layer. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) # Feature has been already transformed. Return the intermediate # representation created by _transform_feature. dense_tensor = inputs.get(self) sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor) + sparse_tensors.id_tensor + ) return _SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length) + dense_tensor=dense_tensor, sequence_length=sequence_length + ) def _verify_static_batch_size_equality(tensors, columns): @@ -3623,18 +3857,22 @@ def _verify_static_batch_size_equality(tensors, columns): if expected_batch_size is None: bath_size_column_index = i expected_batch_size = tensors[i].shape.dims[0] - elif not expected_batch_size.is_compatible_with(tensors[i].shape.dims[0]): + elif not expected_batch_size.is_compatible_with( + tensors[i].shape.dims[0] + ): raise ValueError( - 'Batch size (first dimension) of each feature must be same. ' - 'Batch size of columns ({}, {}): ({}, {})'.format( - columns[bath_size_column_index].name, columns[i].name, - expected_batch_size, tensors[i].shape.dims[0])) + 'Batch size (first dimension) of each feature must be same. ' + 'Batch size of columns ({}, {}): ({}, {})'.format( + columns[bath_size_column_index].name, columns[i].name, + expected_batch_size, tensors[i].shape.dims[0] + ) + ) -class _SequenceCategoricalColumn(_CategoricalColumn, - collections.namedtuple( - '_SequenceCategoricalColumn', - ['categorical_column'])): +class _SequenceCategoricalColumn( + _CategoricalColumn, + collections.namedtuple('_SequenceCategoricalColumn', ['categorical_column']) +): """Represents sequences of categorical data.""" @property @@ -3652,10 +3890,9 @@ def _transform_feature(self, inputs): def _num_buckets(self): return self.categorical_column._num_buckets # pylint: disable=protected-access - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access id_tensor = sparse_tensors.id_tensor weight_tensor = sparse_tensors.weight_tensor diff --git a/easy_rec/python/compat/feature_column/feature_column_v2.py b/easy_rec/python/compat/feature_column/feature_column_v2.py index 4e785a566..d121846ad 100644 --- a/easy_rec/python/compat/feature_column/feature_column_v2.py +++ b/easy_rec/python/compat/feature_column/feature_column_v2.py @@ -124,22 +124,18 @@ the API subject to change, and should not be relied upon! """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import abc import collections import math -import os -import sys - import numpy as np +import os import six +import sys import tensorflow as tf from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops +from tensorflow.python.framework import dtypes, ops from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib from tensorflow.python.framework import tensor_shape # TODO(b/118385027): Dependency on keras can be problematic if Keras moves out @@ -147,37 +143,25 @@ from tensorflow.python.keras import utils from tensorflow.python.keras.engine import training from tensorflow.python.keras.engine.base_layer import Layer -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables +from tensorflow.python.ops import array_ops, check_ops, control_flow_ops, embedding_ops, init_ops, lookup_ops, math_ops, nn_ops, parsing_ops, sparse_ops, string_ops, variable_scope, variables # NOQA from tensorflow.python.platform import gfile from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import checkpoint_utils # from tensorflow.python.training.tracking import tracking -from tensorflow.python.util import deprecation -from tensorflow.python.util import nest +from tensorflow.python.util import deprecation, nest from easy_rec.python.compat import ops as compat_ops from easy_rec.python.compat.feature_column import feature_column as fc_old from easy_rec.python.compat.feature_column import utils as fc_utils -from easy_rec.python.layers import utils as layer_utils - from easy_rec.python.compat.feature_column.feature_column import embedding_lookup_ragged # NOQA +from easy_rec.python.layers import utils as layer_utils _FEATURE_COLUMN_DEPRECATION_DATE = None -_FEATURE_COLUMN_DEPRECATION = ('The old _FeatureColumn APIs are being ' - 'deprecated. Please use the new FeatureColumn ' - 'APIs instead.') +_FEATURE_COLUMN_DEPRECATION = ( + 'The old _FeatureColumn APIs are being ' + 'deprecated. Please use the new FeatureColumn ' + 'APIs instead.' +) if os.getenv('SAFE_EMBEDDING', 'TRUE') == 'TRUE': embedding_lookup_sparse = embedding_ops.safe_embedding_lookup_sparse @@ -194,14 +178,16 @@ class StateManager(object): only. """ - def create_variable(self, - feature_column, - name, - shape, - dtype=None, - trainable=True, - use_resource=True, - initializer=None): + def create_variable( + self, + feature_column, + name, + shape, + dtype=None, + trainable=True, + use_resource=True, + initializer=None + ): """Creates a new variable. Args: @@ -283,28 +269,31 @@ def __init__(self, layer, trainable): self._layer = layer self._cols_to_vars_map = collections.defaultdict(lambda: {}) - def create_variable(self, - feature_column, - name, - shape, - dtype=None, - trainable=True, - use_resource=True, - initializer=None): + def create_variable( + self, + feature_column, + name, + shape, + dtype=None, + trainable=True, + use_resource=True, + initializer=None + ): if name in self._cols_to_vars_map[feature_column]: raise ValueError('Variable already exists.') var = self._layer.add_variable( - name=name, - shape=shape, - dtype=dtype, - initializer=initializer, - trainable=self._trainable and trainable, - use_resource=use_resource, - # TODO(rohanj): Get rid of this hack once we have a mechanism for - # specifying a default partitioner for an entire layer. In that case, - # the default getter for Layers should work. - getter=variable_scope.get_variable) + name=name, + shape=shape, + dtype=dtype, + initializer=initializer, + trainable=self._trainable and trainable, + use_resource=use_resource, + # TODO(rohanj): Get rid of this hack once we have a mechanism for + # specifying a default partitioner for an entire layer. In that case, + # the default getter for Layers should work. + getter=variable_scope.get_variable + ) self._cols_to_vars_map[feature_column][name] = var return var @@ -333,19 +322,22 @@ class _BaseFeaturesLayer(Layer): `expected_column_type`. """ - def __init__(self, feature_columns, expected_column_type, trainable, name, - **kwargs): - super(_BaseFeaturesLayer, self).__init__( - name=name, trainable=trainable, **kwargs) + def __init__( + self, feature_columns, expected_column_type, trainable, name, **kwargs + ): + super(_BaseFeaturesLayer, + self).__init__(name=name, trainable=trainable, **kwargs) self._feature_columns = _normalize_feature_columns(feature_columns) self._state_manager = _StateManagerImpl(self, self.trainable) for column in self._feature_columns: if not isinstance(column, expected_column_type): raise ValueError( - 'Items of feature_columns must be a {}. ' - 'You can wrap a categorical column with an ' - 'embedding_column or indicator_column. Given: {}'.format( - expected_column_type, column)) + 'Items of feature_columns must be a {}. ' + 'You can wrap a categorical column with an ' + 'embedding_column or indicator_column. Given: {}'.format( + expected_column_type, column + ) + ) def build(self, _): for column in self._feature_columns: @@ -436,11 +428,12 @@ def __init__(self, feature_columns, trainable=True, name=None, **kwargs): ValueError: if an item in `feature_columns` is not a `DenseColumn`. """ super(DenseFeatures, self).__init__( - feature_columns=feature_columns, - trainable=trainable, - name=name, - expected_column_type=DenseColumn, - **kwargs) + feature_columns=feature_columns, + trainable=trainable, + name=name, + expected_column_type=DenseColumn, + **kwargs + ) @property def _is_feature_layer(self): @@ -469,14 +462,16 @@ def call(self, features, cols_to_output_tensors=None): ValueError: If features are not a dictionary. """ if not isinstance(features, dict): - raise ValueError('We expected a dictionary here. Instead we got: ', - features) + raise ValueError( + 'We expected a dictionary here. Instead we got: ', features + ) transformation_cache = FeatureTransformationCache(features) output_tensors = [] for column in self._feature_columns: with ops.name_scope(column.name): - tensor = column.get_dense_tensor(transformation_cache, - self._state_manager) + tensor = column.get_dense_tensor( + transformation_cache, self._state_manager + ) processed_tensors = self._process_dense_tensor(column, tensor) if cols_to_output_tensors is not None: cols_to_output_tensors[column] = processed_tensors @@ -487,22 +482,25 @@ def call(self, features, cols_to_output_tensors=None): class _LinearModelLayer(Layer): """Layer that contains logic for `LinearModel`.""" - def __init__(self, - feature_columns, - units=1, - sparse_combiner='sum', - trainable=True, - name=None, - **kwargs): - super(_LinearModelLayer, self).__init__( - name=name, trainable=trainable, **kwargs) + def __init__( + self, + feature_columns, + units=1, + sparse_combiner='sum', + trainable=True, + name=None, + **kwargs + ): + super(_LinearModelLayer, + self).__init__(name=name, trainable=trainable, **kwargs) self._feature_columns = _normalize_feature_columns(feature_columns) for column in self._feature_columns: if not isinstance(column, (DenseColumn, CategoricalColumn)): raise ValueError( - 'Items of feature_columns must be either a ' - 'DenseColumn or CategoricalColumn. Given: {}'.format(column)) + 'Items of feature_columns must be either a ' + 'DenseColumn or CategoricalColumn. Given: {}'.format(column) + ) self._units = units self._sparse_combiner = sparse_combiner @@ -527,32 +525,35 @@ def build(self, _): else: first_dim = column.variable_shape.num_elements() self._state_manager.create_variable( - column, - name='weights', - dtype=dtypes.float32, - shape=(first_dim, self._units), - initializer=init_ops.zeros_initializer(), - trainable=self.trainable) + column, + name='weights', + dtype=dtypes.float32, + shape=(first_dim, self._units), + initializer=init_ops.zeros_initializer(), + trainable=self.trainable + ) # Create a bias variable. self.bias = self.add_variable( - name='bias_weights', - dtype=dtypes.float32, - shape=[self._units], - initializer=init_ops.zeros_initializer(), - trainable=self.trainable, - use_resource=True, - # TODO(rohanj): Get rid of this hack once we have a mechanism for - # specifying a default partitioner for an entire layer. In that case, - # the default getter for Layers should work. - getter=variable_scope.get_variable) + name='bias_weights', + dtype=dtypes.float32, + shape=[self._units], + initializer=init_ops.zeros_initializer(), + trainable=self.trainable, + use_resource=True, + # TODO(rohanj): Get rid of this hack once we have a mechanism for + # specifying a default partitioner for an entire layer. In that case, + # the default getter for Layers should work. + getter=variable_scope.get_variable + ) super(_LinearModelLayer, self).build(None) def call(self, features): if not isinstance(features, dict): raise ValueError( - 'We expected a dictionary here. Instead we got: {}'.format(features)) + 'We expected a dictionary here. Instead we got: {}'.format(features) + ) with ops.name_scope(self.name): transformation_cache = FeatureTransformationCache(features) weighted_sums = [] @@ -563,18 +564,21 @@ def call(self, features): weight_var = self._state_manager.get_variable(column, 'weights') weighted_sum = _create_weighted_sum( - column=column, - transformation_cache=transformation_cache, - state_manager=self._state_manager, - sparse_combiner=self._sparse_combiner, - weight_var=weight_var) + column=column, + transformation_cache=transformation_cache, + state_manager=self._state_manager, + sparse_combiner=self._sparse_combiner, + weight_var=weight_var + ) weighted_sums.append(weighted_sum) _verify_static_batch_size_equality(weighted_sums, self._feature_columns) predictions_no_bias = math_ops.add_n( - weighted_sums, name='weighted_sum_no_bias') + weighted_sums, name='weighted_sum_no_bias' + ) predictions = nn_ops.bias_add( - predictions_no_bias, self.bias, name='weighted_sum') + predictions_no_bias, self.bias, name='weighted_sum' + ) return predictions @@ -617,13 +621,15 @@ class LinearModel(training.Model): ``` """ - def __init__(self, - feature_columns, - units=1, - sparse_combiner='sum', - trainable=True, - name=None, - **kwargs): + def __init__( + self, + feature_columns, + units=1, + sparse_combiner='sum', + trainable=True, + name=None, + **kwargs + ): """Constructs a LinearLayer. Args: @@ -682,12 +688,13 @@ def __init__(self, """ super(LinearModel, self).__init__(name=name, **kwargs) self.layer = _LinearModelLayer( - feature_columns, - units, - sparse_combiner, - trainable, - name=self.name, - **kwargs) + feature_columns, + units, + sparse_combiner, + trainable, + name=self.name, + **kwargs + ) def call(self, features): """Returns a `Tensor` the represents the predictions of a linear model. @@ -749,7 +756,8 @@ def _transform_features_v2(features, feature_columns, state_manager): feature_columns = _normalize_feature_columns(feature_columns) outputs = {} with ops.name_scope( - None, default_name='transform_features', values=features.values()): + None, default_name='transform_features', values=features.values() + ): transformation_cache = FeatureTransformationCache(features) for column in feature_columns: with ops.name_scope(None, default_name=column.name): @@ -805,27 +813,33 @@ def make_parse_example_spec_v2(feature_columns): result = {} for column in feature_columns: if not isinstance(column, FeatureColumn): - raise ValueError('All feature_columns must be FeatureColumn instances. ' - 'Given: {}'.format(column)) + raise ValueError( + 'All feature_columns must be FeatureColumn instances. ' + 'Given: {}'.format(column) + ) config = column.parse_example_spec for key, value in six.iteritems(config): if key in result and value != result[key]: - raise ValueError('feature_columns contain different parse_spec for key ' - '{}. Given {} and {}'.format(key, value, result[key])) + raise ValueError( + 'feature_columns contain different parse_spec for key ' + '{}. Given {} and {}'.format(key, value, result[key]) + ) result.update(config) return result -def embedding_column(categorical_column, - dimension, - combiner='mean', - initializer=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True, - partitioner=None, - ev_params=None): +def embedding_column( + categorical_column, + dimension, + combiner='mean', + initializer=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True, + partitioner=None, + ev_params=None +): """`DenseColumn` that converts from sparse, categorical input. Use this when your inputs are sparse, but you want to convert them to a dense @@ -900,41 +914,48 @@ def model_fn(features, ...): if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError('Must specify both `ckpt_to_load_from` and ' - '`tensor_name_in_ckpt` or none of them.') + raise ValueError( + 'Must specify both `ckpt_to_load_from` and ' + '`tensor_name_in_ckpt` or none of them.' + ) if (initializer is not None) and (not callable(initializer)): - raise ValueError('initializer must be callable if specified. ' - 'Embedding of column_name: {}'.format( - categorical_column.name)) + raise ValueError( + 'initializer must be callable if specified. ' + 'Embedding of column_name: {}'.format(categorical_column.name) + ) if initializer is None: initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=0.01 / math.sqrt(dimension)) + mean=0.0, stddev=0.01 / math.sqrt(dimension) + ) return EmbeddingColumn( - categorical_column=categorical_column, - dimension=dimension, - combiner=combiner, - initializer=initializer, - ckpt_to_load_from=ckpt_to_load_from, - tensor_name_in_ckpt=tensor_name_in_ckpt, - max_norm=max_norm, - trainable=trainable, - partitioner=partitioner, - ev_params=ev_params) - - -def shared_embedding_columns(categorical_columns, - dimension, - combiner='mean', - initializer=None, - shared_embedding_collection_name=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True, - partitioner=None, - ev_params=None): + categorical_column=categorical_column, + dimension=dimension, + combiner=combiner, + initializer=initializer, + ckpt_to_load_from=ckpt_to_load_from, + tensor_name_in_ckpt=tensor_name_in_ckpt, + max_norm=max_norm, + trainable=trainable, + partitioner=partitioner, + ev_params=ev_params + ) + + +def shared_embedding_columns( + categorical_columns, + dimension, + combiner='mean', + initializer=None, + shared_embedding_collection_name=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True, + partitioner=None, + ev_params=None +): """List of dense columns that convert from sparse, categorical input. This is similar to `embedding_column`, except that it produces a list of @@ -1032,20 +1053,25 @@ def model_fn(features, ...): RuntimeError: if eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('shared_embedding_columns are not supported when eager ' - 'execution is enabled.') + raise RuntimeError( + 'shared_embedding_columns are not supported when eager ' + 'execution is enabled.' + ) if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError('Must specify both `ckpt_to_load_from` and ' - '`tensor_name_in_ckpt` or none of them.') + raise ValueError( + 'Must specify both `ckpt_to_load_from` and ' + '`tensor_name_in_ckpt` or none of them.' + ) if (initializer is not None) and (not callable(initializer)): raise ValueError('initializer must be callable if specified.') if initializer is None: initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=0.01 / math.sqrt(dimension)) + mean=0.0, stddev=0.01 / math.sqrt(dimension) + ) # Sort the columns so the default collection name is deterministic even if the # user passes columns from an unsorted collection, such as dict.values(). @@ -1055,21 +1081,26 @@ def model_fn(features, ...): num_buckets = c0._num_buckets # pylint: disable=protected-access if not isinstance(c0, fc_old._CategoricalColumn): # pylint: disable=protected-access raise ValueError( - 'All categorical_columns must be subclasses of _CategoricalColumn. ' - 'Given: {}, of type: {}'.format(c0, type(c0))) - if isinstance(c0, - (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): # pylint: disable=protected-access + 'All categorical_columns must be subclasses of _CategoricalColumn. ' + 'Given: {}, of type: {}'.format(c0, type(c0)) + ) + if isinstance( + c0, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn) + ): # pylint: disable=protected-access c0 = c0.categorical_column for c in sorted_columns[1:]: if isinstance( - c, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): # pylint: disable=protected-access + c, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn) + ): # pylint: disable=protected-access c = c.categorical_column if num_buckets != c._num_buckets: # pylint: disable=protected-access raise ValueError( - 'To use shared_embedding_column, all categorical_columns must have ' - 'the same number of buckets. Given column: {} with buckets: {} does ' - 'not match column: {} with buckets: {}'.format( - c0, num_buckets, c, c._num_buckets)) # pylint: disable=protected-access + 'To use shared_embedding_column, all categorical_columns must have ' + 'the same number of buckets. Given column: {} with buckets: {} does ' + 'not match column: {} with buckets: {}'.format( + c0, num_buckets, c, c._num_buckets + ) + ) # pylint: disable=protected-access if not shared_embedding_collection_name: shared_embedding_collection_name = '_'.join(c.name for c in sorted_columns) @@ -1094,15 +1125,17 @@ def model_fn(features, ...): return result -def shared_embedding_columns_v2(categorical_columns, - dimension, - combiner='mean', - initializer=None, - shared_embedding_collection_name=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True): +def shared_embedding_columns_v2( + categorical_columns, + dimension, + combiner='mean', + initializer=None, + shared_embedding_collection_name=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True +): """List of dense columns that convert from sparse, categorical input. This is similar to `embedding_column`, except that it produces a list of @@ -1199,20 +1232,25 @@ def model_fn(features, ...): RuntimeError: if eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('shared_embedding_columns are not supported when eager ' - 'execution is enabled.') + raise RuntimeError( + 'shared_embedding_columns are not supported when eager ' + 'execution is enabled.' + ) if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError('Must specify both `ckpt_to_load_from` and ' - '`tensor_name_in_ckpt` or none of them.') + raise ValueError( + 'Must specify both `ckpt_to_load_from` and ' + '`tensor_name_in_ckpt` or none of them.' + ) if (initializer is not None) and (not callable(initializer)): raise ValueError('initializer must be callable if specified.') if initializer is None: initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=0.01 / math.sqrt(dimension)) + mean=0.0, stddev=0.01 / math.sqrt(dimension) + ) # Sort the columns so the default collection name is deterministic even if the # user passes columns from an unsorted collection, such as dict.values(). @@ -1222,8 +1260,9 @@ def model_fn(features, ...): num_buckets = c0.num_buckets if not isinstance(c0, CategoricalColumn): raise ValueError( - 'All categorical_columns must be subclasses of CategoricalColumn. ' - 'Given: {}, of type: {}'.format(c0, type(c0))) + 'All categorical_columns must be subclasses of CategoricalColumn. ' + 'Given: {}, of type: {}'.format(c0, type(c0)) + ) if isinstance(c0, WeightedCategoricalColumn): c0 = c0.categorical_column for c in sorted_columns[1:]: @@ -1231,40 +1270,48 @@ def model_fn(features, ...): c = c.categorical_column if not isinstance(c, type(c0)): raise ValueError( - 'To use shared_embedding_column, all categorical_columns must have ' - 'the same type, or be weighted_categorical_column of the same type. ' - 'Given column: {} of type: {} does not match given column: {} of ' - 'type: {}'.format(c0, type(c0), c, type(c))) + 'To use shared_embedding_column, all categorical_columns must have ' + 'the same type, or be weighted_categorical_column of the same type. ' + 'Given column: {} of type: {} does not match given column: {} of ' + 'type: {}'.format(c0, type(c0), c, type(c)) + ) if num_buckets != c.num_buckets: raise ValueError( - 'To use shared_embedding_column, all categorical_columns must have ' - 'the same number of buckets. Given column: {} with buckets: {} does ' - 'not match column: {} with buckets: {}'.format( - c0, num_buckets, c, c.num_buckets)) + 'To use shared_embedding_column, all categorical_columns must have ' + 'the same number of buckets. Given column: {} with buckets: {} does ' + 'not match column: {} with buckets: {}'.format( + c0, num_buckets, c, c.num_buckets + ) + ) if not shared_embedding_collection_name: shared_embedding_collection_name = '_'.join(c.name for c in sorted_columns) shared_embedding_collection_name += '_shared_embedding' column_creator = SharedEmbeddingColumnCreator( - dimension, initializer, ckpt_to_load_from, tensor_name_in_ckpt, - num_buckets, trainable, shared_embedding_collection_name) + dimension, initializer, ckpt_to_load_from, tensor_name_in_ckpt, + num_buckets, trainable, shared_embedding_collection_name + ) result = [] for column in categorical_columns: result.append( - column_creator( - categorical_column=column, combiner=combiner, max_norm=max_norm)) + column_creator( + categorical_column=column, combiner=combiner, max_norm=max_norm + ) + ) return result -def numeric_column(key, - shape=(1,), - default_value=None, - dtype=dtypes.float32, - normalizer_fn=None, - feature_name=None): +def numeric_column( + key, + shape=(1, ), + default_value=None, + dtype=dtypes.float32, + normalizer_fn=None, + feature_name=None +): """Represents real valued or numerical features. Example: @@ -1318,22 +1365,28 @@ def numeric_column(key, """ shape = _check_shape(shape, key) if not (dtype.is_integer or dtype.is_floating): - raise ValueError('dtype must be convertible to float. ' - 'dtype: {}, key: {}'.format(dtype, key)) - default_value = fc_utils.check_default_value(shape, default_value, dtype, key) + raise ValueError( + 'dtype must be convertible to float. ' + 'dtype: {}, key: {}'.format(dtype, key) + ) + default_value = fc_utils.check_default_value( + shape, default_value, dtype, key + ) if normalizer_fn is not None and not callable(normalizer_fn): raise TypeError( - 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) + 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn) + ) fc_utils.assert_key_is_string(key) return NumericColumn( - feature_name=feature_name, - key=key, - shape=shape, - default_value=default_value, - dtype=dtype, - normalizer_fn=normalizer_fn) + feature_name=feature_name, + key=key, + shape=shape, + default_value=default_value, + dtype=dtype, + normalizer_fn=normalizer_fn + ) def bucketized_column(source_column, boundaries): @@ -1404,11 +1457,14 @@ def bucketized_column(source_column, boundaries): """ if not isinstance(source_column, (NumericColumn, fc_old._NumericColumn)): # pylint: disable=protected-access raise ValueError( - 'source_column must be a column generated with numeric_column(). ' - 'Given: {}'.format(source_column)) + 'source_column must be a column generated with numeric_column(). ' + 'Given: {}'.format(source_column) + ) if len(source_column.shape) > 1: - raise ValueError('source_column must be one-dimensional column. ' - 'Given: {}'.format(source_column)) + raise ValueError( + 'source_column must be one-dimensional column. ' + 'Given: {}'.format(source_column) + ) if not boundaries: raise ValueError('boundaries must not be empty.') if not (isinstance(boundaries, list) or isinstance(boundaries, tuple)): @@ -1419,10 +1475,9 @@ def bucketized_column(source_column, boundaries): return BucketizedColumn(source_column, tuple(boundaries)) -def categorical_column_with_hash_bucket(key, - hash_bucket_size, - dtype=dtypes.string, - feature_name=None): +def categorical_column_with_hash_bucket( + key, hash_bucket_size, dtype=dtypes.string, feature_name=None +): """Represents sparse feature where ids are set by hashing. Use this when your sparse features are in string or integer format, and you @@ -1465,12 +1520,14 @@ def categorical_column_with_hash_bucket(key, ValueError: `dtype` is neither string nor integer. """ if hash_bucket_size is None: - raise ValueError('hash_bucket_size must be set. ' 'key: {}'.format(key)) + raise ValueError('hash_bucket_size must be set. ' + 'key: {}'.format(key)) if hash_bucket_size < 1: - raise ValueError('hash_bucket_size must be at least 1. ' - 'hash_bucket_size: {}, key: {}'.format( - hash_bucket_size, key)) + raise ValueError( + 'hash_bucket_size must be at least 1. ' + 'hash_bucket_size: {}, key: {}'.format(hash_bucket_size, key) + ) fc_utils.assert_key_is_string(key) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) @@ -1478,13 +1535,15 @@ def categorical_column_with_hash_bucket(key, return HashedCategoricalColumn(feature_name, key, hash_bucket_size, dtype) -def categorical_column_with_vocabulary_file_v2(key, - vocabulary_file, - vocabulary_size=None, - dtype=dtypes.string, - default_value=None, - num_oov_buckets=0, - feature_name=None): +def categorical_column_with_vocabulary_file_v2( + key, + vocabulary_file, + vocabulary_size=None, + dtype=dtypes.string, + default_value=None, + num_oov_buckets=0, + feature_name=None +): """A `CategoricalColumn` with a vocabulary file. Use this when your inputs are in string or integer format, and you have a @@ -1573,8 +1632,9 @@ def categorical_column_with_vocabulary_file_v2(key, with gfile.GFile(vocabulary_file) as f: vocabulary_size = sum(1 for _ in f) logging.info( - 'vocabulary_size = %d in %s is inferred from the number of elements ' - 'in the vocabulary_file %s.', vocabulary_size, key, vocabulary_file) + 'vocabulary_size = %d in %s is inferred from the number of elements ' + 'in the vocabulary_file %s.', vocabulary_size, key, vocabulary_file + ) # `vocabulary_size` isn't required for lookup, but it is for `_num_buckets`. if vocabulary_size < 1: @@ -1582,29 +1642,34 @@ def categorical_column_with_vocabulary_file_v2(key, if num_oov_buckets: if default_value is not None: raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'.format( - key)) + 'Can\'t specify both num_oov_buckets and default_value in {}.'. + format(key) + ) if num_oov_buckets < 0: - raise ValueError('Invalid num_oov_buckets {} in {}.'.format( - num_oov_buckets, key)) + raise ValueError( + 'Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key) + ) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) return VocabularyFileCategoricalColumn( - feature_name=feature_name, - key=key, - vocabulary_file=vocabulary_file, - vocabulary_size=vocabulary_size, - num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, - default_value=-1 if default_value is None else default_value, - dtype=dtype) - - -def categorical_column_with_vocabulary_list(key, - vocabulary_list, - dtype=None, - default_value=-1, - num_oov_buckets=0, - feature_name=None): + feature_name=feature_name, + key=key, + vocabulary_file=vocabulary_file, + vocabulary_size=vocabulary_size, + num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, + default_value=-1 if default_value is None else default_value, + dtype=dtype + ) + + +def categorical_column_with_vocabulary_list( + key, + vocabulary_list, + dtype=None, + default_value=-1, + num_oov_buckets=0, + feature_name=None +): """A `CategoricalColumn` with in-memory vocabulary. Use this when your inputs are in string or integer format, and you have an @@ -1682,45 +1747,54 @@ def categorical_column_with_vocabulary_list(key, """ if (vocabulary_list is None) or (len(vocabulary_list) < 1): raise ValueError( - 'vocabulary_list {} must be non-empty, column_name: {}'.format( - vocabulary_list, key)) + 'vocabulary_list {} must be non-empty, column_name: {}'.format( + vocabulary_list, key + ) + ) if len(set(vocabulary_list)) != len(vocabulary_list): raise ValueError( - 'Duplicate keys in vocabulary_list {}, column_name: {}'.format( - vocabulary_list, key)) + 'Duplicate keys in vocabulary_list {}, column_name: {}'.format( + vocabulary_list, key + ) + ) vocabulary_dtype = dtypes.as_dtype(np.array(vocabulary_list).dtype) if num_oov_buckets: if default_value != -1: raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'.format( - key)) + 'Can\'t specify both num_oov_buckets and default_value in {}.'. + format(key) + ) if num_oov_buckets < 0: - raise ValueError('Invalid num_oov_buckets {} in {}.'.format( - num_oov_buckets, key)) + raise ValueError( + 'Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key) + ) fc_utils.assert_string_or_int( - vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key)) + vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key) + ) if dtype is None: dtype = vocabulary_dtype elif dtype.is_integer != vocabulary_dtype.is_integer: raise ValueError( - 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format( - dtype, vocabulary_dtype, key)) + 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format( + dtype, vocabulary_dtype, key + ) + ) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) return VocabularyListCategoricalColumn( - feature_name=feature_name, - key=key, - vocabulary_list=tuple(vocabulary_list), - dtype=dtype, - default_value=default_value, - num_oov_buckets=num_oov_buckets) + feature_name=feature_name, + key=key, + vocabulary_list=tuple(vocabulary_list), + dtype=dtype, + default_value=default_value, + num_oov_buckets=num_oov_buckets + ) -def categorical_column_with_identity(key, - num_buckets, - default_value=None, - feature_name=None): +def categorical_column_with_identity( + key, num_buckets, default_value=None, feature_name=None +): """A `CategoricalColumn` that returns identity values. Use this when your inputs are integers in the range `[0, num_buckets)`, and @@ -1775,19 +1849,23 @@ def categorical_column_with_identity(key, ValueError: if `default_value` is not in range `[0, num_buckets)`. """ if num_buckets < 1: - raise ValueError('num_buckets {} < 1, column_name {}'.format( - num_buckets, key)) - if (default_value is not None) and ((default_value < 0) or - (default_value >= num_buckets)): raise ValueError( - 'default_value {} not in range [0, {}), column_name {}'.format( - default_value, num_buckets, key)) + 'num_buckets {} < 1, column_name {}'.format(num_buckets, key) + ) + if (default_value is not None + ) and ((default_value < 0) or (default_value >= num_buckets)): + raise ValueError( + 'default_value {} not in range [0, {}), column_name {}'.format( + default_value, num_buckets, key + ) + ) fc_utils.assert_key_is_string(key) return IdentityCategoricalColumn( - feature_name=feature_name, - key=key, - number_buckets=num_buckets, - default_value=default_value) + feature_name=feature_name, + key=key, + number_buckets=num_buckets, + default_value=default_value + ) def indicator_column(categorical_column): @@ -1824,9 +1902,9 @@ def indicator_column(categorical_column): return IndicatorColumn(categorical_column) -def weighted_categorical_column(categorical_column, - weight_feature_key, - dtype=dtypes.float32): +def weighted_categorical_column( + categorical_column, weight_feature_key, dtype=dtypes.float32 +): """Applies weight values to a `CategoricalColumn`. Use this when each of your sparse inputs has both an ID and a value. For @@ -1894,9 +1972,10 @@ def weighted_categorical_column(categorical_column, if (dtype is None) or not (dtype.is_integer or dtype.is_floating): raise ValueError('dtype {} is not convertible to float.'.format(dtype)) return WeightedCategoricalColumn( - categorical_column=categorical_column, - weight_feature_key=weight_feature_key, - dtype=dtype) + categorical_column=categorical_column, + weight_feature_key=weight_feature_key, + dtype=dtype + ) def crossed_column(keys, hash_bucket_size, hash_key=None, feature_name=None): @@ -2004,29 +2083,38 @@ def crossed_column(keys, hash_bucket_size, hash_key=None, feature_name=None): ValueError: If `hash_bucket_size < 1`. """ if not hash_bucket_size or hash_bucket_size < 1: - raise ValueError('hash_bucket_size must be > 1. ' - 'hash_bucket_size: {}'.format(hash_bucket_size)) + raise ValueError( + 'hash_bucket_size must be > 1. ' + 'hash_bucket_size: {}'.format(hash_bucket_size) + ) if not keys or len(keys) < 2: raise ValueError( - 'keys must be a list with length > 1. Given: {}'.format(keys)) + 'keys must be a list with length > 1. Given: {}'.format(keys) + ) for key in keys: - if (not isinstance(key, six.string_types) and - not isinstance(key, (CategoricalColumn, fc_old._CategoricalColumn))): # pylint: disable=protected-access + if ( + not isinstance(key, six.string_types) + and not isinstance(key, (CategoricalColumn, fc_old._CategoricalColumn)) + ): # pylint: disable=protected-access raise ValueError( - 'Unsupported key type. All keys must be either string, or ' - 'categorical column except HashedCategoricalColumn. ' - 'Given: {}'.format(key)) - if isinstance(key, - (HashedCategoricalColumn, fc_old._HashedCategoricalColumn)): # pylint: disable=protected-access + 'Unsupported key type. All keys must be either string, or ' + 'categorical column except HashedCategoricalColumn. ' + 'Given: {}'.format(key) + ) + if isinstance( + key, (HashedCategoricalColumn, fc_old._HashedCategoricalColumn) + ): # pylint: disable=protected-access raise ValueError( - 'categorical_column_with_hash_bucket is not supported for crossing. ' - 'Hashing before crossing will increase probability of collision. ' - 'Instead, use the feature name as a string. Given: {}'.format(key)) + 'categorical_column_with_hash_bucket is not supported for crossing. ' + 'Hashing before crossing will increase probability of collision. ' + 'Instead, use the feature name as a string. Given: {}'.format(key) + ) return CrossedColumn( - feature_name=feature_name, - keys=tuple(keys), - hash_bucket_size=hash_bucket_size, - hash_key=hash_key) + feature_name=feature_name, + keys=tuple(keys), + hash_bucket_size=hash_bucket_size, + hash_key=hash_key + ) @six.add_metaclass(abc.ABCMeta) @@ -2266,26 +2354,30 @@ def is_feature_column_v2(feature_columns): return True -def _create_weighted_sum(column, transformation_cache, state_manager, - sparse_combiner, weight_var): +def _create_weighted_sum( + column, transformation_cache, state_manager, sparse_combiner, weight_var +): """Creates a weighted sum for a dense/categorical column for linear_model.""" if isinstance(column, CategoricalColumn): return _create_categorical_column_weighted_sum( - column=column, - transformation_cache=transformation_cache, - state_manager=state_manager, - sparse_combiner=sparse_combiner, - weight_var=weight_var) + column=column, + transformation_cache=transformation_cache, + state_manager=state_manager, + sparse_combiner=sparse_combiner, + weight_var=weight_var + ) else: return _create_dense_column_weighted_sum( - column=column, - transformation_cache=transformation_cache, - state_manager=state_manager, - weight_var=weight_var) + column=column, + transformation_cache=transformation_cache, + state_manager=state_manager, + weight_var=weight_var + ) -def _create_dense_column_weighted_sum(column, transformation_cache, - state_manager, weight_var): +def _create_dense_column_weighted_sum( + column, transformation_cache, state_manager, weight_var +): """Create a weighted sum of a dense column for linear_model.""" tensor = column.get_dense_tensor(transformation_cache, state_manager) num_elements = column.variable_shape.num_elements() @@ -2331,9 +2423,9 @@ def get_sparse_tensors(self, transformation_cache, state_manager): pass -def _create_categorical_column_weighted_sum(column, transformation_cache, - state_manager, sparse_combiner, - weight_var): +def _create_categorical_column_weighted_sum( + column, transformation_cache, state_manager, sparse_combiner, weight_var +): # pylint: disable=g-doc-return-or-yield,g-doc-args """Create a weighted sum of a categorical column for linear_model. @@ -2361,22 +2453,26 @@ def _create_categorical_column_weighted_sum(column, transformation_cache, For both cases, we can implement weighted sum via embedding_lookup with sparse_combiner = "sum". """ - sparse_tensors = column.get_sparse_tensors(transformation_cache, - state_manager) + sparse_tensors = column.get_sparse_tensors( + transformation_cache, state_manager + ) id_tensor = sparse_ops.sparse_reshape( - sparse_tensors.id_tensor, - [array_ops.shape(sparse_tensors.id_tensor)[0], -1]) + sparse_tensors.id_tensor, + [array_ops.shape(sparse_tensors.id_tensor)[0], -1] + ) weight_tensor = sparse_tensors.weight_tensor if weight_tensor is not None: weight_tensor = sparse_ops.sparse_reshape( - weight_tensor, [array_ops.shape(weight_tensor)[0], -1]) + weight_tensor, [array_ops.shape(weight_tensor)[0], -1] + ) return embedding_lookup_sparse( - weight_var, - id_tensor, - sparse_weights=weight_tensor, - combiner=sparse_combiner, - name='weighted_sum') + weight_var, + id_tensor, + sparse_weights=weight_tensor, + combiner=sparse_combiner, + name='weighted_sum' + ) class SequenceDenseColumn(FeatureColumn): @@ -2473,8 +2569,10 @@ def get(self, key, state_manager): raise ValueError('Feature {} is not in features dictionary.'.format(key)) if not isinstance(key, FeatureColumn): - raise TypeError('"key" must be either a "str" or "FeatureColumn". ' - 'Provided: {}'.format(key)) + raise TypeError( + '"key" must be either a "str" or "FeatureColumn". ' + 'Provided: {}'.format(key) + ) column = key logging.debug('Transforming feature_column %s.', column) @@ -2504,13 +2602,15 @@ def _get_raw_feature_as_tensor(self, key): """ raw_feature = self._features[key] feature_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - raw_feature) + raw_feature + ) def expand_dims(input_tensor): # Input_tensor must have rank 1. if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): - return sparse_ops.sparse_reshape(input_tensor, - [array_ops.shape(input_tensor)[0], 1]) + return sparse_ops.sparse_reshape( + input_tensor, [array_ops.shape(input_tensor)[0], 1] + ) else: return array_ops.expand_dims(input_tensor, -1) @@ -2518,20 +2618,27 @@ def expand_dims(input_tensor): if rank is not None: if rank == 0: raise ValueError( - 'Feature (key: {}) cannot have rank 0. Give: {}'.format( - key, feature_tensor)) + 'Feature (key: {}) cannot have rank 0. Give: {}'.format( + key, feature_tensor + ) + ) return feature_tensor if rank != 1 else expand_dims(feature_tensor) # Handle dynamic rank. - with ops.control_dependencies([ + with ops.control_dependencies( + [ check_ops.assert_positive( - array_ops.rank(feature_tensor), - message='Feature (key: {}) cannot have rank 0. Given: {}'.format( - key, feature_tensor)) - ]): + array_ops.rank(feature_tensor), + message='Feature (key: {}) cannot have rank 0. Given: {}'.format( + key, feature_tensor + ) + ) + ] + ): return control_flow_ops.cond( - math_ops.equal(1, array_ops.rank(feature_tensor)), - lambda: expand_dims(feature_tensor), lambda: feature_tensor) + math_ops.equal(1, array_ops.rank(feature_tensor)), + lambda: expand_dims(feature_tensor), lambda: feature_tensor + ) # TODO(ptucker): Move to third_party/tensorflow/python/ops/sparse_ops.py @@ -2556,13 +2663,16 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): return input_tensor input_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - input_tensor) + input_tensor + ) if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): return input_tensor - with ops.name_scope(None, 'to_sparse_input', ( + with ops.name_scope( + None, 'to_sparse_input', ( input_tensor, ignore_value, - )): + ) + ): if ignore_value is None: if input_tensor.dtype == dtypes.string: # Exception due to TF strings are converted to numpy objects by default. @@ -2575,14 +2685,18 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): # default value for that type. ignore_value = input_tensor.dtype.as_numpy_dtype() ignore_value = math_ops.cast( - ignore_value, input_tensor.dtype, name='ignore_value') + ignore_value, input_tensor.dtype, name='ignore_value' + ) indices = array_ops.where( - math_ops.not_equal(input_tensor, ignore_value), name='indices') + math_ops.not_equal(input_tensor, ignore_value), name='indices' + ) return sparse_tensor_lib.SparseTensor( - indices=indices, - values=array_ops.gather_nd(input_tensor, indices, name='values'), - dense_shape=array_ops.shape( - input_tensor, out_type=dtypes.int64, name='dense_shape')) + indices=indices, + values=array_ops.gather_nd(input_tensor, indices, name='values'), + dense_shape=array_ops.shape( + input_tensor, out_type=dtypes.int64, name='dense_shape' + ) + ) def _normalize_feature_columns(feature_columns): @@ -2612,30 +2726,36 @@ def _normalize_feature_columns(feature_columns): for column in feature_columns: if not isinstance(column, FeatureColumn): - raise ValueError('Items of feature_columns must be a FeatureColumn. ' - 'Given (type {}): {}.'.format(type(column), column)) + raise ValueError( + 'Items of feature_columns must be a FeatureColumn. ' + 'Given (type {}): {}.'.format(type(column), column) + ) if not feature_columns: raise ValueError('feature_columns must not be empty.') name_to_column = {} for column in feature_columns: if column.name in name_to_column: - raise ValueError('Duplicate feature column name found for columns: {} ' - 'and {}. This usually means that these columns refer to ' - 'same base feature. Either one must be discarded or a ' - 'duplicated but renamed item must be inserted in ' - 'features dict.'.format(column, - name_to_column[column.name])) + raise ValueError( + 'Duplicate feature column name found for columns: {} ' + 'and {}. This usually means that these columns refer to ' + 'same base feature. Either one must be discarded or a ' + 'duplicated but renamed item must be inserted in ' + 'features dict.'.format(column, name_to_column[column.name]) + ) name_to_column[column.name] = column return sorted(feature_columns, key=lambda x: x.name) class NumericColumn( - DenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - collections.namedtuple('NumericColumn', - ('feature_name', 'key', 'shape', 'default_value', - 'dtype', 'normalizer_fn'))): + DenseColumn, + fc_old._DenseColumn, # pylint: disable=protected-access + collections.namedtuple( + 'NumericColumn', ( + 'feature_name', 'key', 'shape', 'default_value', 'dtype', 'normalizer_fn' + ) + ) +): """See `numeric_column`.""" @property @@ -2656,28 +2776,30 @@ def raw_name(self): def parse_example_spec(self): """See `FeatureColumn` base class.""" return { - self.key: - parsing_ops.FixedLenFeature(self.shape, self.dtype, - self.default_value) + self.key: + parsing_ops.FixedLenFeature(self.shape, self.dtype, self.default_value) } @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.parse_example_spec def _transform_input_tensor(self, input_tensor): if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): raise ValueError( - 'The corresponding Tensor of numerical column must be a Tensor. ' - 'SparseTensor is not supported. key: {}'.format(self.key)) + 'The corresponding Tensor of numerical column must be a Tensor. ' + 'SparseTensor is not supported. key: {}'.format(self.key) + ) if self.normalizer_fn is not None: input_tensor = self.normalizer_fn(input_tensor) return math_ops.cast(input_tensor, dtypes.float32) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): input_tensor = inputs.get(self.key) return self._transform_input_tensor(input_tensor) @@ -2708,8 +2830,9 @@ def variable_shape(self): return tensor_shape.TensorShape(self.shape) @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _variable_shape(self): return self.variable_shape @@ -2729,8 +2852,9 @@ def get_dense_tensor(self, transformation_cache, state_manager): # representation created by _transform_feature. return transformation_cache.get(self, state_manager) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -2754,24 +2878,27 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['normalizer_fn'] = utils.deserialize_keras_object( - config['normalizer_fn'], custom_objects=custom_objects) + config['normalizer_fn'], custom_objects=custom_objects + ) kwargs['dtype'] = dtypes.as_dtype(config['dtype']) return cls(**kwargs) class BucketizedColumn( - DenseColumn, - CategoricalColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('BucketizedColumn', - ('source_column', 'boundaries'))): + DenseColumn, + CategoricalColumn, + fc_old._DenseColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, # pylint: disable=protected-access + collections.namedtuple('BucketizedColumn', ('source_column', 'boundaries')) +): """See `bucketized_column`.""" @property def _is_v2_column(self): - return (isinstance(self.source_column, FeatureColumn) and - self.source_column._is_v2_column) # pylint: disable=protected-access + return ( + isinstance(self.source_column, FeatureColumn) + and self.source_column._is_v2_column + ) # pylint: disable=protected-access @property def name(self): @@ -2789,13 +2916,15 @@ def parse_example_spec(self): return self.source_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.source_column._parse_example_spec # pylint: disable=protected-access - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): """Returns bucketized categorical `source_column` tensor.""" source_tensor = inputs.get(self.source_column) @@ -2814,28 +2943,32 @@ def transform_feature(self, transformation_cache, state_manager): def variable_shape(self): """See `DenseColumn` base class.""" return tensor_shape.TensorShape( - tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) + tuple(self.source_column.shape) + (len(self.boundaries) + 1, ) + ) @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _variable_shape(self): return self.variable_shape def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( - indices=math_ops.cast(input_tensor, dtypes.int64), - depth=len(self.boundaries) + 1, - on_value=1., - off_value=0.) + indices=math_ops.cast(input_tensor, dtypes.int64), + depth=len(self.boundaries) + 1, + on_value=1., + off_value=0. + ) def get_dense_tensor(self, transformation_cache, state_manager): """Returns one hot encoded dense `Tensor`.""" input_tensor = transformation_cache.get(self, state_manager) return self._get_dense_tensor_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -2849,8 +2982,9 @@ def num_buckets(self): return (len(self.boundaries) + 1) * self.source_column.shape[0] @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _num_buckets(self): return self.num_buckets @@ -2860,22 +2994,27 @@ def _get_sparse_tensors_for_input_tensor(self, input_tensor): source_dimension = self.source_column.shape[0] i1 = array_ops.reshape( - array_ops.tile( - array_ops.expand_dims(math_ops.range(0, batch_size), 1), - [1, source_dimension]), (-1,)) + array_ops.tile( + array_ops.expand_dims(math_ops.range(0, batch_size), 1), + [1, source_dimension] + ), (-1, ) + ) i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size]) # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets bucket_indices = ( - array_ops.reshape(input_tensor, - (-1,)) + (len(self.boundaries) + 1) * i2) + array_ops.reshape(input_tensor, (-1, )) + (len(self.boundaries) + 1) * i2 + ) indices = math_ops.cast( - array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64) + array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64 + ) dense_shape = math_ops.cast( - array_ops.stack([batch_size, source_dimension]), dtypes.int64) + array_ops.stack([batch_size, source_dimension]), dtypes.int64 + ) sparse_tensor = sparse_tensor_lib.SparseTensor( - indices=indices, values=bucket_indices, dense_shape=dense_shape) + indices=indices, values=bucket_indices, dense_shape=dense_shape + ) return CategoricalColumn.IdWeightPair(sparse_tensor, None) def get_sparse_tensors(self, transformation_cache, state_manager): @@ -2883,12 +3022,12 @@ def get_sparse_tensors(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_sparse_tensors_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): """Converts dense inputs to SparseTensor so downstream code can use it.""" del weight_collections del trainable @@ -2912,23 +3051,28 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['source_column'] = deserialize_feature_column( - config['source_column'], custom_objects, columns_by_name) + config['source_column'], custom_objects, columns_by_name + ) return cls(**kwargs) class SequenceBucketizedColumn( - DenseColumn, - CategoricalColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('SequenceBucketizedColumn', - ('source_column', 'boundaries'))): + DenseColumn, + CategoricalColumn, + fc_old._DenseColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, # pylint: disable=protected-access + collections.namedtuple( + 'SequenceBucketizedColumn', ('source_column', 'boundaries') + ) +): """See `bucketized_column`.""" @property def _is_v2_column(self): - return (isinstance(self.source_column, FeatureColumn) and - self.source_column._is_v2_column) # pylint: disable=protected-access + return ( + isinstance(self.source_column, FeatureColumn) + and self.source_column._is_v2_column + ) # pylint: disable=protected-access @property def name(self): @@ -2946,22 +3090,26 @@ def parse_example_spec(self): return self.source_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.source_column._parse_example_spec # pylint: disable=protected-access - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): """Returns bucketized categorical `source_column` tensor.""" source_tensor = inputs.get(self.source_column) bucketize_values = math_ops._bucketize( - source_tensor.values, boundaries=self.boundaries) + source_tensor.values, boundaries=self.boundaries + ) bucketize_tensor = sparse_tensor_lib.SparseTensor( - indices=source_tensor.indices, - values=bucketize_values, - dense_shape=source_tensor.dense_shape) + indices=source_tensor.indices, + values=bucketize_values, + dense_shape=source_tensor.dense_shape + ) return bucketize_tensor def transform_feature(self, transformation_cache, state_manager): @@ -2975,28 +3123,32 @@ def transform_feature(self, transformation_cache, state_manager): def variable_shape(self): """See `DenseColumn` base class.""" return tensor_shape.TensorShape( - tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) + tuple(self.source_column.shape) + (len(self.boundaries) + 1, ) + ) @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _variable_shape(self): return self.variable_shape def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( - indices=math_ops.cast(input_tensor, dtypes.int64), - depth=len(self.boundaries) + 1, - on_value=1., - off_value=0.) + indices=math_ops.cast(input_tensor, dtypes.int64), + depth=len(self.boundaries) + 1, + on_value=1., + off_value=0. + ) def get_dense_tensor(self, transformation_cache, state_manager): """Returns one hot encoded dense `Tensor`.""" input_tensor = transformation_cache.get(self, state_manager) return self._get_dense_tensor_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -3010,8 +3162,9 @@ def num_buckets(self): return (len(self.boundaries) + 1) * self.source_column.shape[0] @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _num_buckets(self): return self.num_buckets @@ -3026,13 +3179,14 @@ def _get_sparse_tensors_for_input_tensor(self, input_sparse_tensor): # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets bucket_indices = ( - array_ops.reshape(input_tensor, - (-1,)) + (len(self.boundaries) + 1) * i2) + array_ops.reshape(input_tensor, (-1, )) + (len(self.boundaries) + 1) * i2 + ) sparse_tensor = sparse_tensor_lib.SparseTensor( - indices=input_indices, - values=bucket_indices, - dense_shape=input_sparse_tensor.dense_shape) + indices=input_indices, + values=bucket_indices, + dense_shape=input_sparse_tensor.dense_shape + ) # Compute the third dimension explicitly instead of setting it to -1, as # that doesn't work for dynamically shaped tensors with 0-length at runtime. # This happens for empty sequences. @@ -3046,12 +3200,12 @@ def get_sparse_tensors(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_sparse_tensors_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): """Converts dense inputs to SparseTensor so downstream code can use it.""" del weight_collections del trainable @@ -3075,23 +3229,28 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['source_column'] = deserialize_feature_column( - config['source_column'], custom_objects, columns_by_name) + config['source_column'], custom_objects, columns_by_name + ) return cls(**kwargs) class SequenceNumericColumn( - DenseColumn, - CategoricalColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('SequenceNumericColumn', - ('source_column', 'sequence_length'))): + DenseColumn, + CategoricalColumn, + fc_old._DenseColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, # pylint: disable=protected-access + collections.namedtuple( + 'SequenceNumericColumn', ('source_column', 'sequence_length') + ) +): """See `SequenceNumericColumn`.""" @property def _is_v2_column(self): - return (isinstance(self.source_column, FeatureColumn) and - self.source_column._is_v2_column) # pylint: disable=protected-access + return ( + isinstance(self.source_column, FeatureColumn) + and self.source_column._is_v2_column + ) # pylint: disable=protected-access @property def name(self): @@ -3109,13 +3268,15 @@ def parse_example_spec(self): return self.source_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.source_column._parse_example_spec # pylint: disable=protected-access - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): """Returns bucketized categorical `source_column` tensor.""" source_tensor = inputs.get(self.source_column) @@ -3130,28 +3291,32 @@ def transform_feature(self, transformation_cache, state_manager): def variable_shape(self): """See `DenseColumn` base class.""" return tensor_shape.TensorShape( - tuple(self.source_column.shape) + (self.sequence_length,)) + tuple(self.source_column.shape) + (self.sequence_length, ) + ) @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _variable_shape(self): return self.variable_shape def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( - indices=math_ops.cast(input_tensor, dtypes.int64), - depth=self.sequence_length, - on_value=1., - off_value=0.) + indices=math_ops.cast(input_tensor, dtypes.int64), + depth=self.sequence_length, + on_value=1., + off_value=0. + ) def get_dense_tensor(self, transformation_cache, state_manager): """Returns one hot encoded dense `Tensor`.""" input_tensor = transformation_cache.get(self, state_manager) return self._get_dense_tensor_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -3161,14 +3326,17 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): def _get_sequence_dense_tensor(self, inputs): input_tensor = inputs.get(self) sparse_tensors = self._get_sparse_tensors_for_input_tensor( - input_tensor).id_tensor + input_tensor + ).id_tensor sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors) + sparse_tensors + ) sequence_length = tf.cast(sequence_length, tf.int32) shape = array_ops.shape(sparse_tensors) target_shape = [shape[0], shape[1], math_ops.reduce_prod(shape[2:])] - ret_tensor = tf.sparse_to_dense(sparse_tensors.indices, target_shape, - sparse_tensors.values) + ret_tensor = tf.sparse_to_dense( + sparse_tensors.indices, target_shape, sparse_tensors.values + ) return CategoricalColumn.IdWeightPair(ret_tensor, sequence_length) @property @@ -3178,8 +3346,9 @@ def num_buckets(self): return self.sequence_length * self.source_column.shape[0] @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _num_buckets(self): return self.num_buckets @@ -3197,12 +3366,12 @@ def get_sparse_tensors(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_sparse_tensors_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): """Converts dense inputs to SparseTensor so downstream code can use it.""" del weight_collections del trainable @@ -3226,28 +3395,34 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['source_column'] = deserialize_feature_column( - config['source_column'], custom_objects, columns_by_name) + config['source_column'], custom_objects, columns_by_name + ) return cls(**kwargs) class SequenceWeightedCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple( - 'SequenceWeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype'))): + CategoricalColumn, + fc_old._CategoricalColumn, # pylint: disable=protected-access + collections.namedtuple( + 'SequenceWeightedCategoricalColumn', + ('categorical_column', 'weight_feature_key', 'dtype') + ) +): """See `weighted_categorical_column`.""" @property def _is_v2_column(self): - return (isinstance(self.categorical_column, FeatureColumn) and - self.categorical_column._is_v2_column) # pylint: disable=protected-access + return ( + isinstance(self.categorical_column, FeatureColumn) + and self.categorical_column._is_v2_column + ) # pylint: disable=protected-access @property def name(self): """See `FeatureColumn` base class.""" - return '{}_weighted_by_{}'.format(self.categorical_column.name, - self.weight_feature_key) + return '{}_weighted_by_{}'.format( + self.categorical_column.name, self.weight_feature_key + ) @property def raw_name(self): @@ -3259,19 +3434,26 @@ def parse_example_spec(self): """See `FeatureColumn` base class.""" config = self.categorical_column.parse_example_spec if self.weight_feature_key in config: - raise ValueError('Parse config {} already exists for {}.'.format( - config[self.weight_feature_key], self.weight_feature_key)) + raise ValueError( + 'Parse config {} already exists for {}.'.format( + config[self.weight_feature_key], self.weight_feature_key + ) + ) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): config = self.categorical_column._parse_example_spec # pylint: disable=protected-access if self.weight_feature_key in config: - raise ValueError('Parse config {} already exists for {}.'.format( - config[self.weight_feature_key], self.weight_feature_key)) + raise ValueError( + 'Parse config {} already exists for {}.'.format( + config[self.weight_feature_key], self.weight_feature_key + ) + ) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @@ -3281,8 +3463,9 @@ def num_buckets(self): return self.categorical_column.num_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _num_buckets(self): return self.categorical_column._num_buckets # pylint: disable=protected-access @@ -3290,14 +3473,19 @@ def _transform_weight_tensor(self, weight_tensor): if weight_tensor is None: raise ValueError('Missing weights {}.'.format(self.weight_feature_key)) weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - weight_tensor) + weight_tensor + ) if self.dtype != weight_tensor.dtype.base_dtype: - raise ValueError('Bad dtype, expected {}, but got {}.'.format( - self.dtype, weight_tensor.dtype)) + raise ValueError( + 'Bad dtype, expected {}, but got {}.'.format( + self.dtype, weight_tensor.dtype + ) + ) if not isinstance(weight_tensor, sparse_tensor_lib.SparseTensor): # The weight tensor can be a regular Tensor. In this case, sparsify it. weight_tensor = _to_sparse_input_and_drop_ignore_values( - weight_tensor, ignore_value=0.0) + weight_tensor, ignore_value=0.0 + ) if not weight_tensor.dtype.is_floating: weight_tensor = math_ops.cast(weight_tensor, dtypes.float32) shape = tf.shape(weight_tensor) @@ -3307,14 +3495,18 @@ def _transform_weight_tensor(self, weight_tensor): def transform_feature(self, transformation_cache, state_manager): """Applies weights to tensor generated from `categorical_column`'.""" - weight_tensor = transformation_cache.get(self.weight_feature_key, - state_manager) + weight_tensor = transformation_cache.get( + self.weight_feature_key, state_manager + ) weight_tensor = self._transform_weight_tensor(weight_tensor) - return (transformation_cache.get(self.categorical_column, - state_manager), weight_tensor) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + return ( + transformation_cache.get(self.categorical_column, + state_manager), weight_tensor + ) + + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): """Applies weights to tensor generated from `categorical_column`'.""" weight_tensor = inputs.get(self.weight_feature_key) @@ -3326,12 +3518,12 @@ def get_sparse_tensors(self, transformation_cache, state_manager): tensors = transformation_cache.get(self, state_manager) return CategoricalColumn.IdWeightPair(tensors[0], tensors[1]) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): del weight_collections del trainable tensors = inputs.get(self) @@ -3346,7 +3538,8 @@ def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) config['categorical_column'] = serialize_feature_column( - self.categorical_column) + self.categorical_column + ) config['dtype'] = self.dtype.name return config @@ -3356,27 +3549,33 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['categorical_column'] = deserialize_feature_column( - config['categorical_column'], custom_objects, columns_by_name) + config['categorical_column'], custom_objects, columns_by_name + ) kwargs['dtype'] = dtypes.as_dtype(config['dtype']) return cls(**kwargs) class EmbeddingColumn( - DenseColumn, - SequenceDenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._SequenceDenseColumn, # pylint: disable=protected-access - collections.namedtuple( - 'EmbeddingColumn', - ('categorical_column', 'dimension', 'combiner', 'initializer', - 'ckpt_to_load_from', 'tensor_name_in_ckpt', 'max_norm', 'trainable', - 'partitioner', 'ev_params'))): + DenseColumn, + SequenceDenseColumn, + fc_old._DenseColumn, # pylint: disable=protected-access + fc_old._SequenceDenseColumn, # pylint: disable=protected-access + collections.namedtuple( + 'EmbeddingColumn', ( + 'categorical_column', 'dimension', 'combiner', 'initializer', + 'ckpt_to_load_from', 'tensor_name_in_ckpt', 'max_norm', 'trainable', + 'partitioner', 'ev_params' + ) + ) +): """See `embedding_column`.""" @property def _is_v2_column(self): - return (isinstance(self.categorical_column, FeatureColumn) and - self.categorical_column._is_v2_column) # pylint: disable=protected-access + return ( + isinstance(self.categorical_column, FeatureColumn) + and self.categorical_column._is_v2_column + ) # pylint: disable=protected-access @property def name(self): @@ -3394,8 +3593,9 @@ def parse_example_spec(self): return self.categorical_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.categorical_column._parse_example_spec # pylint: disable=protected-access @@ -3403,8 +3603,9 @@ def transform_feature(self, transformation_cache, state_manager): """Transforms underlying `categorical_column`.""" return transformation_cache.get(self.categorical_column, state_manager) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): return inputs.get(self.categorical_column) @@ -3414,8 +3615,9 @@ def variable_shape(self): return tensor_shape.TensorShape([self.dimension]) @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _variable_shape(self): return self.variable_shape @@ -3423,16 +3625,18 @@ def create_state(self, state_manager): """Creates the embedding lookup variable.""" embedding_shape = (self.categorical_column._num_buckets, self.dimension) # pylint: disable=protected-access state_manager.create_variable( - self, - name='embedding_weights', - shape=embedding_shape, - dtype=dtypes.float32, - trainable=self.trainable, - use_resource=True, - initializer=self.initializer) - - def _get_dense_tensor_internal_helper(self, sparse_tensors, - embedding_weights): + self, + name='embedding_weights', + shape=embedding_shape, + dtype=dtypes.float32, + trainable=self.trainable, + use_resource=True, + initializer=self.initializer + ) + + def _get_dense_tensor_internal_helper( + self, sparse_tensors, embedding_weights + ): sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor @@ -3441,49 +3645,58 @@ def _get_dense_tensor_internal_helper(self, sparse_tensors, if isinstance(to_restore, variables.PartitionedVariable): to_restore = to_restore._get_variable_list() # pylint: disable=protected-access checkpoint_utils.init_from_checkpoint( - self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) + self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore} + ) if 'RaggedTensor' in str(type(sparse_ids)): return embedding_lookup_ragged( - embedding_weights, - sparse_ids, - sparse_weights, - combiner=self.combiner, - max_norm=self.max_norm, - name='%s_weights' % self.name) - - # Return embedding lookup result. - return embedding_lookup_sparse( embedding_weights, sparse_ids, sparse_weights, combiner=self.combiner, - name='%s_weights' % self.name, - max_norm=self.max_norm) + max_norm=self.max_norm, + name='%s_weights' % self.name + ) + + # Return embedding lookup result. + return embedding_lookup_sparse( + embedding_weights, + sparse_ids, + sparse_weights, + combiner=self.combiner, + name='%s_weights' % self.name, + max_norm=self.max_norm + ) def _get_dense_tensor_internal(self, sparse_tensors, state_manager): """Private method that follows the signature of get_dense_tensor.""" embedding_weights = state_manager.get_variable( - self, name='embedding_weights') - return self._get_dense_tensor_internal_helper(sparse_tensors, - embedding_weights) - - def _old_get_dense_tensor_internal(self, sparse_tensors, weight_collections, - trainable): + self, name='embedding_weights' + ) + return self._get_dense_tensor_internal_helper( + sparse_tensors, embedding_weights + ) + + def _old_get_dense_tensor_internal( + self, sparse_tensors, weight_collections, trainable + ): """Private method that follows the signature of _get_dense_tensor.""" embedding_shape = (self.categorical_column._num_buckets, self.dimension) # pylint: disable=protected-access - if (weight_collections and - ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections): + if ( + weight_collections + and ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections + ): weight_collections.append(ops.GraphKeys.GLOBAL_VARIABLES) if self.ev_params is None: embedding_weights = variable_scope.get_variable( - name='embedding_weights', - shape=embedding_shape, - dtype=dtypes.float32, # bfloat16, - initializer=self.initializer, - trainable=self.trainable and trainable, - partitioner=self.partitioner, - collections=weight_collections) + name='embedding_weights', + shape=embedding_shape, + dtype=dtypes.float32, # bfloat16, + initializer=self.initializer, + trainable=self.trainable and trainable, + partitioner=self.partitioner, + collections=weight_collections + ) else: # at eval or inference time, it is necessary to set # the initializers to zeros, so that new key will @@ -3497,48 +3710,57 @@ def _old_get_dense_tensor_internal(self, sparse_tensors, weight_collections, if 'EmbeddingVariableConfig' in dir(variables): ev_option = variables.EmbeddingVariableOption() ev_option.filter_strategy = variables.CounterFilter( - filter_freq=self.ev_params.filter_freq) + filter_freq=self.ev_params.filter_freq + ) extra_args['ev_option'] = ev_option else: extra_args['filter_options'] = variables.CounterFilterOptions( - self.ev_params.filter_freq) + self.ev_params.filter_freq + ) embedding_weights = variable_scope.get_embedding_variable( - name='embedding_weights', - embedding_dim=self.dimension, - initializer=initializer, - trainable=self.trainable and trainable, - partitioner=self.partitioner, - collections=weight_collections, - steps_to_live=self.ev_params.steps_to_live, - **extra_args) + name='embedding_weights', + embedding_dim=self.dimension, + initializer=initializer, + trainable=self.trainable and trainable, + partitioner=self.partitioner, + collections=weight_collections, + steps_to_live=self.ev_params.steps_to_live, + **extra_args + ) # Write the embedding configuration to RTP-specified collections. This will inform RTP to # optimize this embedding operation. embedding_attrs = layer_utils.gen_embedding_attrs( - column=self, - variable=embedding_weights, - bucket_size=self.categorical_column._num_buckets, - combiner=self.combiner, - is_embedding_var=(self.ev_params is not None)) + column=self, + variable=embedding_weights, + bucket_size=self.categorical_column._num_buckets, + combiner=self.combiner, + is_embedding_var=(self.ev_params is not None) + ) embedding_attrs['name'] = layer_utils.unique_name_in_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name']) + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name'] + ) layer_utils.update_attr_to_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs) + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs + ) # operate embedding predictions = self._get_dense_tensor_internal_helper( - sparse_tensors, embedding_weights) + sparse_tensors, embedding_weights + ) # Update the information about the output and input nodes of embedding operation to the # previous written RTP-specific collection entry. RTP uses these informations to extract # the embedding subgraph. if isinstance(sparse_tensors.id_tensor, sparse_tensor_lib.SparseTensor): layer_utils.append_tensor_to_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name'], - 'tensor', predictions) + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name'], + 'tensor', predictions + ) layer_utils.append_tensor_to_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name'], - 'input', sparse_tensors.id_tensor) + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name'], + 'input', sparse_tensors.id_tensor + ) return predictions @@ -3559,85 +3781,106 @@ def get_dense_tensor(self, transformation_cache, state_manager): """ if isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must not be of type SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use DenseFeatures, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In embedding_column: {}. ' + 'categorical_column must not be of type SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use DenseFeatures, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'SequenceFeatures instead of DenseFeatures. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) # Get sparse IDs and weights. sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager) + transformation_cache, state_manager + ) return self._get_dense_tensor_internal(sparse_tensors, state_manager) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): if isinstance( - self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn)): # pylint: disable=protected-access + self.categorical_column, + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn) + ): # pylint: disable=protected-access raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must not be of type _SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use DenseFeatures, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In embedding_column: {}. ' + 'categorical_column must not be of type _SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use DenseFeatures, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'SequenceFeatures instead of DenseFeatures. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) sparse_tensors = self.categorical_column._get_sparse_tensors( # pylint: disable=protected-access inputs, weight_collections, trainable) - return self._old_get_dense_tensor_internal(sparse_tensors, - weight_collections, trainable) + return self._old_get_dense_tensor_internal( + sparse_tensors, weight_collections, trainable + ) def get_sequence_dense_tensor(self, transformation_cache, state_manager): """See `SequenceDenseColumn` base class.""" if not isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must be of type SequenceCategoricalColumn ' - 'to use SequenceFeatures. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In embedding_column: {}. ' + 'categorical_column must be of type SequenceCategoricalColumn ' + 'to use SequenceFeatures. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager) - dense_tensor = self._get_dense_tensor_internal(sparse_tensors, - state_manager) + transformation_cache, state_manager + ) + dense_tensor = self._get_dense_tensor_internal( + sparse_tensors, state_manager + ) sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor) + sparse_tensors.id_tensor + ) return SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sequence_dense_tensor(self, - inputs, - weight_collections=None, - trainable=None): + dense_tensor=dense_tensor, sequence_length=sequence_length + ) + + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sequence_dense_tensor( + self, inputs, weight_collections=None, trainable=None + ): if not isinstance( - self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn, - SequenceBucketizedColumn, SequenceNumericColumn, - SequenceWeightedCategoricalColumn)): # pylint: disable=protected-access + self.categorical_column, ( + SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn, + SequenceBucketizedColumn, SequenceNumericColumn, + SequenceWeightedCategoricalColumn + ) + ): # pylint: disable=protected-access raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must be of type SequenceCategoricalColumn ' - 'to use SequenceFeatures. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In embedding_column: {}. ' + 'categorical_column must be of type SequenceCategoricalColumn ' + 'to use SequenceFeatures. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access dense_tensor = self._old_get_dense_tensor_internal( - sparse_tensors, - weight_collections=weight_collections, - trainable=trainable) + sparse_tensors, + weight_collections=weight_collections, + trainable=trainable + ) sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor) + sparse_tensors.id_tensor + ) return SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length) + dense_tensor=dense_tensor, sequence_length=sequence_length + ) @property def parents(self): @@ -3648,7 +3891,8 @@ def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) config['categorical_column'] = serialize_feature_column( - self.categorical_column) + self.categorical_column + ) config['initializer'] = utils.serialize_keras_object(self.initializer) return config @@ -3658,29 +3902,35 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['categorical_column'] = deserialize_feature_column( - config['categorical_column'], custom_objects, columns_by_name) + config['categorical_column'], custom_objects, columns_by_name + ) kwargs['initializer'] = utils.deserialize_keras_object( - config['initializer'], custom_objects=custom_objects) + config['initializer'], custom_objects=custom_objects + ) return cls(**kwargs) def _raise_shared_embedding_column_error(): - raise ValueError('SharedEmbeddingColumns are not supported in ' - '`linear_model` or `input_layer`. Please use ' - '`DenseFeatures` or `LinearModel` instead.') + raise ValueError( + 'SharedEmbeddingColumns are not supported in ' + '`linear_model` or `input_layer`. Please use ' + '`DenseFeatures` or `LinearModel` instead.' + ) # class SharedEmbeddingColumnCreator(tracking.AutoTrackable): class SharedEmbeddingColumnCreator: - def __init__(self, - dimension, - initializer, - ckpt_to_load_from, - tensor_name_in_ckpt, - num_buckets, - trainable, - name='shared_embedding_column_creator'): + def __init__( + self, + dimension, + initializer, + ckpt_to_load_from, + tensor_name_in_ckpt, + num_buckets, + trainable, + name='shared_embedding_column_creator' + ): self._dimension = dimension self._initializer = initializer self._ckpt_to_load_from = ckpt_to_load_from @@ -3700,18 +3950,20 @@ def embedding_weights(self): if key not in self._embedding_weights: embedding_shape = (self._num_buckets, self._dimension) var = variable_scope.get_variable( - name=self._name, - shape=embedding_shape, - dtype=dtypes.float32, - initializer=self._initializer, - trainable=self._trainable) + name=self._name, + shape=embedding_shape, + dtype=dtypes.float32, + initializer=self._initializer, + trainable=self._trainable + ) if self._ckpt_to_load_from is not None: to_restore = var if isinstance(to_restore, variables.PartitionedVariable): to_restore = to_restore._get_variable_list() # pylint: disable=protected-access checkpoint_utils.init_from_checkpoint( - self._ckpt_to_load_from, {self._tensor_name_in_ckpt: to_restore}) + self._ckpt_to_load_from, {self._tensor_name_in_ckpt: to_restore} + ) self._embedding_weights[key] = var return self._embedding_weights[key] @@ -3721,14 +3973,17 @@ def dimension(self): class SharedEmbeddingColumn( - DenseColumn, - SequenceDenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._SequenceDenseColumn, # pylint: disable=protected-access - collections.namedtuple( - 'SharedEmbeddingColumn', - ('categorical_column', 'shared_embedding_column_creator', 'combiner', - 'max_norm'))): + DenseColumn, + SequenceDenseColumn, + fc_old._DenseColumn, # pylint: disable=protected-access + fc_old._SequenceDenseColumn, # pylint: disable=protected-access + collections.namedtuple( + 'SharedEmbeddingColumn', ( + 'categorical_column', 'shared_embedding_column_creator', 'combiner', + 'max_norm' + ) + ) +): """See `embedding_column`.""" @property @@ -3765,7 +4020,8 @@ def _transform_feature(self, inputs): def variable_shape(self): """See `DenseColumn` base class.""" return tensor_shape.TensorShape( - [self.shared_embedding_column_creator.dimension]) + [self.shared_embedding_column_creator.dimension] + ) @property def _variable_shape(self): @@ -3779,7 +4035,8 @@ def _get_dense_tensor_internal(self, transformation_cache, state_manager): with ops.name_scope(None, default_name=self.name): # Get sparse IDs and weights. sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager) + transformation_cache, state_manager + ) sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor @@ -3787,25 +4044,28 @@ def _get_dense_tensor_internal(self, transformation_cache, state_manager): # Return embedding lookup result. return embedding_lookup_sparse( - embedding_weights=embedding_weights, - sparse_ids=sparse_ids, - sparse_weights=sparse_weights, - combiner=self.combiner, - name='%s_weights' % self.name, - max_norm=self.max_norm) + embedding_weights=embedding_weights, + sparse_ids=sparse_ids, + sparse_weights=sparse_weights, + combiner=self.combiner, + name='%s_weights' % self.name, + max_norm=self.max_norm + ) def get_dense_tensor(self, transformation_cache, state_manager): """Returns the embedding lookup result.""" if isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must not be of type SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use DenseFeatures, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In embedding_column: {}. ' + 'categorical_column must not be of type SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use DenseFeatures, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'SequenceFeatures instead of DenseFeatures. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) return self._get_dense_tensor_internal(transformation_cache, state_manager) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): @@ -3815,25 +4075,30 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): """See `SequenceDenseColumn` base class.""" if not isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must be of type SequenceCategoricalColumn ' - 'to use SequenceFeatures. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) - dense_tensor = self._get_dense_tensor_internal(transformation_cache, - state_manager) + 'In embedding_column: {}. ' + 'categorical_column must be of type SequenceCategoricalColumn ' + 'to use SequenceFeatures. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) + dense_tensor = self._get_dense_tensor_internal( + transformation_cache, state_manager + ) sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager) + transformation_cache, state_manager + ) sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor) + sparse_tensors.id_tensor + ) return SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length) + dense_tensor=dense_tensor, sequence_length=sequence_length + ) - def _get_sequence_dense_tensor(self, - inputs, - weight_collections=None, - trainable=None): + def _get_sequence_dense_tensor( + self, inputs, weight_collections=None, trainable=None + ): return _raise_shared_embedding_column_error() @property @@ -3859,19 +4124,25 @@ def _check_shape(shape, key): shape = tuple(shape) for dimension in shape: if not isinstance(dimension, int): - raise TypeError('shape dimensions must be integer. ' - 'shape: {}, key: {}'.format(shape, key)) + raise TypeError( + 'shape dimensions must be integer. ' + 'shape: {}, key: {}'.format(shape, key) + ) if dimension < 1: - raise ValueError('shape dimensions must be greater than 0. ' - 'shape: {}, key: {}'.format(shape, key)) + raise ValueError( + 'shape dimensions must be greater than 0. ' + 'shape: {}, key: {}'.format(shape, key) + ) return shape class HashedCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('HashedCategoricalColumn', - ('feature_name', 'key', 'hash_bucket_size', 'dtype')) + CategoricalColumn, + fc_old._CategoricalColumn, # pylint: disable=protected-access + collections.namedtuple( + 'HashedCategoricalColumn', + ('feature_name', 'key', 'hash_bucket_size', 'dtype') + ) ): """See `categorical_column_with_hash_bucket`.""" @@ -3895,22 +4166,26 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.parse_example_spec def _transform_input_tensor(self, input_tensor): """Hashes the values in the feature_column.""" fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key)) + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key) + ) if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype)) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype + ) + ) if input_tensor.dtype == dtypes.string: sparse_values = input_tensor.values @@ -3918,27 +4193,33 @@ def _transform_input_tensor(self, input_tensor): sparse_values = string_ops.as_string(input_tensor.values) sparse_id_values = string_ops.string_to_hash_bucket_fast( - sparse_values, self.hash_bucket_size, name='lookup') + sparse_values, self.hash_bucket_size, name='lookup' + ) if 'RaggedTensor' in str(type(input_tensor)): from tensorflow.python.ops.ragged import ragged_tensor return ragged_tensor.RaggedTensor.from_row_splits( - values=sparse_id_values, row_splits=input_tensor.row_splits) + values=sparse_id_values, row_splits=input_tensor.row_splits + ) - return sparse_tensor_lib.SparseTensor(input_tensor.indices, - sparse_id_values, - input_tensor.dense_shape) + return sparse_tensor_lib.SparseTensor( + input_tensor.indices, sparse_id_values, input_tensor.dense_shape + ) def transform_feature(self, transformation_cache, state_manager): """Hashes the values in the feature_column.""" input_tensor = _to_sparse_input_and_drop_ignore_values( - transformation_cache.get(self.key, state_manager)) + transformation_cache.get(self.key, state_manager) + ) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + inputs.get(self.key) + ) return self._transform_input_tensor(input_tensor) @property @@ -3947,22 +4228,24 @@ def num_buckets(self): return self.hash_bucket_size @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" return CategoricalColumn.IdWeightPair( - transformation_cache.get(self, state_manager), None) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + transformation_cache.get(self, state_manager), None + ) + + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -3988,12 +4271,15 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class VocabularyFileCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple( - 'VocabularyFileCategoricalColumn', - ('feature_name', 'key', 'vocabulary_file', 'vocabulary_size', - 'num_oov_buckets', 'dtype', 'default_value'))): + CategoricalColumn, + fc_old._CategoricalColumn, # pylint: disable=protected-access + collections.namedtuple( + 'VocabularyFileCategoricalColumn', ( + 'feature_name', 'key', 'vocabulary_file', 'vocabulary_size', + 'num_oov_buckets', 'dtype', 'default_value' + ) + ) +): """See `categorical_column_with_vocabulary_file`.""" @property @@ -4016,8 +4302,9 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.parse_example_spec @@ -4025,13 +4312,16 @@ def _transform_input_tensor(self, input_tensor): """Creates a lookup table for the vocabulary.""" if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype)) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype + ) + ) fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key)) + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key) + ) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -4041,23 +4331,28 @@ def _transform_input_tensor(self, input_tensor): # TODO(rohanj): Use state manager to manage the index table creation. return lookup_ops.index_table_from_file( - vocabulary_file=self.vocabulary_file, - num_oov_buckets=self.num_oov_buckets, - vocab_size=self.vocabulary_size, - default_value=self.default_value, - key_dtype=key_dtype, - name='{}_lookup'.format(self.key)).lookup(input_tensor) + vocabulary_file=self.vocabulary_file, + num_oov_buckets=self.num_oov_buckets, + vocab_size=self.vocabulary_size, + default_value=self.default_value, + key_dtype=key_dtype, + name='{}_lookup'.format(self.key) + ).lookup(input_tensor) def transform_feature(self, transformation_cache, state_manager): """Creates a lookup table for the vocabulary.""" input_tensor = _to_sparse_input_and_drop_ignore_values( - transformation_cache.get(self.key, state_manager)) + transformation_cache.get(self.key, state_manager) + ) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + inputs.get(self.key) + ) return self._transform_input_tensor(input_tensor) @property @@ -4066,22 +4361,24 @@ def num_buckets(self): return self.vocabulary_size + self.num_oov_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" return CategoricalColumn.IdWeightPair( - transformation_cache.get(self, state_manager), None) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + transformation_cache.get(self, state_manager), None + ) + + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -4107,11 +4404,15 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class VocabularyListCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('VocabularyListCategoricalColumn', - ('feature_name', 'key', 'vocabulary_list', 'dtype', - 'default_value', 'num_oov_buckets'))): + CategoricalColumn, + fc_old._CategoricalColumn, # pylint: disable=protected-access + collections.namedtuple( + 'VocabularyListCategoricalColumn', ( + 'feature_name', 'key', 'vocabulary_list', 'dtype', 'default_value', + 'num_oov_buckets' + ) + ) +): """See `categorical_column_with_vocabulary_list`.""" @property @@ -4134,8 +4435,9 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.parse_example_spec @@ -4143,13 +4445,16 @@ def _transform_input_tensor(self, input_tensor): """Creates a lookup table for the vocabulary list.""" if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype)) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype + ) + ) fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key)) + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key) + ) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -4159,22 +4464,27 @@ def _transform_input_tensor(self, input_tensor): # TODO(rohanj): Use state manager to manage the index table creation. return lookup_ops.index_table_from_tensor( - vocabulary_list=tuple(self.vocabulary_list), - default_value=self.default_value, - num_oov_buckets=self.num_oov_buckets, - dtype=key_dtype, - name='{}_lookup'.format(self.key)).lookup(input_tensor) + vocabulary_list=tuple(self.vocabulary_list), + default_value=self.default_value, + num_oov_buckets=self.num_oov_buckets, + dtype=key_dtype, + name='{}_lookup'.format(self.key) + ).lookup(input_tensor) def transform_feature(self, transformation_cache, state_manager): """Creates a lookup table for the vocabulary list.""" input_tensor = _to_sparse_input_and_drop_ignore_values( - transformation_cache.get(self.key, state_manager)) + transformation_cache.get(self.key, state_manager) + ) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + inputs.get(self.key) + ) return self._transform_input_tensor(input_tensor) @property @@ -4183,22 +4493,24 @@ def num_buckets(self): return len(self.vocabulary_list) + self.num_oov_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" return CategoricalColumn.IdWeightPair( - transformation_cache.get(self, state_manager), None) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + transformation_cache.get(self, state_manager), None + ) + + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -4224,11 +4536,13 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class IdentityCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple( - 'IdentityCategoricalColumn', - ('feature_name', 'key', 'number_buckets', 'default_value'))): + CategoricalColumn, + fc_old._CategoricalColumn, # pylint: disable=protected-access + collections.namedtuple( + 'IdentityCategoricalColumn', + ('feature_name', 'key', 'number_buckets', 'default_value') + ) +): """See `categorical_column_with_identity`.""" @property @@ -4251,16 +4565,20 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(dtypes.int64)} @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.parse_example_spec def _transform_input_tensor(self, input_tensor): """Returns a SparseTensor with identity values.""" if not input_tensor.dtype.is_integer: - raise ValueError('Invalid input, not integer. key: {} dtype: {}'.format( - self.key, input_tensor.dtype)) + raise ValueError( + 'Invalid input, not integer. key: {} dtype: {}'.format( + self.key, input_tensor.dtype + ) + ) if 'RaggedTensor' in str(type(input_tensor)): return input_tensor @@ -4268,44 +4586,55 @@ def _transform_input_tensor(self, input_tensor): values = math_ops.cast(input_tensor.values, dtypes.int64, name='values') if self.num_buckets < sys.maxsize: num_buckets = math_ops.cast( - self.num_buckets, dtypes.int64, name='num_buckets') + self.num_buckets, dtypes.int64, name='num_buckets' + ) zero = math_ops.cast(0, dtypes.int64, name='zero') if self.default_value is None: # Fail if values are out-of-range. assert_less = check_ops.assert_less( - values, - num_buckets, - data=(values, num_buckets), - name='assert_less_than_num_buckets') + values, + num_buckets, + data=(values, num_buckets), + name='assert_less_than_num_buckets' + ) assert_greater = check_ops.assert_greater_equal( - values, zero, data=(values,), name='assert_greater_or_equal_0') + values, zero, data=(values, ), name='assert_greater_or_equal_0' + ) with ops.control_dependencies((assert_less, assert_greater)): values = array_ops.identity(values) else: # Assign default for out-of-range values. values = array_ops.where( - math_ops.logical_or( - values < zero, values >= num_buckets, name='out_of_range'), - array_ops.fill( - dims=array_ops.shape(values), - value=math_ops.cast(self.default_value, dtypes.int64), - name='default_values'), values) + math_ops.logical_or( + values < zero, values >= num_buckets, name='out_of_range' + ), + array_ops.fill( + dims=array_ops.shape(values), + value=math_ops.cast(self.default_value, dtypes.int64), + name='default_values' + ), values + ) return sparse_tensor_lib.SparseTensor( - indices=input_tensor.indices, - values=values, - dense_shape=input_tensor.dense_shape) + indices=input_tensor.indices, + values=values, + dense_shape=input_tensor.dense_shape + ) def transform_feature(self, transformation_cache, state_manager): """Returns a SparseTensor with identity values.""" input_tensor = _to_sparse_input_and_drop_ignore_values( - transformation_cache.get(self.key, state_manager)) + transformation_cache.get(self.key, state_manager) + ) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + inputs.get(self.key) + ) return self._transform_input_tensor(input_tensor) @property @@ -4314,22 +4643,24 @@ def num_buckets(self): return self.number_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" return CategoricalColumn.IdWeightPair( - transformation_cache.get(self, state_manager), None) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + transformation_cache.get(self, state_manager), None + ) + + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -4351,23 +4682,28 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class WeightedCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple( - 'WeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype'))): + CategoricalColumn, + fc_old._CategoricalColumn, # pylint: disable=protected-access + collections.namedtuple( + 'WeightedCategoricalColumn', + ('categorical_column', 'weight_feature_key', 'dtype') + ) +): """See `weighted_categorical_column`.""" @property def _is_v2_column(self): - return (isinstance(self.categorical_column, FeatureColumn) and - self.categorical_column._is_v2_column) # pylint: disable=protected-access + return ( + isinstance(self.categorical_column, FeatureColumn) + and self.categorical_column._is_v2_column + ) # pylint: disable=protected-access @property def name(self): """See `FeatureColumn` base class.""" - return '{}_weighted_by_{}'.format(self.categorical_column.name, - self.weight_feature_key) + return '{}_weighted_by_{}'.format( + self.categorical_column.name, self.weight_feature_key + ) @property def raw_name(self): @@ -4379,19 +4715,26 @@ def parse_example_spec(self): """See `FeatureColumn` base class.""" config = self.categorical_column.parse_example_spec if self.weight_feature_key in config: - raise ValueError('Parse config {} already exists for {}.'.format( - config[self.weight_feature_key], self.weight_feature_key)) + raise ValueError( + 'Parse config {} already exists for {}.'.format( + config[self.weight_feature_key], self.weight_feature_key + ) + ) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): config = self.categorical_column._parse_example_spec # pylint: disable=protected-access if self.weight_feature_key in config: - raise ValueError('Parse config {} already exists for {}.'.format( - config[self.weight_feature_key], self.weight_feature_key)) + raise ValueError( + 'Parse config {} already exists for {}.'.format( + config[self.weight_feature_key], self.weight_feature_key + ) + ) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @@ -4401,8 +4744,9 @@ def num_buckets(self): return self.categorical_column.num_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _num_buckets(self): return self.categorical_column._num_buckets # pylint: disable=protected-access @@ -4410,28 +4754,37 @@ def _transform_weight_tensor(self, weight_tensor): if weight_tensor is None: raise ValueError('Missing weights {}.'.format(self.weight_feature_key)) weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - weight_tensor) + weight_tensor + ) if self.dtype != weight_tensor.dtype.base_dtype: - raise ValueError('Bad dtype, expected {}, but got {}.'.format( - self.dtype, weight_tensor.dtype)) + raise ValueError( + 'Bad dtype, expected {}, but got {}.'.format( + self.dtype, weight_tensor.dtype + ) + ) if not isinstance(weight_tensor, sparse_tensor_lib.SparseTensor): # The weight tensor can be a regular Tensor. In this case, sparsify it. weight_tensor = _to_sparse_input_and_drop_ignore_values( - weight_tensor, ignore_value=0.0) + weight_tensor, ignore_value=0.0 + ) if not weight_tensor.dtype.is_floating: weight_tensor = math_ops.cast(weight_tensor, dtypes.float32) return weight_tensor def transform_feature(self, transformation_cache, state_manager): """Applies weights to tensor generated from `categorical_column`'.""" - weight_tensor = transformation_cache.get(self.weight_feature_key, - state_manager) + weight_tensor = transformation_cache.get( + self.weight_feature_key, state_manager + ) weight_tensor = self._transform_weight_tensor(weight_tensor) - return (transformation_cache.get(self.categorical_column, - state_manager), weight_tensor) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + return ( + transformation_cache.get(self.categorical_column, + state_manager), weight_tensor + ) + + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): """Applies weights to tensor generated from `categorical_column`'.""" weight_tensor = inputs.get(self.weight_feature_key) @@ -4443,12 +4796,12 @@ def get_sparse_tensors(self, transformation_cache, state_manager): tensors = transformation_cache.get(self, state_manager) return CategoricalColumn.IdWeightPair(tensors[0], tensors[1]) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): del weight_collections del trainable tensors = inputs.get(self) @@ -4463,7 +4816,8 @@ def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) config['categorical_column'] = serialize_feature_column( - self.categorical_column) + self.categorical_column + ) config['dtype'] = self.dtype.name return config @@ -4473,17 +4827,19 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['categorical_column'] = deserialize_feature_column( - config['categorical_column'], custom_objects, columns_by_name) + config['categorical_column'], custom_objects, columns_by_name + ) kwargs['dtype'] = dtypes.as_dtype(config['dtype']) return cls(**kwargs) class CrossedColumn( - CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple( - 'CrossedColumn', - ('feature_name', 'keys', 'hash_bucket_size', 'hash_key'))): + CategoricalColumn, + fc_old._CategoricalColumn, # pylint: disable=protected-access + collections.namedtuple( + 'CrossedColumn', ('feature_name', 'keys', 'hash_bucket_size', 'hash_key') + ) +): """See `crossed_column`.""" @property @@ -4524,8 +4880,9 @@ def parse_example_spec(self): return config @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.parse_example_spec @@ -4536,23 +4893,27 @@ def transform_feature(self, transformation_cache, state_manager): if isinstance(key, six.string_types): feature_tensors.append(transformation_cache.get(key, state_manager)) elif isinstance(key, (fc_old._CategoricalColumn, CategoricalColumn)): # pylint: disable=protected-access - ids_and_weights = key.get_sparse_tensors(transformation_cache, - state_manager) + ids_and_weights = key.get_sparse_tensors( + transformation_cache, state_manager + ) if ids_and_weights.weight_tensor is not None: raise ValueError( - 'crossed_column does not support weight_tensor, but the given ' - 'column populates weight_tensor. ' - 'Given column: {}'.format(key.name)) + 'crossed_column does not support weight_tensor, but the given ' + 'column populates weight_tensor. ' + 'Given column: {}'.format(key.name) + ) feature_tensors.append(ids_and_weights.id_tensor) else: raise ValueError('Unsupported column type. Given: {}'.format(key)) return sparse_ops.sparse_cross_hashed( - inputs=feature_tensors, - num_buckets=self.hash_bucket_size, - hash_key=self.hash_key) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + inputs=feature_tensors, + num_buckets=self.hash_bucket_size, + hash_key=self.hash_key + ) + + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): """Generates a hashed sparse cross from the input tensors.""" feature_tensors = [] @@ -4563,16 +4924,18 @@ def _transform_feature(self, inputs): ids_and_weights = key._get_sparse_tensors(inputs) # pylint: disable=protected-access if ids_and_weights.weight_tensor is not None: raise ValueError( - 'crossed_column does not support weight_tensor, but the given ' - 'column populates weight_tensor. ' - 'Given column: {}'.format(key.name)) + 'crossed_column does not support weight_tensor, but the given ' + 'column populates weight_tensor. ' + 'Given column: {}'.format(key.name) + ) feature_tensors.append(ids_and_weights.id_tensor) else: raise ValueError('Unsupported column type. Given: {}'.format(key)) return sparse_ops.sparse_cross_hashed( - inputs=feature_tensors, - num_buckets=self.hash_bucket_size, - hash_key=self.hash_key) + inputs=feature_tensors, + num_buckets=self.hash_bucket_size, + hash_key=self.hash_key + ) @property def num_buckets(self): @@ -4580,22 +4943,24 @@ def num_buckets(self): return self.hash_bucket_size @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" return CategoricalColumn.IdWeightPair( - transformation_cache.get(self, state_manager), None) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + transformation_cache.get(self, state_manager), None + ) + + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): """See `CategoricalColumn` base class.""" del weight_collections del trainable @@ -4617,10 +4982,12 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): """See 'FeatureColumn` base class.""" _check_config_keys(config, cls._fields) kwargs = config.copy() - kwargs['keys'] = tuple([ + kwargs['keys'] = tuple( + [ deserialize_feature_column(c, custom_objects, columns_by_name) for c in config['keys'] - ]) + ] + ) return cls(**kwargs) @@ -4647,8 +5014,9 @@ def _prune_invalid_ids(sparse_ids, sparse_weights): is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) if sparse_weights is not None: is_id_valid = math_ops.logical_and( - is_id_valid, - array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) + is_id_valid, + array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool) + ) sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) if sparse_weights is not None: sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) @@ -4665,11 +5033,12 @@ def _prune_invalid_weights(sparse_ids, sparse_weights): class IndicatorColumn( - DenseColumn, - SequenceDenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._SequenceDenseColumn, # pylint: disable=protected-access - collections.namedtuple('IndicatorColumn', ('categorical_column'))): + DenseColumn, + SequenceDenseColumn, + fc_old._DenseColumn, # pylint: disable=protected-access + fc_old._SequenceDenseColumn, # pylint: disable=protected-access + collections.namedtuple('IndicatorColumn', ('categorical_column')) +): """Represents a one-hot column for use in deep networks. Args: @@ -4679,8 +5048,10 @@ class IndicatorColumn( @property def _is_v2_column(self): - return (isinstance(self.categorical_column, FeatureColumn) and - self.categorical_column._is_v2_column) # pylint: disable=protected-access + return ( + isinstance(self.categorical_column, FeatureColumn) + and self.categorical_column._is_v2_column + ) # pylint: disable=protected-access @property def name(self): @@ -4699,28 +5070,33 @@ def _transform_id_weight_pair(self, id_weight_pair): # If the underlying column is weighted, return the input as a dense tensor. if weight_tensor is not None: weighted_column = sparse_ops.sparse_merge( - sp_ids=id_tensor, - sp_values=weight_tensor, - vocab_size=int(self._variable_shape[-1])) + sp_ids=id_tensor, + sp_values=weight_tensor, + vocab_size=int(self._variable_shape[-1]) + ) # Remove (?, -1) index. - weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0], - weighted_column.dense_shape) + weighted_column = sparse_ops.sparse_slice( + weighted_column, [0, 0], weighted_column.dense_shape + ) # Use scatter_nd to merge duplicated indices if existed, # instead of sparse_tensor_to_dense. - return array_ops.scatter_nd(weighted_column.indices, - weighted_column.values, - weighted_column.dense_shape) + return array_ops.scatter_nd( + weighted_column.indices, weighted_column.values, + weighted_column.dense_shape + ) dense_id_tensor = sparse_ops.sparse_tensor_to_dense( - id_tensor, default_value=-1) + id_tensor, default_value=-1 + ) # One hot must be float for tf.concat reasons since all other inputs to # input_layer are float32. one_hot_id_tensor = array_ops.one_hot( - dense_id_tensor, - depth=self._variable_shape[-1], - on_value=1.0, - off_value=0.0) + dense_id_tensor, + depth=self._variable_shape[-1], + on_value=1.0, + off_value=0.0 + ) # Reduce to get a multi-hot per example. return math_ops.reduce_sum(one_hot_id_tensor, axis=[-2]) @@ -4741,11 +5117,13 @@ def transform_feature(self, transformation_cache, state_manager): ValueError: if input rank is not known at graph building time. """ id_weight_pair = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager) + transformation_cache, state_manager + ) return self._transform_id_weight_pair(id_weight_pair) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): id_weight_pair = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access return self._transform_id_weight_pair(id_weight_pair) @@ -4756,8 +5134,9 @@ def parse_example_spec(self): return self.categorical_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.categorical_column._parse_example_spec # pylint: disable=protected-access @@ -4767,11 +5146,14 @@ def variable_shape(self): if isinstance(self.categorical_column, FeatureColumn): return tensor_shape.TensorShape([1, self.categorical_column.num_buckets]) else: - return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) # pylint: disable=protected-access + return tensor_shape.TensorShape( + [1, self.categorical_column._num_buckets] + ) # pylint: disable=protected-access @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _variable_shape(self): return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) # pylint: disable=protected-access @@ -4792,35 +5174,41 @@ def get_dense_tensor(self, transformation_cache, state_manager): """ if isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must not be of type SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use DenseFeatures, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In indicator_column: {}. ' + 'categorical_column must not be of type SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use DenseFeatures, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'SequenceFeatures instead of DenseFeatures. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) # Feature has been already transformed. Return the intermediate # representation created by transform_feature. return transformation_cache.get(self, state_manager) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable if isinstance( - self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn)): # pylint: disable=protected-access + self.categorical_column, + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn) + ): # pylint: disable=protected-access raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must not be of type _SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use DenseFeatures, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In indicator_column: {}. ' + 'categorical_column must not be of type _SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use DenseFeatures, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'SequenceFeatures instead of DenseFeatures. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) # Feature has been already transformed. Return the intermediate # representation created by transform_feature. return inputs.get(self) @@ -4829,50 +5217,60 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): """See `SequenceDenseColumn` base class.""" if not isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must be of type SequenceCategoricalColumn ' - 'to use SequenceFeatures. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In indicator_column: {}. ' + 'categorical_column must be of type SequenceCategoricalColumn ' + 'to use SequenceFeatures. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) # Feature has been already transformed. Return the intermediate # representation created by transform_feature. dense_tensor = transformation_cache.get(self, state_manager) sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager) + transformation_cache, state_manager + ) sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor) + sparse_tensors.id_tensor + ) return SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sequence_dense_tensor(self, - inputs, - weight_collections=None, - trainable=None): + dense_tensor=dense_tensor, sequence_length=sequence_length + ) + + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sequence_dense_tensor( + self, inputs, weight_collections=None, trainable=None + ): # Do nothing with weight_collections and trainable since no variables are # created in this function. del weight_collections del trainable if not isinstance( - self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn)): # pylint: disable=protected-access + self.categorical_column, + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn) + ): # pylint: disable=protected-access raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must be of type _SequenceCategoricalColumn ' - 'to use SequenceFeatures. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), - self.categorical_column)) + 'In indicator_column: {}. ' + 'categorical_column must be of type _SequenceCategoricalColumn ' + 'to use SequenceFeatures. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format( + self.name, type(self.categorical_column), self.categorical_column + ) + ) # Feature has been already transformed. Return the intermediate # representation created by _transform_feature. dense_tensor = inputs.get(self) sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor) + sparse_tensors.id_tensor + ) return SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length) + dense_tensor=dense_tensor, sequence_length=sequence_length + ) @property def parents(self): @@ -4883,7 +5281,8 @@ def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) config['categorical_column'] = serialize_feature_column( - self.categorical_column) + self.categorical_column + ) return config @classmethod @@ -4892,7 +5291,8 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['categorical_column'] = deserialize_feature_column( - config['categorical_column'], custom_objects, columns_by_name) + config['categorical_column'], custom_objects, columns_by_name + ) return cls(**kwargs) @@ -4910,30 +5310,35 @@ def _verify_static_batch_size_equality(tensors, columns): expected_batch_size = None for i in range(0, len(tensors)): batch_size = tensor_shape.Dimension( - tensor_shape.dimension_value(tensors[i].shape[0])) + tensor_shape.dimension_value(tensors[i].shape[0]) + ) if batch_size.value is not None: if expected_batch_size is None: bath_size_column_index = i expected_batch_size = batch_size elif not expected_batch_size.is_compatible_with(batch_size): raise ValueError( - 'Batch size (first dimension) of each feature must be same. ' - 'Batch size of columns ({}, {}): ({}, {})'.format( - columns[bath_size_column_index].name, columns[i].name, - expected_batch_size, batch_size)) + 'Batch size (first dimension) of each feature must be same. ' + 'Batch size of columns ({}, {}): ({}, {})'.format( + columns[bath_size_column_index].name, columns[i].name, + expected_batch_size, batch_size + ) + ) class SequenceCategoricalColumn( - CategoricalColumn, - fc_old._SequenceCategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('SequenceCategoricalColumn', - ('categorical_column'))): + CategoricalColumn, + fc_old._SequenceCategoricalColumn, # pylint: disable=protected-access + collections.namedtuple('SequenceCategoricalColumn', ('categorical_column')) +): """Represents sequences of categorical data.""" @property def _is_v2_column(self): - return (isinstance(self.categorical_column, FeatureColumn) and - self.categorical_column._is_v2_column) # pylint: disable=protected-access + return ( + isinstance(self.categorical_column, FeatureColumn) + and self.categorical_column._is_v2_column + ) # pylint: disable=protected-access @property def name(self): @@ -4951,22 +5356,25 @@ def parse_example_spec(self): return self.categorical_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _parse_example_spec(self): return self.categorical_column._parse_example_spec # pylint: disable=protected-access def transform_feature(self, transformation_cache, state_manager): """See `FeatureColumn` base class.""" ret_tensor = self.categorical_column.transform_feature( - transformation_cache, state_manager) + transformation_cache, state_manager + ) shape = array_ops.shape(ret_tensor) target_shape = [shape[0], shape[1], math_ops.reduce_prod(shape[2:])] ret_tensor = sparse_ops.sparse_reshape(ret_tensor, target_shape) return ret_tensor - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _transform_feature(self, inputs): ret_tensor = self.categorical_column._transform_feature(inputs) shape = array_ops.shape(ret_tensor) @@ -4980,8 +5388,9 @@ def num_buckets(self): return self.categorical_column.num_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) def _num_buckets(self): return self.categorical_column._num_buckets # pylint: disable=protected-access @@ -5021,15 +5430,16 @@ def get_sparse_tensors(self, transformation_cache, state_manager): lookup tables. """ sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager) + transformation_cache, state_manager + ) return self._get_sparse_tensors_helper(sparse_tensors) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, - _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, - inputs, - weight_collections=None, - trainable=None): + @deprecation.deprecated( + _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION + ) + def _get_sparse_tensors( + self, inputs, weight_collections=None, trainable=None + ): sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access return self._get_sparse_tensors_helper(sparse_tensors) @@ -5042,7 +5452,8 @@ def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) config['categorical_column'] = serialize_feature_column( - self.categorical_column) + self.categorical_column + ) return config @classmethod @@ -5051,7 +5462,8 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['categorical_column'] = deserialize_feature_column( - config['categorical_column'], custom_objects, columns_by_name) + config['categorical_column'], custom_objects, columns_by_name + ) return cls(**kwargs) @@ -5061,8 +5473,9 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): def _check_config_keys(config, expected_keys): """Checks that a config has all expected_keys.""" if set(config.keys()) != set(expected_keys): - raise ValueError('Invalid config: {}, expected keys: {}'.format( - config, expected_keys)) + raise ValueError( + 'Invalid config: {}, expected keys: {}'.format(config, expected_keys) + ) def serialize_feature_column(fc): @@ -5108,15 +5521,16 @@ def serialize_feature_column(fc): if isinstance(fc, six.string_types): return fc elif isinstance(fc, FeatureColumn): - return utils.serialize_keras_class_and_config(fc.__class__.__name__, - fc._get_config()) + return utils.serialize_keras_class_and_config( + fc.__class__.__name__, fc._get_config() + ) else: raise ValueError('Instance: {} is not a FeatureColumn'.format(fc)) -def deserialize_feature_column(config, - custom_objects=None, - columns_by_name=None): +def deserialize_feature_column( + config, custom_objects=None, columns_by_name=None +): """Deserializes a `config` generated with `serialize_feature_column`. This method should only be used to deserialize parent FeatureColumns when @@ -5144,26 +5558,29 @@ def deserialize_feature_column(config, # A dict from class_name to class for all FeatureColumns in this module. # FeatureColumns not part of the module can be passed as custom_objects. module_feature_column_classes = { - cls.__name__: cls for cls in [ - BucketizedColumn, EmbeddingColumn, HashedCategoricalColumn, - IdentityCategoricalColumn, IndicatorColumn, NumericColumn, - SequenceCategoricalColumn, SequenceDenseColumn, SharedEmbeddingColumn, - VocabularyFileCategoricalColumn, VocabularyListCategoricalColumn, - WeightedCategoricalColumn, init_ops.TruncatedNormal - ] + cls.__name__: cls + for cls in [ + BucketizedColumn, EmbeddingColumn, HashedCategoricalColumn, + IdentityCategoricalColumn, IndicatorColumn, NumericColumn, + SequenceCategoricalColumn, SequenceDenseColumn, SharedEmbeddingColumn, + VocabularyFileCategoricalColumn, VocabularyListCategoricalColumn, + WeightedCategoricalColumn, init_ops.TruncatedNormal + ] } if columns_by_name is None: columns_by_name = {} (cls, cls_config) = utils.class_and_config_for_serialized_keras_object( - config, - module_objects=module_feature_column_classes, - custom_objects=custom_objects, - printable_module_name='feature_column_v2') + config, + module_objects=module_feature_column_classes, + custom_objects=custom_objects, + printable_module_name='feature_column_v2' + ) if not issubclass(cls, FeatureColumn): raise ValueError( - 'Expected FeatureColumn class, instead found: {}'.format(cls)) + 'Expected FeatureColumn class, instead found: {}'.format(cls) + ) # Always deserialize the FeatureColumn, in order to get the name. new_instance = cls._from_config( # pylint: disable=protected-access @@ -5215,8 +5632,8 @@ def deserialize_feature_columns(configs, custom_objects=None): """ columns_by_name = {} return [ - deserialize_feature_column(c, custom_objects, columns_by_name) - for c in configs + deserialize_feature_column(c, custom_objects, columns_by_name) + for c in configs ] diff --git a/easy_rec/python/compat/feature_column/sequence_feature_column.py b/easy_rec/python/compat/feature_column/sequence_feature_column.py index b0fcdc9f7..5bf97107f 100644 --- a/easy_rec/python/compat/feature_column/sequence_feature_column.py +++ b/easy_rec/python/compat/feature_column/sequence_feature_column.py @@ -18,20 +18,11 @@ NOTE: This API is a work in progress and will likely be changing frequently. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import collections - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import sparse_ops +from tensorflow.python.framework import dtypes, ops, tensor_shape +from tensorflow.python.ops import array_ops, check_ops, math_ops, parsing_ops, sparse_ops # NOQA from easy_rec.python.compat.feature_column import feature_column as fc_v1 from easy_rec.python.compat.feature_column import feature_column_v2 as fc @@ -92,11 +83,12 @@ def __init__(self, feature_columns, trainable=True, name=None, **kwargs): `SequenceDenseColumn`. """ super(SequenceFeatures, self).__init__( - feature_columns=feature_columns, - trainable=trainable, - name=name, - expected_column_type=fc.SequenceDenseColumn, - **kwargs) + feature_columns=feature_columns, + trainable=trainable, + name=name, + expected_column_type=fc.SequenceDenseColumn, + **kwargs + ) def _target_shape(self, input_shape, total_elements): return (input_shape[0], input_shape[1], total_elements) @@ -120,8 +112,9 @@ def call(self, features): ValueError: If features are not a dictionary. """ if not isinstance(features, dict): - raise ValueError('We expected a dictionary here. Instead we got: ', - features) + raise ValueError( + 'We expected a dictionary here. Instead we got: ', features + ) transformation_cache = fc.FeatureTransformationCache(features) output_tensors = [] sequence_lengths = [] @@ -129,14 +122,16 @@ def call(self, features): for column in self._feature_columns: with ops.name_scope(column.name): dense_tensor, sequence_length = column.get_sequence_dense_tensor( - transformation_cache, self._state_manager) + transformation_cache, self._state_manager + ) # Flattens the final dimension to produce a 3D Tensor. output_tensors.append(self._process_dense_tensor(column, dense_tensor)) sequence_lengths.append(sequence_length) # Check and process sequence lengths. - fc._verify_static_batch_size_equality(sequence_lengths, - self._feature_columns) + fc._verify_static_batch_size_equality( + sequence_lengths, self._feature_columns + ) sequence_length = _assert_all_equal_and_return(sequence_lengths) return self._verify_and_concat_tensors(output_tensors), sequence_length @@ -163,38 +158,45 @@ def concatenate_context_input(context_input, sequence_input): not have rank 2. """ seq_rank_check = check_ops.assert_rank( - sequence_input, - 3, - message='sequence_input must have rank 3', - data=[array_ops.shape(sequence_input)]) + sequence_input, + 3, + message='sequence_input must have rank 3', + data=[array_ops.shape(sequence_input)] + ) seq_type_check = check_ops.assert_type( - sequence_input, - dtypes.float32, - message='sequence_input must have dtype float32; got {}.'.format( - sequence_input.dtype)) + sequence_input, + dtypes.float32, + message='sequence_input must have dtype float32; got {}.'.format( + sequence_input.dtype + ) + ) ctx_rank_check = check_ops.assert_rank( - context_input, - 2, - message='context_input must have rank 2', - data=[array_ops.shape(context_input)]) + context_input, + 2, + message='context_input must have rank 2', + data=[array_ops.shape(context_input)] + ) ctx_type_check = check_ops.assert_type( - context_input, - dtypes.float32, - message='context_input must have dtype float32; got {}.'.format( - context_input.dtype)) + context_input, + dtypes.float32, + message='context_input must have dtype float32; got {}.'.format( + context_input.dtype + ) + ) with ops.control_dependencies( - [seq_rank_check, seq_type_check, ctx_rank_check, ctx_type_check]): + [seq_rank_check, seq_type_check, ctx_rank_check, ctx_type_check] + ): padded_length = array_ops.shape(sequence_input)[1] tiled_context_input = array_ops.tile( - array_ops.expand_dims(context_input, 1), - array_ops.concat([[1], [padded_length], [1]], 0)) + array_ops.expand_dims(context_input, 1), + array_ops.concat([[1], [padded_length], [1]], 0) + ) return array_ops.concat([sequence_input, tiled_context_input], 2) -def sequence_categorical_column_with_identity(key, - num_buckets, - default_value=None, - feature_name=None): +def sequence_categorical_column_with_identity( + key, num_buckets, default_value=None, feature_name=None +): """Returns a feature column that represents sequences of integers. Pass this to `embedding_column` or `indicator_column` to convert sequence @@ -235,21 +237,26 @@ def sequence_categorical_column_with_identity(key, ValueError: if `default_value` is not in range `[0, num_buckets)`. """ return fc.SequenceCategoricalColumn( - fc.categorical_column_with_identity( - feature_name=feature_name, - key=key, - num_buckets=num_buckets, - default_value=default_value)) + fc.categorical_column_with_identity( + feature_name=feature_name, + key=key, + num_buckets=num_buckets, + default_value=default_value + ) + ) def sequence_numeric_column_with_bucketized_column(source_column, boundaries): - if not isinstance(source_column, (SequenceNumericColumn,)): # pylint: disable=protected-access + if not isinstance(source_column, (SequenceNumericColumn, )): # pylint: disable=protected-access raise ValueError( - 'source_column must be a column generated with sequence_numeric_column(). ' - 'Given: {}'.format(source_column)) + 'source_column must be a column generated with sequence_numeric_column(). ' + 'Given: {}'.format(source_column) + ) if len(source_column.shape) > 1: - raise ValueError('source_column must be one-dimensional column. ' - 'Given: {}'.format(source_column)) + raise ValueError( + 'source_column must be one-dimensional column. ' + 'Given: {}'.format(source_column) + ) if not boundaries: raise ValueError('boundaries must not be empty.') if not (isinstance(boundaries, list) or isinstance(boundaries, tuple)): @@ -261,32 +268,35 @@ def sequence_numeric_column_with_bucketized_column(source_column, boundaries): def sequence_numeric_column_with_raw_column(source_column, sequence_length): - if not isinstance(source_column, (SequenceNumericColumn,)): # pylint: disable=protected-access + if not isinstance(source_column, (SequenceNumericColumn, )): # pylint: disable=protected-access raise ValueError( - 'source_column must be a column generated with sequence_numeric_column(). ' - 'Given: {}'.format(source_column)) + 'source_column must be a column generated with sequence_numeric_column(). ' + 'Given: {}'.format(source_column) + ) if len(source_column.shape) > 1: - raise ValueError('source_column must be one-dimensional column. ' - 'Given: {}'.format(source_column)) + raise ValueError( + 'source_column must be one-dimensional column. ' + 'Given: {}'.format(source_column) + ) return fc.SequenceNumericColumn(source_column, sequence_length) -def sequence_weighted_categorical_column(categorical_column, - weight_feature_key, - dtype=dtypes.float32): +def sequence_weighted_categorical_column( + categorical_column, weight_feature_key, dtype=dtypes.float32 +): if (dtype is None) or not (dtype.is_integer or dtype.is_floating): raise ValueError('dtype {} is not convertible to float.'.format(dtype)) return fc.SequenceWeightedCategoricalColumn( - categorical_column=categorical_column, - weight_feature_key=weight_feature_key, - dtype=dtype) + categorical_column=categorical_column, + weight_feature_key=weight_feature_key, + dtype=dtype + ) -def sequence_categorical_column_with_hash_bucket(key, - hash_bucket_size, - dtype=dtypes.string, - feature_name=None): +def sequence_categorical_column_with_hash_bucket( + key, hash_bucket_size, dtype=dtypes.string, feature_name=None +): """A sequence of categorical terms where ids are set by hashing. Pass this to `embedding_column` or `indicator_column` to convert sequence @@ -324,20 +334,24 @@ def sequence_categorical_column_with_hash_bucket(key, ValueError: `dtype` is neither string nor integer. """ return fc.SequenceCategoricalColumn( - fc.categorical_column_with_hash_bucket( - feature_name=feature_name, - key=key, - hash_bucket_size=hash_bucket_size, - dtype=dtype)) - - -def sequence_categorical_column_with_vocabulary_file(key, - vocabulary_file, - vocabulary_size=None, - num_oov_buckets=0, - default_value=None, - dtype=dtypes.string, - feature_name=None): + fc.categorical_column_with_hash_bucket( + feature_name=feature_name, + key=key, + hash_bucket_size=hash_bucket_size, + dtype=dtype + ) + ) + + +def sequence_categorical_column_with_vocabulary_file( + key, + vocabulary_file, + vocabulary_size=None, + num_oov_buckets=0, + default_value=None, + dtype=dtypes.string, + feature_name=None +): """A sequence of categorical terms where ids use a vocabulary file. Pass this to `embedding_column` or `indicator_column` to convert sequence @@ -390,22 +404,26 @@ def sequence_categorical_column_with_vocabulary_file(key, ValueError: `dtype` is neither string nor integer. """ return fc.SequenceCategoricalColumn( - fc.categorical_column_with_vocabulary_file( - feature_name=feature_name, - key=key, - vocabulary_file=vocabulary_file, - vocabulary_size=vocabulary_size, - num_oov_buckets=num_oov_buckets, - default_value=default_value, - dtype=dtype)) - - -def sequence_categorical_column_with_vocabulary_list(key, - vocabulary_list, - dtype=None, - default_value=-1, - num_oov_buckets=0, - feature_name=None): + fc.categorical_column_with_vocabulary_file( + feature_name=feature_name, + key=key, + vocabulary_file=vocabulary_file, + vocabulary_size=vocabulary_size, + num_oov_buckets=num_oov_buckets, + default_value=default_value, + dtype=dtype + ) + ) + + +def sequence_categorical_column_with_vocabulary_list( + key, + vocabulary_list, + dtype=None, + default_value=-1, + num_oov_buckets=0, + feature_name=None +): """A sequence of categorical terms where ids use an in-memory list. Pass this to `embedding_column` or `indicator_column` to convert sequence @@ -457,21 +475,25 @@ def sequence_categorical_column_with_vocabulary_list(key, ValueError: if `dtype` is not integer or string. """ return fc.SequenceCategoricalColumn( - fc.categorical_column_with_vocabulary_list( - feature_name=feature_name, - key=key, - vocabulary_list=vocabulary_list, - dtype=dtype, - default_value=default_value, - num_oov_buckets=num_oov_buckets)) - - -def sequence_numeric_column(key, - shape=(1,), - default_value=0., - dtype=dtypes.float32, - normalizer_fn=None, - feature_name=None): + fc.categorical_column_with_vocabulary_list( + feature_name=feature_name, + key=key, + vocabulary_list=vocabulary_list, + dtype=dtype, + default_value=default_value, + num_oov_buckets=num_oov_buckets + ) + ) + + +def sequence_numeric_column( + key, + shape=(1, ), + default_value=0., + dtype=dtypes.float32, + normalizer_fn=None, + feature_name=None +): """Returns a feature column that represents sequences of numeric data. Example: @@ -514,19 +536,23 @@ def sequence_numeric_column(key, """ shape = fc._check_shape(shape=shape, key=key) if not (dtype.is_integer or dtype.is_floating): - raise ValueError('dtype must be convertible to float. ' - 'dtype: {}, key: {}'.format(dtype, key)) + raise ValueError( + 'dtype must be convertible to float. ' + 'dtype: {}, key: {}'.format(dtype, key) + ) if normalizer_fn is not None and not callable(normalizer_fn): raise TypeError( - 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) + 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn) + ) return SequenceNumericColumn( - feature_name=feature_name, - key=key, - shape=shape, - default_value=default_value, - dtype=dtype, - normalizer_fn=normalizer_fn) + feature_name=feature_name, + key=key, + shape=shape, + default_value=default_value, + dtype=dtype, + normalizer_fn=normalizer_fn + ) def _assert_all_equal_and_return(tensors, name=None): @@ -542,10 +568,13 @@ def _assert_all_equal_and_return(tensors, name=None): class SequenceNumericColumn( - fc.SequenceDenseColumn, fc_v1._FeatureColumn, - collections.namedtuple('SequenceNumericColumn', - ('feature_name', 'key', 'shape', 'default_value', - 'dtype', 'normalizer_fn'))): + fc.SequenceDenseColumn, fc_v1._FeatureColumn, + collections.namedtuple( + 'SequenceNumericColumn', ( + 'feature_name', 'key', 'shape', 'default_value', 'dtype', 'normalizer_fn' + ) + ) +): """Represents sequences of numeric data.""" @property @@ -609,10 +638,12 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): """ sp_tensor = transformation_cache.get(self, state_manager) dense_tensor = sparse_ops.sparse_tensor_to_dense( - sp_tensor, default_value=self.default_value) + sp_tensor, default_value=self.default_value + ) # Reshape into [batch_size, T, variable_shape]. dense_shape = array_ops.concat( - [array_ops.shape(dense_tensor)[:1], [-1], self.variable_shape], axis=0) + [array_ops.shape(dense_tensor)[:1], [-1], self.variable_shape], axis=0 + ) dense_tensor = array_ops.reshape(dense_tensor, shape=dense_shape) # Get the number of timesteps per example @@ -624,10 +655,12 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): else: num_elements = 1 seq_length = fc_utils.sequence_length_from_sparse_tensor( - sp_tensor, num_elements=num_elements) + sp_tensor, num_elements=num_elements + ) return fc.SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=seq_length) + dense_tensor=dense_tensor, sequence_length=seq_length + ) # TODO(b/119409767): Implement parents, _{get,from}_config. @property diff --git a/easy_rec/python/compat/feature_column/utils.py b/easy_rec/python/compat/feature_column/utils.py index 35870e89d..7418d1e66 100644 --- a/easy_rec/python/compat/feature_column/utils.py +++ b/easy_rec/python/compat/feature_column/utils.py @@ -15,15 +15,11 @@ # ============================================================================== """Defines functions common to multiple feature column files.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import six -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops +from tensorflow.python.framework import dtypes, ops +from tensorflow.python.ops import array_ops, math_ops from tensorflow.python.util import nest @@ -43,7 +39,8 @@ def sequence_length_from_sparse_tensor(sp_tensor, num_elements=1): # row_ids = (0, 0, 1), seq_length = [2, 1]. If num_elements = 2, # these will get grouped, and the final seq_length is [1, 1] seq_length = math_ops.cast( - math_ops.ceil(seq_length / num_elements), dtypes.int64) + math_ops.ceil(seq_length / num_elements), dtypes.int64 + ) # If the last n rows do not have ids, seq_length will have shape # [batch_size - n]. Pad the remaining values with zeros. @@ -54,15 +51,18 @@ def sequence_length_from_sparse_tensor(sp_tensor, num_elements=1): def assert_string_or_int(dtype, prefix): if (dtype != dtypes.string) and (not dtype.is_integer): - raise ValueError('{} dtype must be string or integer. dtype: {}.'.format( - prefix, dtype)) + raise ValueError( + '{} dtype must be string or integer. dtype: {}.'.format(prefix, dtype) + ) def assert_key_is_string(key): if not isinstance(key, six.string_types): raise ValueError( - 'key must be a string. Got: type {}. Given key: {}.'.format( - type(key), key)) + 'key must be a string. Got: type {}. Given key: {}.'.format( + type(key), key + ) + ) def check_default_value(shape, default_value, dtype, key): @@ -106,22 +106,27 @@ def check_default_value(shape, default_value, dtype, key): if nest.is_sequence(default_value): if not _is_shape_and_default_value_compatible(default_value, shape): raise ValueError( - 'The shape of default_value must be equal to given shape. ' - 'default_value: {}, shape: {}, key: {}'.format( - default_value, shape, key)) + 'The shape of default_value must be equal to given shape. ' + 'default_value: {}, shape: {}, key: {}'.format( + default_value, shape, key + ) + ) # Check if the values in the list are all integers or are convertible to # floats. is_list_all_int = all( - isinstance(v, int) for v in nest.flatten(default_value)) + isinstance(v, int) for v in nest.flatten(default_value) + ) is_list_has_float = any( - isinstance(v, float) for v in nest.flatten(default_value)) + isinstance(v, float) for v in nest.flatten(default_value) + ) if is_list_all_int: return _as_tuple(default_value) if is_list_has_float and dtype.is_floating: return _as_tuple(default_value) - raise TypeError('default_value must be compatible with dtype. ' - 'default_value: {}, dtype: {}, key: {}'.format( - default_value, dtype, key)) + raise TypeError( + 'default_value must be compatible with dtype. ' + 'default_value: {}, dtype: {}, key: {}'.format(default_value, dtype, key) + ) def _create_tuple(shape, value): diff --git a/easy_rec/python/compat/layers.py b/easy_rec/python/compat/layers.py index 651eefac8..0311f8cf5 100644 --- a/easy_rec/python/compat/layers.py +++ b/easy_rec/python/compat/layers.py @@ -15,30 +15,26 @@ # ============================================================================== """Higher level ops for building layers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import variable_scope - - -def layer_norm(inputs, - center=True, - scale=True, - activation_fn=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - begin_norm_axis=1, - begin_params_axis=-1, - scope=None): +from tensorflow.python.framework import dtypes, ops +from tensorflow.python.ops import init_ops, nn, variable_scope + + +def layer_norm( + inputs, + center=True, + scale=True, + activation_fn=None, + reuse=None, + variables_collections=None, + outputs_collections=None, + trainable=True, + begin_norm_axis=1, + begin_params_axis=-1, + scope=None +): """Adds a Layer Normalization layer. Based on the paper: @@ -98,7 +94,8 @@ def layer_norm(inputs, graph build time. """ with variable_scope.variable_scope( - scope, 'LayerNorm', [inputs], reuse=reuse) as sc: + scope, 'LayerNorm', [inputs], reuse=reuse + ) as sc: inputs = ops.convert_to_tensor(inputs) inputs_shape = inputs.shape inputs_rank = inputs_shape.ndims @@ -108,47 +105,56 @@ def layer_norm(inputs, if begin_norm_axis < 0: begin_norm_axis = inputs_rank + begin_norm_axis if begin_params_axis >= inputs_rank or begin_norm_axis >= inputs_rank: - raise ValueError('begin_params_axis (%d) and begin_norm_axis (%d) ' - 'must be < rank(inputs) (%d)' % - (begin_params_axis, begin_norm_axis, inputs_rank)) + raise ValueError( + 'begin_params_axis (%d) and begin_norm_axis (%d) ' + 'must be < rank(inputs) (%d)' % + (begin_params_axis, begin_norm_axis, inputs_rank) + ) params_shape = inputs_shape[begin_params_axis:] if not params_shape.is_fully_defined(): raise ValueError( - 'Inputs %s: shape(inputs)[%s:] is not fully defined: %s' % - (inputs.name, begin_params_axis, inputs_shape)) + 'Inputs %s: shape(inputs)[%s:] is not fully defined: %s' % + (inputs.name, begin_params_axis, inputs_shape) + ) # Allocate parameters for the beta and gamma of the normalization. beta, gamma = None, None if center: - beta_collections = get_variable_collections(variables_collections, 'beta') + beta_collections = get_variable_collections( + variables_collections, 'beta' + ) beta = model_variable( - 'beta', - shape=params_shape, - dtype=dtype, - initializer=init_ops.zeros_initializer(), - collections=beta_collections, - trainable=trainable) + 'beta', + shape=params_shape, + dtype=dtype, + initializer=init_ops.zeros_initializer(), + collections=beta_collections, + trainable=trainable + ) if scale: - gamma_collections = get_variable_collections(variables_collections, - 'gamma') + gamma_collections = get_variable_collections( + variables_collections, 'gamma' + ) gamma = model_variable( - 'gamma', - shape=params_shape, - dtype=dtype, - initializer=init_ops.ones_initializer(), - collections=gamma_collections, - trainable=trainable) + 'gamma', + shape=params_shape, + dtype=dtype, + initializer=init_ops.ones_initializer(), + collections=gamma_collections, + trainable=trainable + ) # Calculate the moments on the last axis (layer activations). norm_axes = list(range(begin_norm_axis, inputs_rank)) mean, variance = nn.moments(inputs, norm_axes, keep_dims=True) # Compute layer normalization using the batch_normalization function. variance_epsilon = 1e-12 outputs = nn.batch_normalization( - inputs, - mean, - variance, - offset=beta, - scale=gamma, - variance_epsilon=variance_epsilon) + inputs, + mean, + variance, + offset=beta, + scale=gamma, + variance_epsilon=variance_epsilon + ) outputs.set_shape(inputs_shape) if activation_fn is not None: outputs = activation_fn(outputs) @@ -205,18 +211,20 @@ def append_tensor_alias(tensor, alias): return tensor -def variable(name, - shape=None, - dtype=None, - initializer=None, - regularizer=None, - trainable=True, - collections=None, - caching_device=None, - device=None, - partitioner=None, - custom_getter=None, - use_resource=None): +def variable( + name, + shape=None, + dtype=None, + initializer=None, + regularizer=None, + trainable=True, + collections=None, + caching_device=None, + device=None, + partitioner=None, + custom_getter=None, + use_resource=None +): """Gets an existing variable with these parameters or creates a new one. Args: @@ -246,41 +254,47 @@ def variable(name, Returns: The created or existing variable. """ - collections = list(collections if collections is not None else - [ops.GraphKeys.GLOBAL_VARIABLES]) + collections = list( + collections + if collections is not None else [ops.GraphKeys.GLOBAL_VARIABLES] + ) # Remove duplicates collections = list(set(collections)) getter = variable_scope.get_variable if custom_getter is not None: getter = functools.partial( - custom_getter, reuse=variable_scope.get_variable_scope().reuse) + custom_getter, reuse=variable_scope.get_variable_scope().reuse + ) with ops.device(device or ''): return getter( - name, - shape=shape, - dtype=dtype, - initializer=initializer, - regularizer=regularizer, - trainable=trainable, - collections=collections, - caching_device=caching_device, - partitioner=partitioner, - use_resource=use_resource) - - -def model_variable(name, - shape=None, - dtype=dtypes.float32, - initializer=None, - regularizer=None, - trainable=True, - collections=None, - caching_device=None, - device=None, - partitioner=None, - custom_getter=None, - use_resource=None): + name, + shape=shape, + dtype=dtype, + initializer=initializer, + regularizer=regularizer, + trainable=trainable, + collections=collections, + caching_device=caching_device, + partitioner=partitioner, + use_resource=use_resource + ) + + +def model_variable( + name, + shape=None, + dtype=dtypes.float32, + initializer=None, + regularizer=None, + trainable=True, + collections=None, + caching_device=None, + device=None, + partitioner=None, + custom_getter=None, + use_resource=None +): """Gets an existing model variable with these parameters or creates a new one. Args: @@ -312,18 +326,21 @@ def model_variable(name, The created or existing variable. """ collections = list(collections or []) - collections += [ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.MODEL_VARIABLES] + collections += [ + ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.MODEL_VARIABLES + ] var = variable( - name, - shape=shape, - dtype=dtype, - initializer=initializer, - regularizer=regularizer, - trainable=trainable, - collections=collections, - caching_device=caching_device, - device=device, - partitioner=partitioner, - custom_getter=custom_getter, - use_resource=use_resource) + name, + shape=shape, + dtype=dtype, + initializer=initializer, + regularizer=regularizer, + trainable=trainable, + collections=collections, + caching_device=caching_device, + device=device, + partitioner=partitioner, + custom_getter=custom_getter, + use_resource=use_resource + ) return var diff --git a/easy_rec/python/compat/optimizers.py b/easy_rec/python/compat/optimizers.py index d31a4cd41..35d0fd58f 100644 --- a/easy_rec/python/compat/optimizers.py +++ b/easy_rec/python/compat/optimizers.py @@ -15,25 +15,15 @@ # ============================================================================== """Optimizer ops for use in layers and tf.learn.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import logging - import six import tensorflow as tf # from tensorflow.contrib import framework as contrib_framework -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops +from tensorflow.python.framework import dtypes, ops # from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_nn_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops +from tensorflow.python.ops import array_ops, clip_ops, control_flow_ops, gen_nn_ops, init_ops, math_ops, random_ops # NOQA from tensorflow.python.ops import variable_scope as vs from tensorflow.python.ops import variables as vars_ from tensorflow.python.summary import summary @@ -42,8 +32,7 @@ from tensorflow.python.training import training as train from easy_rec.python.ops.incr_record import set_sparse_indices -from easy_rec.python.utils import constant -from easy_rec.python.utils import estimator_utils +from easy_rec.python.utils import constant, estimator_utils try: from tensorflow.python.framework import indexed_slices @@ -57,52 +46,54 @@ try: from sparse_operation_kit import experiment as sok + from easy_rec.python.compat import sok_optimizer except Exception: sok = None OPTIMIZER_CLS_NAMES = { - 'Adagrad': - train.AdagradOptimizer, - 'Adam': - train.AdamOptimizer, - 'Ftrl': - train.FtrlOptimizer, - 'Momentum': - lambda learning_rate: train.MomentumOptimizer( - learning_rate, momentum=0.9), # pylint: disable=line-too-long - 'RMSProp': - train.RMSPropOptimizer, - 'SGD': - train.GradientDescentOptimizer, + 'Adagrad': + train.AdagradOptimizer, + 'Adam': + train.AdamOptimizer, + 'Ftrl': + train.FtrlOptimizer, + 'Momentum': + lambda learning_rate: train.MomentumOptimizer(learning_rate, momentum=0.9), # pylint: disable=line-too-long + 'RMSProp': + train.RMSPropOptimizer, + 'SGD': + train.GradientDescentOptimizer, } OPTIMIZER_SUMMARIES = [ - 'learning_rate', - 'loss', - 'gradients', - 'gradient_norm', - 'global_gradient_norm', + 'learning_rate', + 'loss', + 'gradients', + 'gradient_norm', + 'global_gradient_norm', ] -def optimize_loss(loss, - global_step, - learning_rate, - optimizer, - gradient_noise_scale=None, - gradient_multipliers=None, - clip_gradients=None, - learning_rate_decay_fn=None, - update_ops=None, - variables=None, - name=None, - summaries=None, - colocate_gradients_with_ops=False, - not_apply_grad_after_first_step=False, - increment_global_step=True, - incr_save=False, - embedding_parallel=False): +def optimize_loss( + loss, + global_step, + learning_rate, + optimizer, + gradient_noise_scale=None, + gradient_multipliers=None, + clip_gradients=None, + learning_rate_decay_fn=None, + update_ops=None, + variables=None, + name=None, + summaries=None, + colocate_gradients_with_ops=False, + not_apply_grad_after_first_step=False, + increment_global_step=True, + incr_save=False, + embedding_parallel=False +): """Given loss and parameters for optimizer, returns a training op. Various ways of passing optimizers include: @@ -208,27 +199,33 @@ def optimize_loss(loss, # Learning rate variable, with possible decay. lr = None if learning_rate is not None: - if (isinstance(learning_rate, ops.Tensor) and - learning_rate.get_shape().ndims == 0): + if ( + isinstance(learning_rate, ops.Tensor) + and learning_rate.get_shape().ndims == 0 + ): lr = learning_rate elif isinstance(learning_rate, float): if learning_rate < 0.0: raise ValueError('Invalid learning_rate %s.', learning_rate) lr = vs.get_variable( - 'learning_rate', [], - trainable=False, - initializer=init_ops.constant_initializer(learning_rate)) + 'learning_rate', [], + trainable=False, + initializer=init_ops.constant_initializer(learning_rate) + ) else: - raise ValueError('Learning rate should be 0d Tensor or float. ' - 'Got %s of type %s' % - (str(learning_rate), str(type(learning_rate)))) + raise ValueError( + 'Learning rate should be 0d Tensor or float. ' + 'Got %s of type %s' % (str(learning_rate), str(type(learning_rate))) + ) if summaries is None: summaries = ['loss', 'learning_rate', 'global_gradient_norm'] else: for summ in summaries: if summ not in OPTIMIZER_SUMMARIES: - raise ValueError('Summaries should be one of [%s], you provided %s.' % - (', '.join(OPTIMIZER_SUMMARIES), summ)) + raise ValueError( + 'Summaries should be one of [%s], you provided %s.' % + (', '.join(OPTIMIZER_SUMMARIES), summ) + ) if learning_rate is not None and learning_rate_decay_fn is not None: if global_step is None: raise ValueError('global_step is required for learning_rate_decay_fn.') @@ -239,18 +236,25 @@ def optimize_loss(loss, # Create optimizer, given specified parameters. if isinstance(optimizer, six.string_types): if lr is None: - raise ValueError('Learning rate is None, but should be specified if ' - 'optimizer is string (%s).' % optimizer) + raise ValueError( + 'Learning rate is None, but should be specified if ' + 'optimizer is string (%s).' % optimizer + ) if optimizer not in OPTIMIZER_CLS_NAMES: raise ValueError( - 'Optimizer name should be one of [%s], you provided %s.' % - (', '.join(OPTIMIZER_CLS_NAMES), optimizer)) + 'Optimizer name should be one of [%s], you provided %s.' % + (', '.join(OPTIMIZER_CLS_NAMES), optimizer) + ) opt = OPTIMIZER_CLS_NAMES[optimizer](learning_rate=lr) - elif (isinstance(optimizer, type) and - issubclass(optimizer, optimizer_.Optimizer)): + elif ( + isinstance(optimizer, type) + and issubclass(optimizer, optimizer_.Optimizer) + ): if lr is None: - raise ValueError('Learning rate is None, but should be specified if ' - 'optimizer is class (%s).' % optimizer) + raise ValueError( + 'Learning rate is None, but should be specified if ' + 'optimizer is class (%s).' % optimizer + ) opt = optimizer(learning_rate=lr) elif isinstance(optimizer, optimizer_.Optimizer): opt = optimizer @@ -260,17 +264,20 @@ def optimize_loss(loss, else: opt = optimizer() if not isinstance(opt, optimizer_.Optimizer): - raise ValueError('Unrecognized optimizer: function should return ' - 'subclass of Optimizer. Got %s.' % str(opt)) + raise ValueError( + 'Unrecognized optimizer: function should return ' + 'subclass of Optimizer. Got %s.' % str(opt) + ) elif isinstance(optimizer, sok_optimizer.OptimizerWrapperV1) or \ isinstance(optimizer, sok_optimizer.OptimizerWrapperV2): opt = optimizer else: - raise ValueError('Unrecognized optimizer: should be string, ' - 'subclass of Optimizer, instance of ' - 'subclass of Optimizer or function with one argument. ' - 'Got %s[type=%s].' % - (str(optimizer), str(type(optimizer)))) + raise ValueError( + 'Unrecognized optimizer: should be string, ' + 'subclass of Optimizer, instance of ' + 'subclass of Optimizer or function with one argument. ' + 'Got %s[type=%s].' % (str(optimizer), str(type(optimizer))) + ) # All trainable variables, if specific variables are not specified. if variables is None: @@ -278,18 +285,21 @@ def optimize_loss(loss, # Compute gradients. gradients = opt.compute_gradients( - loss, - variables, - colocate_gradients_with_ops=colocate_gradients_with_ops) + loss, variables, colocate_gradients_with_ops=colocate_gradients_with_ops + ) if estimator_utils.has_hvd() and hvd.size() > 1: if not embedding_parallel: # embedding parameters not partitioned reduced_grads = [] for g, v in gradients: - reduced_grads.append((hvd.allreduce( - g, op=hvd.Average, - compression=hvd.compression.NoneCompressor), v)) + reduced_grads.append( + ( + hvd.allreduce( + g, op=hvd.Average, compression=hvd.compression.NoneCompressor + ), v + ) + ) gradients = reduced_grads else: # embedding parameters partitioned: @@ -312,50 +322,61 @@ def optimize_loss(loss, part_grads.append(g) part_vars.append(v) else: - reduced_grads.append((indexed_slices.IndexedSlices( - indices=g.indices, values=g.values / hvd.size()), v)) + reduced_grads.append( + ( + indexed_slices.IndexedSlices( + indices=g.indices, values=g.values / hvd.size() + ), v + ) + ) group_allreduce = False if len(part_grads) > 0: if group_allreduce: reduced_part_grads = hvd.grouped_allreduce( - part_grads, - op=hvd.Average, - compression=hvd.compression.NoneCompressor) + part_grads, + op=hvd.Average, + compression=hvd.compression.NoneCompressor + ) for g, v in zip(reduced_part_grads, part_vars): reduced_grads.append((g, v)) else: for g, v in zip(part_grads, part_vars): g = hvd.allreduce( - g, op=hvd.Average, compression=hvd.compression.NoneCompressor) + g, op=hvd.Average, compression=hvd.compression.NoneCompressor + ) reduced_grads.append((g, v)) if len(part_sparse_grads) > 0: if group_allreduce: reduced_part_grads = hvd.grouped_allreduce( - part_sparse_grads, - op=hvd.Average, - compression=hvd.compression.NoneCompressor) + part_sparse_grads, + op=hvd.Average, + compression=hvd.compression.NoneCompressor + ) for g, v in zip(reduced_part_grads, part_sparse_vars): reduced_grads.append((g, v)) else: for g, v in zip(part_sparse_grads, part_sparse_vars): g = hvd.allreduce( - g, op=hvd.Average, compression=hvd.compression.NoneCompressor) + g, op=hvd.Average, compression=hvd.compression.NoneCompressor + ) reduced_grads.append((g, v)) gradients = reduced_grads # Optionally add gradient noise. if gradient_noise_scale is not None: - gradients = _add_scaled_noise_to_gradients(gradients, - gradient_noise_scale) + gradients = _add_scaled_noise_to_gradients( + gradients, gradient_noise_scale + ) # Multiply some gradients. if gradient_multipliers is not None: gradients = _multiply_gradients(gradients, gradient_multipliers) if not gradients: raise ValueError( - 'Empty list of (gradient, var) pairs encountered. This is most ' - 'likely to be caused by an improper value of gradient_multipliers.') + 'Empty list of (gradient, var) pairs encountered. This is most ' + 'likely to be caused by an improper value of gradient_multipliers.' + ) # if 'global_gradient_norm' in summaries or 'gradient_norm' in summaries: # summary.scalar('global_norm/gradient_norm', @@ -365,20 +386,23 @@ def optimize_loss(loss, if isinstance(clip_gradients, float): # gradients = _clip_gradients_by_norm(gradients, clip_gradients) sparse_norm, dense_norm, grad_norm = _get_grad_norm( - gradients, embedding_parallel) + gradients, embedding_parallel + ) summary.scalar('global_norm/sparse_grad', sparse_norm) summary.scalar('global_norm/dense_grad', dense_norm) summary.scalar('global_norm/gradient_norm', grad_norm) grads = [x[0] for x in gradients] vars = [x[1] for x in gradients] clipped_grads, _ = clip_ops.clip_by_global_norm( - grads, clip_gradients, use_norm=grad_norm) + grads, clip_gradients, use_norm=grad_norm + ) gradients = list(zip(clipped_grads, vars)) elif callable(clip_gradients): gradients = clip_gradients(gradients) elif clip_gradients is not None: - raise ValueError('Unknown type %s for clip_gradients' % - type(clip_gradients)) + raise ValueError( + 'Unknown type %s for clip_gradients' % type(clip_gradients) + ) # Add scalar summary for loss. if 'loss' in summaries: @@ -397,13 +421,17 @@ def optimize_loss(loss, if 'gradients' in summaries: summary.histogram('gradients/%s' % var_name, grad_values) if 'gradient_norm' in summaries: - summary.scalar('gradient_norm/%s' % var_name, - clip_ops.global_norm([grad_values])) - - if clip_gradients is not None and ('global_gradient_norm' in summaries or - 'gradient_norm' in summaries): + summary.scalar( + 'gradient_norm/%s' % var_name, + clip_ops.global_norm([grad_values]) + ) + + if clip_gradients is not None and ( + 'global_gradient_norm' in summaries or 'gradient_norm' in summaries + ): sparse_norm, dense_norm, grad_norm = _get_grad_norm( - gradients, embedding_parallel) + gradients, embedding_parallel + ) summary.scalar('global_norm/clipped_sparse_grad', sparse_norm) summary.scalar('global_norm/clipped_dense_grad', dense_norm) summary.scalar('global_norm/clipped_gradient_norm', grad_norm) @@ -411,9 +439,10 @@ def optimize_loss(loss, # Create gradient updates. def _apply_grad(): grad_updates = opt.apply_gradients( - gradients, - global_step=global_step if increment_global_step else None, - name='train') + gradients, + global_step=global_step if increment_global_step else None, + name='train' + ) embed_para_vars = ops.get_collection(constant.EmbeddingParallel) slot_names = opt.get_slot_names() @@ -421,7 +450,9 @@ def _apply_grad(): if var.name in embed_para_vars: for slot_name in slot_names: tmp_var = opt.get_slot(var, slot_name) - logging.info('add shard embedding optimizer var: %s' % tmp_var.name) + logging.info( + 'add shard embedding optimizer var: %s' % tmp_var.name + ) ops.add_to_collection(constant.EmbeddingParallel, tmp_var.name) incr_save_ops = [] @@ -430,11 +461,13 @@ def _apply_grad(): if isinstance(grad, indexed_slices.IndexedSlices): indices = grad.indices with ops.colocate_with(var), ops.control_dependencies( - [grad_updates]): + [grad_updates] + ): incr_save_op = set_sparse_indices(indices, var_name=var.op.name) incr_save_ops.append(incr_save_op) - ops.add_to_collection('SPARSE_UPDATE_VARIABLES', - (var, grad.indices.dtype)) + ops.add_to_collection( + 'SPARSE_UPDATE_VARIABLES', (var, grad.indices.dtype) + ) else: ops.add_to_collection('DENSE_UPDATE_VARIABLES', var) return tf.group(incr_save_ops) @@ -465,15 +498,19 @@ def _get_grad_norm(grads_and_vars, embedding_parallel=False): else: dense_norms.append(gen_nn_ops.l2_loss(grad)) reduced_norms = hvd.grouped_allreduce( - part_norms, op=hvd.Sum, compression=hvd.compression.NoneCompressor) + part_norms, op=hvd.Sum, compression=hvd.compression.NoneCompressor + ) sparse_norms = sparse_norms + reduced_norms all_norms = reduced_norms + dense_norms sparse_norm = math_ops.sqrt( - math_ops.reduce_sum(array_ops.stack(sparse_norms) * 2.0)) + math_ops.reduce_sum(array_ops.stack(sparse_norms) * 2.0) + ) dense_norm = math_ops.sqrt( - math_ops.reduce_sum(array_ops.stack(dense_norms) * 2.0)) + math_ops.reduce_sum(array_ops.stack(dense_norms) * 2.0) + ) grad_norm = math_ops.sqrt( - math_ops.reduce_sum(array_ops.stack(all_norms)) * 2.0) + math_ops.reduce_sum(array_ops.stack(all_norms)) * 2.0 + ) return sparse_norm, dense_norm, grad_norm @@ -481,7 +518,9 @@ def _clip_gradients_by_norm(grads_and_vars, clip_gradients): """Clips gradients by global norm.""" gradients, variables = zip(*grads_and_vars) - clipped_gradients, _ = clip_ops.clip_by_global_norm(gradients, clip_gradients) + clipped_gradients, _ = clip_ops.clip_by_global_norm( + gradients, clip_gradients + ) return list(zip(clipped_gradients, variables)) @@ -492,13 +531,15 @@ def _adaptive_max_norm(norm, std_factor, decay, global_step, epsilon, name): def moving_average(name, value, decay): moving_average_variable = vs.get_variable( - name, - shape=value.get_shape(), - dtype=value.dtype, - initializer=init_ops.zeros_initializer(), - trainable=False) + name, + shape=value.get_shape(), + dtype=value.dtype, + initializer=init_ops.zeros_initializer(), + trainable=False + ) return moving_averages.assign_moving_average( - moving_average_variable, value, decay, zero_debias=False) + moving_average_variable, value, decay, zero_debias=False + ) # quicker adaptation at the beginning if global_step is not None: @@ -515,13 +556,15 @@ def moving_average(name, value, decay): return max_norms, mean -def adaptive_clipping_fn(std_factor=2., - decay=0.95, - static_max_norm=None, - global_step=None, - report_summary=False, - epsilon=1e-8, - name=None): +def adaptive_clipping_fn( + std_factor=2., + decay=0.95, + static_max_norm=None, + global_step=None, + report_summary=False, + epsilon=1e-8, + name=None +): """Adapt the clipping value using statistics on the norms. Implement adaptive gradient as presented in section 3.2.1 of @@ -553,16 +596,19 @@ def gradient_clipping(grads_and_vars): norm = clip_ops.global_norm(grads) - max_norm, log_mean = _adaptive_max_norm(norm, std_factor, decay, - global_step, epsilon, name) + max_norm, log_mean = _adaptive_max_norm( + norm, std_factor, decay, global_step, epsilon, name + ) # reports the max gradient norm for debugging if report_summary: summary.scalar('global_norm/adaptive_max_gradient_norm', max_norm) # factor will be 1. if norm is smaller than max_norm - factor = array_ops.where(norm < max_norm, array_ops.ones_like(norm), - math_ops.exp(log_mean) / norm) + factor = array_ops.where( + norm < max_norm, array_ops.ones_like(norm), + math_ops.exp(log_mean) / norm + ) if static_max_norm is not None: factor = math_ops.minimum(static_max_norm / norm, factor) @@ -574,8 +620,10 @@ def gradient_clipping(grads_and_vars): clipped_grads.append(None) elif isinstance(grad, indexed_slices.IndexedSlices): clipped_grads.append( - indexed_slices.IndexedSlices(grad.values * factor, grad.indices, - grad.dense_shape)) + indexed_slices.IndexedSlices( + grad.values * factor, grad.indices, grad.dense_shape + ) + ) else: clipped_grads.append(grad * factor) @@ -605,14 +653,17 @@ def _multiply_gradients(grads_and_vars, gradient_multipliers): """Multiply specified gradients.""" multiplied_grads_and_vars = [] for grad, var in grads_and_vars: - if (grad is not None and - (var in gradient_multipliers or var.name in gradient_multipliers)): + if ( + grad is not None + and (var in gradient_multipliers or var.name in gradient_multipliers) + ): key = var if var in gradient_multipliers else var.name multiplier = gradient_multipliers[key] if isinstance(grad, indexed_slices.IndexedSlices): grad_values = grad.values * multiplier - grad = indexed_slices.IndexedSlices(grad_values, grad.indices, - grad.dense_shape) + grad = indexed_slices.IndexedSlices( + grad_values, grad.indices, grad.dense_shape + ) else: grad *= math_ops.cast(multiplier, grad.dtype) multiplied_grads_and_vars.append((grad, var)) diff --git a/easy_rec/python/compat/queues.py b/easy_rec/python/compat/queues.py index c7063d966..1be904921 100644 --- a/easy_rec/python/compat/queues.py +++ b/easy_rec/python/compat/queues.py @@ -11,18 +11,14 @@ import errno import logging import os +import six import sys import threading import time import weakref from multiprocessing import connection -from multiprocessing.util import Finalize -from multiprocessing.util import is_exiting -from multiprocessing.util import register_after_fork -from queue import Empty -from queue import Full - -import six +from multiprocessing.util import Finalize, is_exiting, register_after_fork +from queue import Empty, Full try: from multiprocessing import context @@ -69,13 +65,16 @@ def __init__(self, ctx, maxsize=0, name=''): def __getstate__(self): context.assert_spawning(self) - return (self._ignore_epipe, self._maxsize, self._reader, self._writer, - self._rlock, self._wlock, self._sem, self._opid, self._name, - self._run) + return ( + self._ignore_epipe, self._maxsize, self._reader, self._writer, + self._rlock, self._wlock, self._sem, self._opid, self._name, self._run + ) def __setstate__(self, state): - (self._ignore_epipe, self._maxsize, self._reader, self._writer, self._rlock, - self._wlock, self._sem, self._opid, self._name, self._run) = state + ( + self._ignore_epipe, self._maxsize, self._reader, self._writer, + self._rlock, self._wlock, self._sem, self._opid, self._name, self._run + ) = state self._reset() def _after_fork(self): @@ -192,11 +191,14 @@ def _start_thread(self): # Start thread which transfers data from buffer to pipe self._buffer.clear() self._thread = threading.Thread( - target=self._feed, - args=(self._buffer, self._notempty, self._send_bytes, self._wlock, - self._reader.close, self._writer.close, self._ignore_epipe, - self._on_queue_feeder_error, self._sem), - name='QueueFeederThread') + target=self._feed, + args=( + self._buffer, self._notempty, self._send_bytes, self._wlock, + self._reader.close, self._writer.close, self._ignore_epipe, + self._on_queue_feeder_error, self._sem + ), + name='QueueFeederThread' + ) self._thread.daemon = True logging.debug('doing self._thread.start()') @@ -205,15 +207,17 @@ def _start_thread(self): if not self._joincancelled: self._jointhread = Finalize( - self._thread, - Queue._finalize_join, [weakref.ref(self._thread)], - exitpriority=-5) + self._thread, + Queue._finalize_join, [weakref.ref(self._thread)], + exitpriority=-5 + ) # Send sentinel to the thread queue object when garbage collected self._close = Finalize( - self, - Queue._finalize_close, [self._buffer, self._notempty], - exitpriority=10) + self, + Queue._finalize_close, [self._buffer, self._notempty], + exitpriority=10 + ) @staticmethod def _finalize_join(twr): @@ -232,8 +236,10 @@ def _finalize_close(buffer, notempty): buffer.append(Queue._sentinel) notempty.notify() - def _feed(self, buffer, notempty, send_bytes, writelock, reader_close, - writer_close, ignore_epipe, onerror, queue_sem): + def _feed( + self, buffer, notempty, send_bytes, writelock, reader_close, writer_close, + ignore_epipe, onerror, queue_sem + ): logging.debug('starting thread to feed data to pipe') nacquire = notempty.acquire nrelease = notempty.release @@ -279,16 +285,20 @@ def _feed(self, buffer, notempty, send_bytes, writelock, reader_close, pass except Exception as e: if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: - logging.warning('Queue[' + name + '] exception: pid=' + str(pid) + - ' run=' + str(self._run) + ' e=' + str(e)) + logging.warning( + 'Queue[' + name + '] exception: pid=' + str(pid) + ' run=' + + str(self._run) + ' e=' + str(e) + ) return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. if is_exiting(): - logging.warning('Queue[' + name + '] thread error in exiting: pid=' + - str(pid) + ' run=' + str(self._run) + ' e=' + str(e)) + logging.warning( + 'Queue[' + name + '] thread error in exiting: pid=' + str(pid) + + ' run=' + str(self._run) + ' e=' + str(e) + ) return else: # Since the object has not been sent in the queue, we need diff --git a/easy_rec/python/compat/regularizers.py b/easy_rec/python/compat/regularizers.py index 9609c037b..840c35671 100644 --- a/easy_rec/python/compat/regularizers.py +++ b/easy_rec/python/compat/regularizers.py @@ -16,22 +16,16 @@ # from tf.contrib """Regularizers for use with layers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numbers - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import standard_ops +from tensorflow.python.framework import constant_op, ops +from tensorflow.python.ops import math_ops, nn, standard_ops from tensorflow.python.platform import tf_logging as logging __all__ = [ - 'l1_regularizer', 'l2_regularizer', 'l1_l2_regularizer', 'sum_regularizer', - 'apply_regularization' + 'l1_regularizer', 'l2_regularizer', 'l1_l2_regularizer', 'sum_regularizer', + 'apply_regularization' ] @@ -54,8 +48,9 @@ def l1_regularizer(scale, scope=None): raise ValueError('scale cannot be an integer: %s' % scale) if isinstance(scale, numbers.Real): if scale < 0.: - raise ValueError('Setting a scale less than 0 on a regularizer: %g' % - scale) + raise ValueError( + 'Setting a scale less than 0 on a regularizer: %g' % scale + ) if scale == 0.: logging.info('Scale of 0 disables regularizer.') return lambda _: None @@ -64,11 +59,13 @@ def l1(weights, name=None): """Applies L1 regularization to weights.""" with ops.name_scope(scope, 'l1_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor( - scale, dtype=weights.dtype.base_dtype, name='scale') + scale, dtype=weights.dtype.base_dtype, name='scale' + ) return standard_ops.multiply( - my_scale, - standard_ops.reduce_sum(standard_ops.abs(weights)), - name=name) + my_scale, + standard_ops.reduce_sum(standard_ops.abs(weights)), + name=name + ) return l1 @@ -89,11 +86,12 @@ def l2_regularizer(scale, scope=None): ValueError: If scale is negative or if scale is not a float. """ if isinstance(scale, numbers.Integral): - raise ValueError('scale cannot be an integer: %s' % (scale,)) + raise ValueError('scale cannot be an integer: %s' % (scale, )) if isinstance(scale, numbers.Real): if scale < 0.: - raise ValueError('Setting a scale less than 0 on a regularizer: %g.' % - scale) + raise ValueError( + 'Setting a scale less than 0 on a regularizer: %g.' % scale + ) if scale == 0.: logging.info('Scale of 0 disables regularizer.') return lambda _: None @@ -102,7 +100,8 @@ def l2(weights): """Applies l2 regularization to weights.""" with ops.name_scope(scope, 'l2_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor( - scale, dtype=weights.dtype.base_dtype, name='scale') + scale, dtype=weights.dtype.base_dtype, name='scale' + ) return standard_ops.multiply(my_scale, nn.l2_loss(weights), name=name) return l2 @@ -124,17 +123,18 @@ def l1_l2_regularizer(scale_l1=1.0, scale_l2=1.0, scope=None): ValueError: If scale is negative or if scale is not a float. """ if isinstance(scale_l1, numbers.Integral): - raise ValueError('scale_l1 cannot be an integer: %s' % (scale_l1,)) + raise ValueError('scale_l1 cannot be an integer: %s' % (scale_l1, )) if isinstance(scale_l2, numbers.Integral): - raise ValueError('scale_l2 cannot be an integer: %s' % (scale_l2,)) + raise ValueError('scale_l2 cannot be an integer: %s' % (scale_l2, )) scope = scope or 'l1_l2_regularizer' if scale_l1 == 0.: return l2_regularizer(scale_l2, scope) if scale_l2 == 0.: return l1_regularizer(scale_l1, scope) - return sum_regularizer([l1_regularizer(scale_l1), - l2_regularizer(scale_l2)], - scope=scope) + return sum_regularizer( + [l1_regularizer(scale_l1), + l2_regularizer(scale_l2)], scope=scope + ) def sum_regularizer(regularizer_list, scope=None): @@ -161,7 +161,8 @@ def sum_reg(weights): if tensor is not None: regularizer_tensors.append(tensor) return math_ops.add_n( - regularizer_tensors, name=name) if regularizer_tensors else None + regularizer_tensors, name=name + ) if regularizer_tensors else None return sum_reg @@ -193,15 +194,18 @@ def apply_regularization(regularizer, weights_list=None): if not weights_list: raise ValueError('No weights to regularize.') with ops.name_scope( - 'get_regularization_penalty', values=weights_list) as scope: + 'get_regularization_penalty', values=weights_list + ) as scope: penalties = [regularizer(w) for w in weights_list] penalties = [ - p if p is not None else constant_op.constant(0.0) for p in penalties + p if p is not None else constant_op.constant(0.0) for p in penalties ] for p in penalties: if p.get_shape().ndims != 0: - raise ValueError('regularizer must return a scalar Tensor instead of a ' - 'Tensor with rank %d.' % p.get_shape().ndims) + raise ValueError( + 'regularizer must return a scalar Tensor instead of a ' + 'Tensor with rank %d.' % p.get_shape().ndims + ) summed_penalty = math_ops.add_n(penalties, name=scope) ops.add_to_collection(ops.GraphKeys.REGULARIZATION_LOSSES, summed_penalty) diff --git a/easy_rec/python/compat/sok_optimizer.py b/easy_rec/python/compat/sok_optimizer.py index 7f368a9a1..470bbb471 100644 --- a/easy_rec/python/compat/sok_optimizer.py +++ b/easy_rec/python/compat/sok_optimizer.py @@ -19,10 +19,7 @@ # from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops # from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops +from tensorflow.python.ops import array_ops, gradients, resource_variable_ops, state_ops # NOQA from easy_rec.python.compat.dynamic_variable import DynamicVariable @@ -83,10 +80,9 @@ class OptimizerWrapperV1(object): def __init__(self, optimizer): self._optimizer = optimizer # slots - unused = tf.Variable([0.0], - dtype=tf.float32, - name='unused', - trainable=False) + unused = tf.Variable( + [0.0], dtype=tf.float32, name='unused', trainable=False + ) self._optimizer._create_slots([unused]) names, slots = [], [] for name in self._optimizer.get_slot_names(): @@ -101,12 +97,14 @@ def __init__(self, optimizer): self._initial_vals[name] = slots[i] # self._optimizer._prepare() - def compute_gradients(self, - loss, - var_list=None, - aggregation_method=None, - colocate_gradients_with_ops=False, - grad_loss=None): + def compute_gradients( + self, + loss, + var_list=None, + aggregation_method=None, + colocate_gradients_with_ops=False, + grad_loss=None + ): self._loss = loss tmp_grads = gradients.gradients(loss, var_list) return list(zip(tmp_grads, var_list)) @@ -137,21 +135,23 @@ def _create_slots_dynamic(self, var): if var.backend_type == 'hbm': with ops.colocate_with(var): slot = DynamicVariable( - dimension=var.dimension, - initializer=self._initial_vals[slot_name], - name='DynamicSlot', - trainable=False) + dimension=var.dimension, + initializer=self._initial_vals[slot_name], + name='DynamicSlot', + trainable=False + ) else: tmp_config = var.config_dict # tmp_initializer = var.initializer_str with ops.colocate_with(var): slot = DynamicVariable( - dimension=var.dimension, - initializer=self._initial_vals[slot_name], - var_type=var.backend_type, - name='DynamicSlot', - trainable=False, - **tmp_config) + dimension=var.dimension, + initializer=self._initial_vals[slot_name], + var_type=var.backend_type, + name='DynamicSlot', + trainable=False, + **tmp_config + ) self._optimizer._slots[slot_name][key] = slot @@ -168,9 +168,11 @@ def _slots(self): def apply_gradients(self, grads_and_vars, global_step=None, name=None): gradients = grads_and_vars - sparse_vars = [x for x in gradients if 'DynamicVariable' in str(type(x[1]))] + sparse_vars = [ + x for x in gradients if 'DynamicVariable' in str(type(x[1])) + ] dense_vars = [ - x for x in gradients if 'DynamicVariable' not in str(type(x[1])) + x for x in gradients if 'DynamicVariable' not in str(type(x[1])) ] def _dummy_finish(update_ops, name_scope): @@ -182,7 +184,8 @@ def _dummy_finish(update_ops, name_scope): sparse_grad_updates = self.apply_sparse_gradients(sparse_vars, name=name) dense_grad_updates = self._optimizer.apply_gradients( - dense_vars, global_step=None, name=name) + dense_vars, global_step=None, name=name + ) if sparse_grad_updates is not None and dense_grad_updates is not None: grad_updates = sparse_grad_updates + dense_grad_updates elif sparse_grad_updates is not None: @@ -197,9 +200,10 @@ def _dummy_finish(update_ops, name_scope): # TODO(apassos): the implicit read in assign_add is slow; consider # making it less so. apply_updates = resource_variable_ops.assign_add_variable_op( - global_step.handle, - ops.convert_to_tensor(1, dtype=global_step.dtype), - name=name) + global_step.handle, + ops.convert_to_tensor(1, dtype=global_step.dtype), + name=name + ) else: apply_updates = state_ops.assign_add(global_step, 1, name=name) @@ -212,7 +216,9 @@ def _dummy_finish(update_ops, name_scope): return apply_updates - def apply_sparse_gradients(self, grads_and_vars, global_step=None, name=None): + def apply_sparse_gradients( + self, grads_and_vars, global_step=None, name=None + ): # 1. Create slots and do sparse_read to_static_ops = [] grad_list, var_list = [], [] @@ -231,22 +237,23 @@ def apply_sparse_gradients(self, grads_and_vars, global_step=None, name=None): if v.backend_type == 'hbm': with ops.colocate_with(v): slot = DynamicVariable( - dimension=v.dimension, - initializer=self._initial_vals[slot_name], - name=tmp_slot_var_name, - trainable=False, + dimension=v.dimension, + initializer=self._initial_vals[slot_name], + name=tmp_slot_var_name, + trainable=False, ) else: tmp_config = v.config_dict # tmp_initializer = v.initializer_str with ops.colocate_with(v): slot = DynamicVariable( - dimension=v.dimension, - initializer=self._initial_vals[slot_name], - var_type=v.backend_type, - name=tmp_slot_var_name, - trainable=False, - **tmp_config) + dimension=v.dimension, + initializer=self._initial_vals[slot_name], + var_type=v.backend_type, + name=tmp_slot_var_name, + trainable=False, + **tmp_config + ) self._optimizer._slots[slot_name][key] = slot else: @@ -259,7 +266,8 @@ def apply_sparse_gradients(self, grads_and_vars, global_step=None, name=None): # 3. Call tf-optimizer with ops.control_dependencies(to_static_ops): train_op = self._optimizer.apply_gradients( - zip(grad_list, var_list), global_step=global_step, name=name) + zip(grad_list, var_list), global_step=global_step, name=name + ) # 5. Write buffer back to dynamic variables to_dynamic_ops = [] @@ -282,10 +290,9 @@ def __init__(self, optimizer): self._optimizer = optimizer # slots if tf.__version__[0] == '1': - unused = tf.Variable([0.0], - name='unused', - trainable=False, - use_resource=True) + unused = tf.Variable( + [0.0], name='unused', trainable=False, use_resource=True + ) else: unused = tf.Variable([0.0], name='unused', trainable=False) self._optimizer._create_slots([unused]) @@ -320,21 +327,22 @@ def _create_slots_dynamic(self, var): if slot_name not in self._optimizer._slots[key]: if var.backend_type == 'hbm': slot = DynamicVariable( - dimension=var.dimension, - initializer=self._initial_vals[slot_name], - name='DynamicSlot', - trainable=False, + dimension=var.dimension, + initializer=self._initial_vals[slot_name], + name='DynamicSlot', + trainable=False, ) else: tmp_config = var.config_dict # tmp_initializer = var.initializer_str slot = DynamicVariable( - dimension=var.dimension, - initializer=self._initial_vals[slot_name], - var_type=var.backend_type, - name='DynamicSlot', - trainable=False, - **tmp_config) + dimension=var.dimension, + initializer=self._initial_vals[slot_name], + var_type=var.backend_type, + name='DynamicSlot', + trainable=False, + **tmp_config + ) self._optimizer._slots[key][slot_name] = slot def _var_key(self, var): @@ -373,21 +381,22 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): if slot_name not in self._optimizer._slots[key]: if v.backend_type == 'hbm': slot = DynamicVariable( - dimension=v.dimension, - initializer=self._initial_vals[slot_name], - name='DynamicSlot', - trainable=False, + dimension=v.dimension, + initializer=self._initial_vals[slot_name], + name='DynamicSlot', + trainable=False, ) else: tmp_config = v.config_dict # tmp_initializer = v.initializer_str slot = DynamicVariable( - dimension=v.dimension, - initializer=self._initial_vals[slot_name], - var_type=v.backend_type, - name='DynamicSlot', - trainable=False, - **tmp_config) + dimension=v.dimension, + initializer=self._initial_vals[slot_name], + var_type=v.backend_type, + name='DynamicSlot', + trainable=False, + **tmp_config + ) self._optimizer._slots[key][slot_name] = slot else: @@ -404,7 +413,8 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): # 3. Call tf-optimizer with tf.control_dependencies(to_static_ops): train_op = self._optimizer.apply_gradients( - zip(grad_list, var_list), name=name) + zip(grad_list, var_list), name=name + ) # 4. Switch iterations self._optimizer._iterations = iterations @@ -434,7 +444,8 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): train_ops = [] for g, v in grads_and_vars: if g is not None: - scaled_g = ops.IndexedSlices(g.values * self._lr, g.indices, - g.dense_shape) + scaled_g = ops.IndexedSlices( + g.values * self._lr, g.indices, g.dense_shape + ) train_ops.append(v.scatter_sub(scaled_g)) return tf.group(train_ops) diff --git a/easy_rec/python/compat/sync_replicas_optimizer.py b/easy_rec/python/compat/sync_replicas_optimizer.py index 24c2921ba..8245f26db 100644 --- a/easy_rec/python/compat/sync_replicas_optimizer.py +++ b/easy_rec/python/compat/sync_replicas_optimizer.py @@ -13,24 +13,13 @@ # limitations under the License. # ============================================================================== """Synchronize replicas for training.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function from tensorflow.core.framework import types_pb2 -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables +from tensorflow.python.framework import errors_impl, ops +from tensorflow.python.ops import array_ops, control_flow_ops, data_flow_ops, state_ops, variable_scope, variables # NOQA from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import optimizer -from tensorflow.python.training import queue_runner -from tensorflow.python.training import session_manager -from tensorflow.python.training import session_run_hook +from tensorflow.python.training import optimizer, queue_runner, session_manager, session_run_hook # NOQA from tensorflow.python.util.tf_export import tf_export @@ -137,15 +126,17 @@ class SyncReplicasOptimizer(optimizer.Optimizer): sync_que_id = -1 - def __init__(self, - opt, - replicas_to_aggregate, - total_num_replicas=None, - variable_averages=None, - variables_to_average=None, - use_locking=False, - name='sync_replicas', - **extra_args): + def __init__( + self, + opt, + replicas_to_aggregate, + total_num_replicas=None, + variable_averages=None, + variables_to_average=None, + use_locking=False, + name='sync_replicas', + **extra_args + ): """Construct a sync_replicas optimizer. Args: @@ -172,8 +163,9 @@ def __init__(self, super(SyncReplicasOptimizer, self).__init__(use_locking, name) logging.info( - 'SyncReplicasV2: replicas_to_aggregate=%s; total_num_replicas=%s', - replicas_to_aggregate, total_num_replicas) + 'SyncReplicasV2: replicas_to_aggregate=%s; total_num_replicas=%s', + replicas_to_aggregate, total_num_replicas + ) self._opt = opt self._replicas_to_aggregate = replicas_to_aggregate self._gradients_applied = False @@ -252,16 +244,18 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): # Colocating local_step variable prevents it being placed on the PS. with ops.colocate_with(local_anchor): self._local_step = variable_scope.variable( - initial_value=0, - trainable=False, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - dtype=global_step.dtype.base_dtype, - name='sync_rep_local_step') + initial_value=0, + trainable=False, + collections=[ops.GraphKeys.LOCAL_VARIABLES], + dtype=global_step.dtype.base_dtype, + name='sync_rep_local_step' + ) self.local_step_init_op = state_ops.assign(self._local_step, global_step) chief_init_ops = [self.local_step_init_op] self.ready_for_local_init_op = variables.report_uninitialized_variables( - variables.global_variables()) + variables.global_variables() + ) with ops.name_scope(None, self._name): for grad, var in grads_and_vars: @@ -273,24 +267,30 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): continue elif isinstance(grad, ops.Tensor): grad_accum = data_flow_ops.ConditionalAccumulator( - grad.dtype, - shape=var.get_shape(), - shared_name=var.name + '/grad_accum') + grad.dtype, + shape=var.get_shape(), + shared_name=var.name + '/grad_accum' + ) train_ops.append( - grad_accum.apply_grad(grad, local_step=self._local_step)) + grad_accum.apply_grad(grad, local_step=self._local_step) + ) aggregated_grad.append( - grad_accum.take_grad(self._replicas_to_aggregate)) + grad_accum.take_grad(self._replicas_to_aggregate) + ) else: if not isinstance(grad, ops.IndexedSlices): raise ValueError('Unknown grad type!') grad_accum = data_flow_ops.SparseConditionalAccumulator( - grad.dtype, shape=(), shared_name=var.name + '/grad_accum') + grad.dtype, shape=(), shared_name=var.name + '/grad_accum' + ) train_ops.append( - grad_accum.apply_indexed_slices_grad( - grad, local_step=self._local_step)) + grad_accum.apply_indexed_slices_grad( + grad, local_step=self._local_step + ) + ) aggregated_grad.append( - grad_accum.take_indexed_slices_grad( - self._replicas_to_aggregate)) + grad_accum.take_indexed_slices_grad(self._replicas_to_aggregate) + ) self._accumulator_list.append((grad_accum, var.device)) @@ -298,8 +298,9 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): # sync_op will be assigned to the same device as the global step. with ops.device(global_step.device), ops.name_scope(''): - update_op = self._opt.apply_gradients(aggregated_grads_and_vars, - global_step) + update_op = self._opt.apply_gradients( + aggregated_grads_and_vars, global_step + ) def _get_token_qname(): SyncReplicasOptimizer.sync_que_id += 1 @@ -313,27 +314,32 @@ def _get_token_qname(): logging.info('create sync_token_queue[%s]' % token_qname) with ops.device(global_step.device), ops.name_scope(''): sync_token_queue = ( - data_flow_ops.FIFOQueue( - -1, - global_step.dtype.base_dtype, - shapes=(), - name=token_qname, - shared_name=token_qname)) + data_flow_ops.FIFOQueue( + -1, + global_step.dtype.base_dtype, + shapes=(), + name=token_qname, + shared_name=token_qname + ) + ) self._sync_token_queue = sync_token_queue self._is_sync_que_closed = sync_token_queue.is_closed() self._close_sync_que = sync_token_queue.close( - cancel_pending_enqueues=True, name='close_sync_token_queue') + cancel_pending_enqueues=True, name='close_sync_token_queue' + ) # dummy_queue is passed to the queue runner. Don't use the real queues # because the queue runner doesn't automatically reopen it once it # closed queues in PS devices. dummy_queue = ( - data_flow_ops.FIFOQueue( - 1, - types_pb2.DT_INT32, - shapes=(), - name='dummy_queue', - shared_name='dummy_queue')) + data_flow_ops.FIFOQueue( + 1, + types_pb2.DT_INT32, + shapes=(), + name='dummy_queue', + shared_name='dummy_queue' + ) + ) with ops.device(global_step.device), ops.name_scope(''): # Replicas have to wait until they can get a token from the token queue. @@ -345,20 +351,23 @@ def _get_token_qname(): # Sync_op needs to insert tokens to the token queue at the end of the # step so the replicas can fetch them to start the next step. tokens = array_ops.fill([self._tokens_per_step], global_step) - sync_op = sync_token_queue.enqueue_many((tokens,)) + sync_op = sync_token_queue.enqueue_many((tokens, )) if self._variable_averages is not None: with ops.control_dependencies([sync_op]), ops.name_scope(''): sync_op = self._variable_averages.apply(self._variables_to_average) self._chief_queue_runner = queue_runner.QueueRunner( - dummy_queue, [sync_op]) - ops.add_to_collection(ops.GraphKeys.QUEUE_RUNNERS, - self._chief_queue_runner) + dummy_queue, [sync_op] + ) + ops.add_to_collection( + ops.GraphKeys.QUEUE_RUNNERS, self._chief_queue_runner + ) for accum, dev in self._accumulator_list: with ops.device(dev): chief_init_ops.append( - accum.set_global_step(global_step, name='SetGlobalStep')) + accum.set_global_step(global_step, name='SetGlobalStep') + ) self.chief_init_op = control_flow_ops.group(*(chief_init_ops)) self._gradients_applied = True return train_op @@ -444,20 +453,22 @@ def get_init_tokens_op(self, num_tokens=-1): """ if self._gradients_applied is False: raise ValueError( - 'get_init_tokens_op() should be called after apply_gradients().') + 'get_init_tokens_op() should be called after apply_gradients().' + ) tokens_needed = self._replicas_to_aggregate - self._total_num_replicas if num_tokens == -1: num_tokens = self._replicas_to_aggregate elif num_tokens < tokens_needed: raise ValueError( - 'Too few tokens to finish the first step: %d (given) vs %d (needed)' % - (num_tokens, tokens_needed)) + 'Too few tokens to finish the first step: %d (given) vs %d (needed)' % + (num_tokens, tokens_needed) + ) if num_tokens > 0: with ops.device(self._global_step.device), ops.name_scope(''): tokens = array_ops.fill([num_tokens], self._global_step) - init_tokens = self._sync_token_queue.enqueue_many((tokens,)) + init_tokens = self._sync_token_queue.enqueue_many((tokens, )) else: init_tokens = control_flow_ops.no_op(name='no_init_tokens') @@ -486,18 +497,22 @@ def __init__(self, sync_optimizer, is_chief, num_tokens): def begin(self): if self._sync_optimizer._gradients_applied is False: # pylint: disable=protected-access raise ValueError( - 'SyncReplicasOptimizer.apply_gradient should be called before using ' - 'the hook.') + 'SyncReplicasOptimizer.apply_gradient should be called before using ' + 'the hook.' + ) if self._is_chief: self._local_init_op = self._sync_optimizer.chief_init_op self._ready_for_local_init_op = ( - self._sync_optimizer.ready_for_local_init_op) + self._sync_optimizer.ready_for_local_init_op + ) self._init_tokens_op = self._sync_optimizer.get_init_tokens_op( - self._num_tokens) + self._num_tokens + ) else: self._local_init_op = self._sync_optimizer.local_step_init_op self._ready_for_local_init_op = ( - self._sync_optimizer.ready_for_local_init_op) + self._sync_optimizer.ready_for_local_init_op + ) self._init_tokens_op = None def after_create_session(self, session, coord): @@ -507,9 +522,9 @@ def after_create_session(self, session, coord): 'Model is not ready for SyncReplicasOptimizer local init.') if not local_init_success: raise RuntimeError( - 'Init operations did not make model ready for SyncReplicasOptimizer ' - 'local_init. Init op: %s, error: %s' % - (self._local_init_op.name, msg)) + 'Init operations did not make model ready for SyncReplicasOptimizer ' + 'local_init. Init op: %s, error: %s' % (self._local_init_op.name, msg) + ) session.run(self._local_init_op) is_closed = session.run(self._sync_optimizer._is_sync_que_closed) assert not is_closed, 'sync_que is closed' diff --git a/easy_rec/python/compat/weight_decay_optimizers.py b/easy_rec/python/compat/weight_decay_optimizers.py index 47a755e0f..5a5e6b538 100755 --- a/easy_rec/python/compat/weight_decay_optimizers.py +++ b/easy_rec/python/compat/weight_decay_optimizers.py @@ -13,15 +13,10 @@ # limitations under the License. # ============================================================================== """Base class to make optimizers weight decay ready.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops +from tensorflow.python.ops import array_ops, control_flow_ops, resource_variable_ops, state_ops # NOQA from tensorflow.python.training import adam from tensorflow.python.training import momentum as momentum_opt from tensorflow.python.training import optimizer @@ -91,16 +86,18 @@ def __init__(self, weight_decay, **kwargs): self._weight_decay_tensor = None super(DecoupledWeightDecayExtension, self).__init__(**kwargs) - def minimize(self, - loss, - global_step=None, - var_list=None, - gate_gradients=optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - name=None, - grad_loss=None, - decay_var_list=None): + def minimize( + self, + loss, + global_step=None, + var_list=None, + gate_gradients=optimizer.Optimizer.GATE_OP, + aggregation_method=None, + colocate_gradients_with_ops=False, + name=None, + grad_loss=None, + decay_var_list=None + ): """Add operations to minimize `loss` by updating `var_list` with decay. This function is the same as Optimizer.minimize except that it allows to @@ -132,20 +129,19 @@ def minimize(self, """ self._decay_var_list = set(decay_var_list) if decay_var_list else False return super(DecoupledWeightDecayExtension, self).minimize( - loss, - global_step=global_step, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - name=name, - grad_loss=grad_loss) - - def apply_gradients(self, - grads_and_vars, - global_step=None, - name=None, - decay_var_list=None): + loss, + global_step=global_step, + var_list=var_list, + gate_gradients=gate_gradients, + aggregation_method=aggregation_method, + colocate_gradients_with_ops=colocate_gradients_with_ops, + name=name, + grad_loss=grad_loss + ) + + def apply_gradients( + self, grads_and_vars, global_step=None, name=None, decay_var_list=None + ): """Apply gradients to variables and decay the variables. This function is the same as Optimizer.apply_gradients except that it @@ -170,14 +166,16 @@ def apply_gradients(self, """ self._decay_var_list = set(decay_var_list) if decay_var_list else False return super(DecoupledWeightDecayExtension, self).apply_gradients( - grads_and_vars, global_step=global_step, name=name) + grads_and_vars, global_step=global_step, name=name + ) def _prepare(self): weight_decay = self._weight_decay if callable(weight_decay): weight_decay = weight_decay() self._weight_decay_tensor = ops.convert_to_tensor( - weight_decay, name='weight_decay') + weight_decay, name='weight_decay' + ) # Call the optimizers _prepare function. super(DecoupledWeightDecayExtension, self)._prepare() @@ -207,13 +205,15 @@ def _apply_sparse(self, grad, var): scatter_add = state_ops.scatter_add decay_op = self._decay_weights_sparse_op(var, grad.indices, scatter_add) with ops.control_dependencies([decay_op]): - return super(DecoupledWeightDecayExtension, self)._apply_sparse(grad, var) + return super(DecoupledWeightDecayExtension, + self)._apply_sparse(grad, var) def _resource_scatter_add(self, x, i, v, _=None): # last argument allows for one overflow argument, to have the same function # signature as state_ops.scatter_add with ops.control_dependencies( - [resource_variable_ops.resource_scatter_add(x.handle, i, v)]): + [resource_variable_ops.resource_scatter_add(x.handle, i, v)] + ): return x.value() def _resource_apply_sparse(self, grad, var, indices): @@ -262,8 +262,9 @@ def extend_with_decoupled_weight_decay(base_optimizer): and base_optimizer. """ - class OptimizerWithDecoupledWeightDecay(DecoupledWeightDecayExtension, - base_optimizer): + class OptimizerWithDecoupledWeightDecay( + DecoupledWeightDecayExtension, base_optimizer + ): """Base_optimizer with decoupled weight decay. This class computes the update step of `base_optimizer` and @@ -288,8 +289,9 @@ def __init__(self, weight_decay, *args, **kwargs): @tf_export('contrib.opt.MomentumWOptimizer') -class MomentumWOptimizer(DecoupledWeightDecayExtension, - momentum_opt.MomentumOptimizer): +class MomentumWOptimizer( + DecoupledWeightDecayExtension, momentum_opt.MomentumOptimizer +): """Optimizer that implements the Momentum algorithm with weight_decay. This is an implementation of the SGDW optimizer described in "Fixing @@ -311,13 +313,15 @@ class MomentumWOptimizer(DecoupledWeightDecayExtension, ``` """ - def __init__(self, - weight_decay, - learning_rate, - momentum, - use_locking=False, - name='MomentumW', - use_nesterov=False): + def __init__( + self, + weight_decay, + learning_rate, + momentum, + use_locking=False, + name='MomentumW', + use_nesterov=False + ): """Construct a new MomentumW optimizer. For further information see the documentation of the Momentum Optimizer. @@ -342,12 +346,13 @@ def __init__(self, functions. @end_compatibility """ super(MomentumWOptimizer, self).__init__( - weight_decay, - learning_rate=learning_rate, - momentum=momentum, - use_locking=use_locking, - name=name, - use_nesterov=use_nesterov) + weight_decay, + learning_rate=learning_rate, + momentum=momentum, + use_locking=use_locking, + name=name, + use_nesterov=use_nesterov + ) @tf_export('contrib.opt.AdamWOptimizer') @@ -374,14 +379,16 @@ class AdamWOptimizer(DecoupledWeightDecayExtension, adam.AdamOptimizer): ``` """ - def __init__(self, - weight_decay, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - use_locking=False, - name='AdamW'): + def __init__( + self, + weight_decay, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + use_locking=False, + name='AdamW' + ): """Construct a new AdamW optimizer. For further information see the documentation of the Adam Optimizer. @@ -401,13 +408,14 @@ def __init__(self, Defaults to "Adam". """ super(AdamWOptimizer, self).__init__( - weight_decay, - learning_rate=learning_rate, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - use_locking=use_locking, - name=name) + weight_decay, + learning_rate=learning_rate, + beta1=beta1, + beta2=beta2, + epsilon=epsilon, + use_locking=use_locking, + name=name + ) try: @@ -437,14 +445,16 @@ class AdamAsyncWOptimizer(DecoupledWeightDecayExtension, AdamAsyncOptimizer): ``` """ - def __init__(self, - weight_decay, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - use_locking=False, - name='AdamAsyncW'): + def __init__( + self, + weight_decay, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + use_locking=False, + name='AdamAsyncW' + ): """Construct a new AdamW optimizer. For further information see the documentation of the Adam Optimizer. @@ -464,12 +474,13 @@ def __init__(self, Defaults to "Adam". """ super(AdamAsyncWOptimizer, self).__init__( - weight_decay, - learning_rate=learning_rate, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - use_locking=use_locking, - name=name) + weight_decay, + learning_rate=learning_rate, + beta1=beta1, + beta2=beta2, + epsilon=epsilon, + use_locking=use_locking, + name=name + ) except ImportError: pass diff --git a/easy_rec/python/core/easyrec_metrics/__init__.py b/easy_rec/python/core/easyrec_metrics/__init__.py index cba3ebc08..2783b9e27 100644 --- a/easy_rec/python/core/easyrec_metrics/__init__.py +++ b/easy_rec/python/core/easyrec_metrics/__init__.py @@ -1,6 +1,5 @@ import logging import os - import tensorflow as tf from easy_rec.python.utils import pai_util @@ -13,10 +12,10 @@ if distribute_eval == 'True': if pai_util.is_on_pai() or tf.__version__ <= '1.13': logging.info('Will use distribute pai_tf metrics impl') - from easy_rec.python.core.easyrec_metrics import distribute_metrics_impl_pai as metrics_tf + from easy_rec.python.core.easyrec_metrics import distribute_metrics_impl_pai as metrics_tf # NOQA else: logging.info('Will use distribute tf metrics impl') - from easy_rec.python.core.easyrec_metrics import distribute_metrics_impl_tf as metrics_tf + from easy_rec.python.core.easyrec_metrics import distribute_metrics_impl_tf as metrics_tf # NOQA else: if tf.__version__ >= '2.0': from tensorflow.compat.v1 import metrics as metrics_tf diff --git a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py index ef6e10f86..d30f36d8b 100644 --- a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py +++ b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py @@ -14,25 +14,11 @@ # ============================================================================== """Implementation of tf.metrics module.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import confusion_matrix -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import sets -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import weights_broadcast_ops +from tensorflow.python.framework import dtypes, ops, sparse_tensor +from tensorflow.python.ops import array_ops, check_ops, confusion_matrix, control_flow_ops, math_ops, nn, sets, sparse_ops, state_ops, variable_scope, weights_broadcast_ops # NOQA from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import distribution_strategy_context from tensorflow.python.util.deprecation import deprecated @@ -76,14 +62,15 @@ def metric_variable(shape, dtype, validate_shape=True, name=None): """ # Note that synchronization "ON_READ" implies trainable=False. return variable_scope.variable( - lambda: array_ops.zeros(shape, dtype), - collections=[ - ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES - ], - validate_shape=validate_shape, - synchronization=variable_scope.VariableSynchronization.ON_READ, - aggregation=variable_scope.VariableAggregation.SUM, - name=name) + lambda: array_ops.zeros(shape, dtype), + collections=[ + ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES + ], + validate_shape=validate_shape, + synchronization=variable_scope.VariableSynchronization.ON_READ, + aggregation=variable_scope.VariableAggregation.SUM, + name=name + ) def _remove_squeezable_dimensions(predictions, labels, weights): @@ -112,7 +99,8 @@ def _remove_squeezable_dimensions(predictions, labels, weights): predictions = ops.convert_to_tensor(predictions) if labels is not None: labels, predictions = confusion_matrix.remove_squeezable_dimensions( - labels, predictions) + labels, predictions + ) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if weights is None: @@ -139,12 +127,15 @@ def _remove_squeezable_dimensions(predictions, labels, weights): def _maybe_expand_weights(): return control_flow_ops.cond( - math_ops.equal(rank_diff, -1), - lambda: array_ops.expand_dims(weights, [-1]), lambda: weights) + math_ops.equal(rank_diff, -1), + lambda: array_ops.expand_dims(weights, [-1]), lambda: weights + ) # Don't attempt squeeze if it will fail based on static check. - if ((weights_rank is not None) and - (not weights_shape.dims[-1].is_compatible_with(1))): + if ( + (weights_rank is not None) + and (not weights_shape.dims[-1].is_compatible_with(1)) + ): maybe_squeeze_weights = lambda: weights # noqa: E731 else: maybe_squeeze_weights = lambda: array_ops.squeeze( # noqa: E731 @@ -152,14 +143,16 @@ def _maybe_expand_weights(): def _maybe_adjust_weights(): return control_flow_ops.cond( - math_ops.equal(rank_diff, 1), maybe_squeeze_weights, - _maybe_expand_weights) + math_ops.equal(rank_diff, 1), maybe_squeeze_weights, + _maybe_expand_weights + ) # If weights are scalar, do nothing. Otherwise, try to add or remove a # dimension to match predictions. weights = control_flow_ops.cond( - math_ops.equal(weights_rank_tensor, 0), lambda: weights, - _maybe_adjust_weights) + math_ops.equal(weights_rank_tensor, 0), lambda: weights, + _maybe_adjust_weights + ) return predictions, labels, weights @@ -185,12 +178,11 @@ def _maybe_expand_labels(labels, predictions): # If sparse, expand sparse shape. if isinstance(labels, sparse_tensor.SparseTensor): return control_flow_ops.cond( - math_ops.equal( - array_ops.rank(predictions), - array_ops.size(labels.dense_shape) + 1), + math_ops.equal(array_ops.rank(predictions), + array_ops.size(labels.dense_shape) + 1), lambda: sparse_ops.sparse_reshape( # pylint: disable=g-long-lambda labels, - shape=array_ops.concat((labels.dense_shape, (1,)), 0), + shape=array_ops.concat((labels.dense_shape, (1, )), 0), name=scope), lambda: labels) @@ -204,14 +196,16 @@ def _maybe_expand_labels(labels, predictions): if predictions_rank == labels_rank + 1: return array_ops.expand_dims(labels, -1, name=scope) raise ValueError( - 'Unexpected labels shape %s for predictions shape %s.' % - (labels.get_shape(), predictions.get_shape())) + 'Unexpected labels shape %s for predictions shape %s.' % + (labels.get_shape(), predictions.get_shape()) + ) # Otherwise, use dynamic shape. return control_flow_ops.cond( - math_ops.equal(array_ops.rank(predictions), - array_ops.rank(labels) + 1), - lambda: array_ops.expand_dims(labels, -1, name=scope), lambda: labels) + math_ops.equal(array_ops.rank(predictions), + array_ops.rank(labels) + 1), + lambda: array_ops.expand_dims(labels, -1, name=scope), lambda: labels + ) def _safe_div(numerator, denominator, name): @@ -246,14 +240,16 @@ def _safe_scalar_div(numerator, denominator, name): numerator.get_shape().with_rank_at_most(1) denominator.get_shape().with_rank_at_most(1) return control_flow_ops.cond( - math_ops.equal( - array_ops.constant(0.0, dtype=dtypes.float64), denominator), - lambda: array_ops.constant(0.0, dtype=dtypes.float64), - lambda: math_ops.div(numerator, denominator), - name=name) + math_ops.equal(array_ops.constant(0.0, dtype=dtypes.float64), denominator), + lambda: array_ops.constant(0.0, dtype=dtypes.float64), + lambda: math_ops.div(numerator, denominator), + name=name + ) -def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): +def _streaming_confusion_matrix( + labels, predictions, num_classes, weights=None +): """Calculate a streaming confusion matrix. Calculates a confusion matrix. For estimation over a stream of data, @@ -277,9 +273,9 @@ def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): update_op: An operation that increments the confusion matrix. """ # Local variable to accumulate the predictions in the confusion matrix. - total_cm = metric_variable([num_classes, num_classes], - dtypes.float64, - name='total_confusion_matrix') + total_cm = metric_variable( + [num_classes, num_classes], dtypes.float64, name='total_confusion_matrix' + ) # Cast the type to int64 required by confusion_matrix_ops. predictions = math_ops.to_int64(predictions) @@ -298,7 +294,8 @@ def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): # Accumulate the prediction to current confusion matrix. current_cm = confusion_matrix.confusion_matrix( - labels, predictions, num_classes, weights=weights, dtype=dtypes.float64) + labels, predictions, num_classes, weights=weights, dtype=dtypes.float64 + ) update_op = state_ops.assign_add(total_cm, current_cm, use_locking=True) return total_cm, update_op @@ -335,15 +332,19 @@ def fn(distribution, *a): ops.add_to_collections(metrics_collections, metric_value) return metric_value - return distribution_strategy_context.get_tower_context().merge_call(fn, *args) + return distribution_strategy_context.get_tower_context().merge_call( + fn, *args + ) @tf_export('metrics.mean') -def mean(values, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean( + values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the (weighted) mean of the given values. The `mean` function creates two local variables, `total` and `count` @@ -382,8 +383,10 @@ def mean(values, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean is not supported when eager execution ' - 'is enabled.') + raise RuntimeError( + 'tf.metrics.mean is not supported when eager execution ' + 'is enabled.' + ) with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.to_float(values) @@ -395,22 +398,27 @@ def mean(values, num_values = math_ops.to_float(array_ops.size(values)) else: values, _, weights = _remove_squeezable_dimensions( - predictions=values, labels=None, weights=weights) + predictions=values, labels=None, weights=weights + ) weights = weights_broadcast_ops.broadcast_weights( - math_ops.to_float(weights), values) + math_ops.to_float(weights), values + ) values = math_ops.multiply(values, weights) num_values = math_ops.reduce_sum(weights) update_total_op = state_ops.assign_add( - total, math_ops.reduce_sum(values), use_locking=True) + total, math_ops.reduce_sum(values), use_locking=True + ) with ops.control_dependencies([values]): update_count_op = state_ops.assign_add( - count, num_values, use_locking=True) + count, num_values, use_locking=True + ) compute_mean = lambda _, t, c: _safe_div(t, c, 'value') # noqa: E731 - mean_t = _aggregate_across_towers(metrics_collections, compute_mean, total, - count) + mean_t = _aggregate_across_towers( + metrics_collections, compute_mean, total, count + ) update_op = _safe_div(update_total_op, update_count_op, 'update_op') if updates_collections: @@ -420,12 +428,14 @@ def mean(values, @tf_export('metrics.accuracy') -def accuracy(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def accuracy( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Calculates how often `predictions` matches `labels`. The `accuracy` function creates two local variables, `total` and @@ -470,24 +480,27 @@ def accuracy(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.accuracy is not supported when eager ' - 'execution is enabled.') + raise RuntimeError( + 'tf.metrics.accuracy is not supported when eager ' + 'execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) + predictions=predictions, labels=labels, weights=weights + ) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if labels.dtype != predictions.dtype: predictions = math_ops.cast(predictions, labels.dtype) is_correct = math_ops.to_float(math_ops.equal(predictions, labels)) - return mean(is_correct, weights, metrics_collections, updates_collections, - name or 'accuracy') + return mean( + is_correct, weights, metrics_collections, updates_collections, name + or 'accuracy' + ) -def _confusion_matrix_at_thresholds(labels, - predictions, - thresholds, - weights=None, - includes=None): +def _confusion_matrix_at_thresholds( + labels, predictions, thresholds, weights=None, includes=None +): """Computes true_positives, false_negatives, true_negatives, false_positives. This function creates up to four local variables, `true_positives`, @@ -538,27 +551,33 @@ def _confusion_matrix_at_thresholds(labels, if include not in all_includes: raise ValueError('Invalid key: %s.' % include) - with ops.control_dependencies([ + with ops.control_dependencies( + [ check_ops.assert_greater_equal( - predictions, - math_ops.cast(0.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]'), + predictions, + math_ops.cast(0.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]' + ), check_ops.assert_less_equal( - predictions, - math_ops.cast(1.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]') - ]): + predictions, + math_ops.cast(1.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]' + ) + ] + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.to_float(predictions), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.to_float(predictions), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) num_thresholds = len(thresholds) # Reshape predictions and labels. predictions_2d = array_ops.reshape(predictions, [-1, 1]) labels_2d = array_ops.reshape( - math_ops.cast(labels, dtype=dtypes.bool), [1, -1]) + math_ops.cast(labels, dtype=dtypes.bool), [1, -1] + ) # Use static shape if known. num_predictions = predictions_2d.get_shape().as_list()[0] @@ -567,13 +586,15 @@ def _confusion_matrix_at_thresholds(labels, if num_predictions is None: num_predictions = array_ops.shape(predictions_2d)[0] thresh_tiled = array_ops.tile( - array_ops.expand_dims(array_ops.constant(thresholds), [1]), - array_ops.stack([1, num_predictions])) + array_ops.expand_dims(array_ops.constant(thresholds), [1]), + array_ops.stack([1, num_predictions]) + ) # Tile the predictions after thresholding them across different thresholds. pred_is_pos = math_ops.greater( - array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]), - thresh_tiled) + array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]), + thresh_tiled + ) if ('fn' in includes) or ('tn' in includes): pred_is_neg = math_ops.logical_not(pred_is_pos) @@ -584,11 +605,14 @@ def _confusion_matrix_at_thresholds(labels, if weights is not None: weights = weights_broadcast_ops.broadcast_weights( - math_ops.to_float(weights), predictions) + math_ops.to_float(weights), predictions + ) weights_tiled = array_ops.tile( - array_ops.reshape(weights, [1, -1]), [num_thresholds, 1]) + array_ops.reshape(weights, [1, -1]), [num_thresholds, 1] + ) thresh_tiled.get_shape().assert_is_compatible_with( - weights_tiled.get_shape()) + weights_tiled.get_shape() + ) else: weights_tiled = None @@ -596,51 +620,59 @@ def _confusion_matrix_at_thresholds(labels, update_ops = {} if 'tp' in includes: - true_p = metric_variable([num_thresholds], - dtypes.float32, - name='true_positives') + true_p = metric_variable( + [num_thresholds], dtypes.float32, name='true_positives' + ) is_true_positive = math_ops.cast( - math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32) + math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32 + ) if weights_tiled is not None: is_true_positive *= weights_tiled update_ops['tp'] = state_ops.assign_add( - true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True) + true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True + ) values['tp'] = true_p if 'fn' in includes: - false_n = metric_variable([num_thresholds], - dtypes.float32, - name='false_negatives') + false_n = metric_variable( + [num_thresholds], dtypes.float32, name='false_negatives' + ) is_false_negative = math_ops.cast( - math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32) + math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32 + ) if weights_tiled is not None: is_false_negative *= weights_tiled update_ops['fn'] = state_ops.assign_add( - false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True) + false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True + ) values['fn'] = false_n if 'tn' in includes: - true_n = metric_variable([num_thresholds], - dtypes.float32, - name='true_negatives') + true_n = metric_variable( + [num_thresholds], dtypes.float32, name='true_negatives' + ) is_true_negative = math_ops.cast( - math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32) + math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32 + ) if weights_tiled is not None: is_true_negative *= weights_tiled update_ops['tn'] = state_ops.assign_add( - true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True) + true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True + ) values['tn'] = true_n if 'fp' in includes: - false_p = metric_variable([num_thresholds], - dtypes.float32, - name='false_positives') + false_p = metric_variable( + [num_thresholds], dtypes.float32, name='false_positives' + ) is_false_positive = math_ops.cast( - math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32) + math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32 + ) if weights_tiled is not None: is_false_positive *= weights_tiled update_ops['fp'] = state_ops.assign_add( - false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True) + false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True + ) values['fp'] = false_p return values, update_ops @@ -652,15 +684,17 @@ def _aggregate_variable(v, collections): @tf_export('metrics.auc') -def auc(labels, - predictions, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - curve='ROC', - name=None, - summation_method='trapezoidal'): +def auc( + labels, + predictions, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + curve='ROC', + name=None, + summation_method='trapezoidal' +): """Computes the approximate AUC via a Riemann sum. The `auc` function creates four local variables, `true_positives`, @@ -733,21 +767,25 @@ def auc(labels, print('use_distribute_pai_auc') logging.info('use_distribute_pai_auc') if context.executing_eagerly(): - raise RuntimeError('tf.metrics.auc is not supported when eager execution ' - 'is enabled.') - - with variable_scope.variable_scope(name, 'auc', - (labels, predictions, weights)): + raise RuntimeError( + 'tf.metrics.auc is not supported when eager execution ' + 'is enabled.' + ) + + with variable_scope.variable_scope( + name, 'auc', (labels, predictions, weights) + ): if curve != 'ROC' and curve != 'PR': raise ValueError('curve must be either ROC or PR, %s unknown' % (curve)) kepsilon = 1e-7 # to account for floating point imprecisions thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights) + labels, predictions, thresholds, weights + ) # Add epsilons to avoid dividing by 0. epsilon = 1.0e-6 @@ -788,23 +826,27 @@ def interpolate_pr_auc(tp, fp, fn): prec_slope = _safe_div(dtp, p[:num_thresholds - 1] - p[1:], 'prec_slope') intercept = tp[1:] - math_ops.multiply(prec_slope, p[1:]) safe_p_ratio = array_ops.where( - math_ops.logical_and(p[:num_thresholds - 1] > 0, p[1:] > 0), - _safe_div(p[:num_thresholds - 1], p[1:], 'recall_relative_ratio'), - array_ops.ones_like(p[1:])) + math_ops.logical_and(p[:num_thresholds - 1] > 0, p[1:] > 0), + _safe_div(p[:num_thresholds - 1], p[1:], 'recall_relative_ratio'), + array_ops.ones_like(p[1:]) + ) return math_ops.reduce_sum( - _safe_div( - prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)), - tp[1:] + fn[1:], - name='pr_auc_increment'), - name='interpolate_pr_auc') + _safe_div( + prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)), + tp[1:] + fn[1:], + name='pr_auc_increment' + ), + name='interpolate_pr_auc' + ) def compute_auc(tp, fn, tn, fp, name): """Computes the roc-auc or pr-auc based on confusion counts.""" if curve == 'PR': if summation_method == 'trapezoidal': logging.warning( - 'Trapezoidal rule is known to produce incorrect PR-AUCs; ' - 'please switch to "careful_interpolation" instead.') + 'Trapezoidal rule is known to produce incorrect PR-AUCs; ' + 'please switch to "careful_interpolation" instead.' + ) elif summation_method == 'careful_interpolation': # This one is a bit tricky and is handled separately. return interpolate_pr_auc(tp, fp, fn) @@ -821,31 +863,44 @@ def compute_auc(tp, fn, tn, fp, name): # Note that the case ('PR', 'careful_interpolation') has been handled # above. return math_ops.reduce_sum( - math_ops.multiply(x[:num_thresholds - 1] - x[1:], - (y[:num_thresholds - 1] + y[1:]) / 2.), - name=name) + math_ops.multiply( + x[:num_thresholds - 1] - x[1:], + (y[:num_thresholds - 1] + y[1:]) / 2. + ), + name=name + ) elif summation_method == 'minoring': return math_ops.reduce_sum( - math_ops.multiply(x[:num_thresholds - 1] - x[1:], - math_ops.minimum(y[:num_thresholds - 1], y[1:])), - name=name) + math_ops.multiply( + x[:num_thresholds - 1] - x[1:], + math_ops.minimum(y[:num_thresholds - 1], y[1:]) + ), + name=name + ) elif summation_method == 'majoring': return math_ops.reduce_sum( - math_ops.multiply(x[:num_thresholds - 1] - x[1:], - math_ops.maximum(y[:num_thresholds - 1], y[1:])), - name=name) + math_ops.multiply( + x[:num_thresholds - 1] - x[1:], + math_ops.maximum(y[:num_thresholds - 1], y[1:]) + ), + name=name + ) else: raise ValueError('Invalid summation_method: %s' % summation_method) # sum up the areas of all the trapeziums def compute_auc_value(_, values): - return compute_auc(values['tp'], values['fn'], values['tn'], values['fp'], - 'value') - - auc_value = _aggregate_across_towers(metrics_collections, compute_auc_value, - values) - update_op = compute_auc(update_ops['tp'], update_ops['fn'], - update_ops['tn'], update_ops['fp'], 'update_op') + return compute_auc( + values['tp'], values['fn'], values['tn'], values['fp'], 'value' + ) + + auc_value = _aggregate_across_towers( + metrics_collections, compute_auc_value, values + ) + update_op = compute_auc( + update_ops['tp'], update_ops['fn'], update_ops['tn'], update_ops['fp'], + 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -854,12 +909,14 @@ def compute_auc_value(_, values): @tf_export('metrics.mean_absolute_error') -def mean_absolute_error(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_absolute_error( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the mean absolute error between the labels and predictions. The `mean_absolute_error` function creates two local variables, @@ -904,24 +961,31 @@ def mean_absolute_error(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_absolute_error is not supported ' - 'when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.mean_absolute_error is not supported ' + 'when eager execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) + predictions=predictions, labels=labels, weights=weights + ) absolute_errors = math_ops.abs(predictions - labels) - return mean(absolute_errors, weights, metrics_collections, - updates_collections, name or 'mean_absolute_error') + return mean( + absolute_errors, weights, metrics_collections, updates_collections, name + or 'mean_absolute_error' + ) @tf_export('metrics.mean_cosine_distance') -def mean_cosine_distance(labels, - predictions, - dim, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_cosine_distance( + labels, + predictions, + dim, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the cosine distance between the labels and predictions. The `mean_cosine_distance` function creates two local variables, @@ -964,18 +1028,23 @@ def mean_cosine_distance(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_cosine_distance is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.mean_cosine_distance is not supported when ' + 'eager execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) + predictions=predictions, labels=labels, weights=weights + ) radial_diffs = math_ops.multiply(predictions, labels) radial_diffs = math_ops.reduce_sum( - radial_diffs, reduction_indices=[ - dim, - ], keepdims=True) - mean_distance, update_op = mean(radial_diffs, weights, None, None, name or - 'mean_cosine_distance') + radial_diffs, reduction_indices=[ + dim, + ], keepdims=True + ) + mean_distance, update_op = mean( + radial_diffs, weights, None, None, name or 'mean_cosine_distance' + ) mean_distance = math_ops.subtract(1.0, mean_distance) update_op = math_ops.subtract(1.0, update_op) @@ -989,13 +1058,15 @@ def mean_cosine_distance(labels, @tf_export('metrics.mean_per_class_accuracy') -def mean_per_class_accuracy(labels, - predictions, - num_classes, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_per_class_accuracy( + labels, + predictions, + num_classes, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Calculates the mean of the per-class accuracies. Calculates the accuracy for each class, then takes the mean of that. @@ -1037,11 +1108,14 @@ def mean_per_class_accuracy(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_per_class_accuracy is not supported ' - 'when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'mean_accuracy', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.mean_per_class_accuracy is not supported ' + 'when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'mean_accuracy', (predictions, labels, weights) + ): labels = math_ops.to_int64(labels) # Flatten the input if its rank > 1. @@ -1072,19 +1146,22 @@ def mean_per_class_accuracy(labels, ones *= weights update_total_op = state_ops.scatter_add( - total, labels, ones, use_locking=True) + total, labels, ones, use_locking=True + ) update_count_op = state_ops.scatter_add( - count, labels, is_correct, use_locking=True) + count, labels, is_correct, use_locking=True + ) def compute_mean_accuracy(_, count, total): per_class_accuracy = _safe_div(count, total, None) mean_accuracy_v = math_ops.reduce_mean( - per_class_accuracy, name='mean_accuracy') + per_class_accuracy, name='mean_accuracy' + ) return mean_accuracy_v - mean_accuracy_v = _aggregate_across_towers(metrics_collections, - compute_mean_accuracy, count, - total) + mean_accuracy_v = _aggregate_across_towers( + metrics_collections, compute_mean_accuracy, count, total + ) update_op = _safe_div(update_count_op, update_total_op, name='update_op') if updates_collections: @@ -1094,13 +1171,15 @@ def compute_mean_accuracy(_, count, total): @tf_export('metrics.mean_iou') -def mean_iou(labels, - predictions, - num_classes, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_iou( + labels, + predictions, + num_classes, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Calculate per-step mean Intersection-Over-Union (mIOU). Mean Intersection-Over-Union is a common evaluation metric for @@ -1146,16 +1225,20 @@ def mean_iou(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_iou is not supported when ' - 'eager execution is enabled.') - - with variable_scope.variable_scope(name, 'mean_iou', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.mean_iou is not supported when ' + 'eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'mean_iou', (predictions, labels, weights) + ): # Check if shape is compatible. predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - total_cm, update_op = _streaming_confusion_matrix(labels, predictions, - num_classes, weights) + total_cm, update_op = _streaming_confusion_matrix( + labels, predictions, num_classes, weights + ) def compute_mean_iou(_, total_cm): """Compute the mean intersection-over-union via the confusion matrix.""" @@ -1168,25 +1251,30 @@ def compute_mean_iou(_, total_cm): # label or prediction tensor. If the denominator is 0, we need to # ignore the class. num_valid_entries = math_ops.reduce_sum( - math_ops.cast( - math_ops.not_equal(denominator, 0), dtype=dtypes.float32)) + math_ops.cast( + math_ops.not_equal(denominator, 0), dtype=dtypes.float32 + ) + ) # If the value of the denominator is 0, set it to 1 to avoid # zero division. denominator = array_ops.where( - math_ops.greater(denominator, 0), denominator, - array_ops.ones_like(denominator)) + math_ops.greater(denominator, 0), denominator, + array_ops.ones_like(denominator) + ) iou = math_ops.div(cm_diag, denominator) # If the number of valid entries is 0 (no classes) we return 0. result = array_ops.where( - math_ops.greater(num_valid_entries, 0), - math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, 0) + math_ops.greater(num_valid_entries, 0), + math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, 0 + ) return result # TODO(priyag): Use outside_compilation if in TPU context. - mean_iou_v = _aggregate_across_towers(metrics_collections, compute_mean_iou, - total_cm) + mean_iou_v = _aggregate_across_towers( + metrics_collections, compute_mean_iou, total_cm + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1195,13 +1283,15 @@ def compute_mean_iou(_, total_cm): @tf_export('metrics.mean_relative_error') -def mean_relative_error(labels, - predictions, - normalizer, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_relative_error( + labels, + predictions, + normalizer, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the mean relative error by normalizing with the given values. The `mean_relative_error` function creates two local variables, @@ -1247,29 +1337,38 @@ def mean_relative_error(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_relative_error is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.mean_relative_error is not supported when ' + 'eager execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) + predictions=predictions, labels=labels, weights=weights + ) predictions, normalizer = confusion_matrix.remove_squeezable_dimensions( - predictions, normalizer) + predictions, normalizer + ) predictions.get_shape().assert_is_compatible_with(normalizer.get_shape()) relative_errors = array_ops.where( - math_ops.equal(normalizer, 0.0), array_ops.zeros_like(labels), - math_ops.div(math_ops.abs(labels - predictions), normalizer)) - return mean(relative_errors, weights, metrics_collections, - updates_collections, name or 'mean_relative_error') + math_ops.equal(normalizer, 0.0), array_ops.zeros_like(labels), + math_ops.div(math_ops.abs(labels - predictions), normalizer) + ) + return mean( + relative_errors, weights, metrics_collections, updates_collections, name + or 'mean_relative_error' + ) @tf_export('metrics.mean_squared_error') -def mean_squared_error(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_squared_error( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the mean squared error between the labels and predictions. The `mean_squared_error` function creates two local variables, @@ -1314,22 +1413,29 @@ def mean_squared_error(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_squared_error is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.mean_squared_error is not supported when ' + 'eager execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) + predictions=predictions, labels=labels, weights=weights + ) squared_error = math_ops.square(labels - predictions) - return mean(squared_error, weights, metrics_collections, updates_collections, - name or 'mean_squared_error') + return mean( + squared_error, weights, metrics_collections, updates_collections, name + or 'mean_squared_error' + ) @tf_export('metrics.mean_tensor') -def mean_tensor(values, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_tensor( + values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the element-wise (weighted) mean of the given tensors. In contrast to the `mean` function which returns a scalar with the @@ -1372,34 +1478,42 @@ def mean_tensor(values, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_tensor is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.mean_tensor is not supported when ' + 'eager execution is enabled.' + ) with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.to_float(values) total = metric_variable( - values.get_shape(), dtypes.float32, name='total_tensor') + values.get_shape(), dtypes.float32, name='total_tensor' + ) count = metric_variable( - values.get_shape(), dtypes.float32, name='count_tensor') + values.get_shape(), dtypes.float32, name='count_tensor' + ) num_values = array_ops.ones_like(values) if weights is not None: values, _, weights = _remove_squeezable_dimensions( - predictions=values, labels=None, weights=weights) + predictions=values, labels=None, weights=weights + ) weights = weights_broadcast_ops.broadcast_weights( - math_ops.to_float(weights), values) + math_ops.to_float(weights), values + ) values = math_ops.multiply(values, weights) num_values = math_ops.multiply(num_values, weights) update_total_op = state_ops.assign_add(total, values, use_locking=True) with ops.control_dependencies([values]): update_count_op = state_ops.assign_add( - count, num_values, use_locking=True) + count, num_values, use_locking=True + ) compute_mean = lambda _, t, c: _safe_div(t, c, 'value') # noqa: E731 - mean_t = _aggregate_across_towers(metrics_collections, compute_mean, total, - count) + mean_t = _aggregate_across_towers( + metrics_collections, compute_mean, total, count + ) update_op = _safe_div(update_total_op, update_count_op, 'update_op') if updates_collections: @@ -1409,12 +1523,14 @@ def mean_tensor(values, @tf_export('metrics.percentage_below') -def percentage_below(values, - threshold, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def percentage_below( + values, + threshold, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the percentage of values less than the given threshold. The `percentage_below` function creates two local variables, @@ -1454,18 +1570,21 @@ def percentage_below(values, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.percentage_below is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.percentage_below is not supported when ' + 'eager execution is enabled.' + ) is_below_threshold = math_ops.to_float(math_ops.less(values, threshold)) - return mean(is_below_threshold, weights, metrics_collections, - updates_collections, name or 'percentage_below_threshold') + return mean( + is_below_threshold, weights, metrics_collections, updates_collections, name + or 'percentage_below_threshold' + ) -def _count_condition(values, - weights=None, - metrics_collections=None, - updates_collections=None): +def _count_condition( + values, weights=None, metrics_collections=None, updates_collections=None +): """Sums the weights of cases where the given values are True. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1495,14 +1614,16 @@ def _count_condition(values, values = math_ops.to_float(values) if weights is not None: with ops.control_dependencies( - (check_ops.assert_rank_in(weights, (0, array_ops.rank(values))),)): + (check_ops.assert_rank_in(weights, (0, array_ops.rank(values))), ) + ): weights = math_ops.to_float(weights) values = math_ops.multiply(values, weights) value_tensor = _aggregate_variable(count, metrics_collections) update_op = state_ops.assign_add( - count, math_ops.reduce_sum(values), use_locking=True) + count, math_ops.reduce_sum(values), use_locking=True + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1510,12 +1631,14 @@ def _count_condition(values, @tf_export('metrics.false_negatives') -def false_negatives(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def false_negatives( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the total number of false negatives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1545,30 +1668,38 @@ def false_negatives(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_negatives is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.false_negatives is not supported when ' + 'eager execution is enabled.' + ) - with variable_scope.variable_scope(name, 'false_negatives', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'false_negatives', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) is_false_negative = math_ops.logical_and( - math_ops.equal(labels, True), math_ops.equal(predictions, False)) - return _count_condition(is_false_negative, weights, metrics_collections, - updates_collections) + math_ops.equal(labels, True), math_ops.equal(predictions, False) + ) + return _count_condition( + is_false_negative, weights, metrics_collections, updates_collections + ) @tf_export('metrics.false_negatives_at_thresholds') -def false_negatives_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def false_negatives_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes false negatives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1601,13 +1732,17 @@ def false_negatives_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_negatives_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'false_negatives', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.false_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'false_negatives', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fn',)) + labels, predictions, thresholds, weights=weights, includes=('fn', ) + ) fn_value = _aggregate_variable(values['fn'], metrics_collections) @@ -1618,12 +1753,14 @@ def false_negatives_at_thresholds(labels, @tf_export('metrics.false_positives') -def false_positives(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def false_positives( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Sum the weights of false positives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1654,30 +1791,38 @@ def false_positives(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_positives is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.false_positives is not supported when ' + 'eager execution is enabled.' + ) - with variable_scope.variable_scope(name, 'false_positives', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'false_positives', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) is_false_positive = math_ops.logical_and( - math_ops.equal(labels, False), math_ops.equal(predictions, True)) - return _count_condition(is_false_positive, weights, metrics_collections, - updates_collections) + math_ops.equal(labels, False), math_ops.equal(predictions, True) + ) + return _count_condition( + is_false_positive, weights, metrics_collections, updates_collections + ) @tf_export('metrics.false_positives_at_thresholds') -def false_positives_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def false_positives_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes false positives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1710,13 +1855,17 @@ def false_positives_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_positives_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'false_positives', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.false_positives_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'false_positives', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fp',)) + labels, predictions, thresholds, weights=weights, includes=('fp', ) + ) fp_value = _aggregate_variable(values['fp'], metrics_collections) @@ -1727,12 +1876,14 @@ def false_positives_at_thresholds(labels, @tf_export('metrics.true_negatives') -def true_negatives(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def true_negatives( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Sum the weights of true_negatives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1763,30 +1914,38 @@ def true_negatives(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_negatives is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.true_negatives is not ' + 'supported when eager execution is enabled.' + ) - with variable_scope.variable_scope(name, 'true_negatives', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'true_negatives', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) is_true_negative = math_ops.logical_and( - math_ops.equal(labels, False), math_ops.equal(predictions, False)) - return _count_condition(is_true_negative, weights, metrics_collections, - updates_collections) + math_ops.equal(labels, False), math_ops.equal(predictions, False) + ) + return _count_condition( + is_true_negative, weights, metrics_collections, updates_collections + ) @tf_export('metrics.true_negatives_at_thresholds') -def true_negatives_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def true_negatives_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes true negatives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1819,13 +1978,17 @@ def true_negatives_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_negatives_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'true_negatives', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.true_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'true_negatives', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tn',)) + labels, predictions, thresholds, weights=weights, includes=('tn', ) + ) tn_value = _aggregate_variable(values['tn'], metrics_collections) @@ -1836,12 +1999,14 @@ def true_negatives_at_thresholds(labels, @tf_export('metrics.true_positives') -def true_positives(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def true_positives( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Sum the weights of true_positives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1872,30 +2037,38 @@ def true_positives(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_positives is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.true_positives is not ' + 'supported when eager execution is enabled.' + ) - with variable_scope.variable_scope(name, 'true_positives', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'true_positives', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) is_true_positive = math_ops.logical_and( - math_ops.equal(labels, True), math_ops.equal(predictions, True)) - return _count_condition(is_true_positive, weights, metrics_collections, - updates_collections) + math_ops.equal(labels, True), math_ops.equal(predictions, True) + ) + return _count_condition( + is_true_positive, weights, metrics_collections, updates_collections + ) @tf_export('metrics.true_positives_at_thresholds') -def true_positives_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def true_positives_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes true positives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1928,13 +2101,17 @@ def true_positives_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_positives_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'true_positives', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.true_positives_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'true_positives', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tp',)) + labels, predictions, thresholds, weights=weights, includes=('tp', ) + ) tp_value = _aggregate_variable(values['tp'], metrics_collections) @@ -1945,12 +2122,14 @@ def true_positives_at_thresholds(labels, @tf_export('metrics.precision') -def precision(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def precision( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the precision of the predictions with respect to the labels. The `precision` function creates two local variables, @@ -1995,44 +2174,53 @@ def precision(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.precision is not ' + 'supported when eager execution is enabled.' + ) - with variable_scope.variable_scope(name, 'precision', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'precision', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) true_p, true_positives_update_op = true_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None + ) false_p, false_positives_update_op = false_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None + ) def compute_precision(tp, fp, name): return array_ops.where( - math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name) + math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name + ) def once_across_towers(_, true_p, false_p): return compute_precision(true_p, false_p, 'value') - p = _aggregate_across_towers(metrics_collections, once_across_towers, - true_p, false_p) + p = _aggregate_across_towers( + metrics_collections, once_across_towers, true_p, false_p + ) - update_op = compute_precision(true_positives_update_op, - false_positives_update_op, 'update_op') + update_op = compute_precision( + true_positives_update_op, false_positives_update_op, 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2040,13 +2228,15 @@ def once_across_towers(_, true_p, false_p): @tf_export('metrics.precision_at_thresholds') -def precision_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def precision_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes precision values for different `thresholds` on `predictions`. The `precision_at_thresholds` function creates four local variables, @@ -2092,13 +2282,17 @@ def precision_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'precision_at_thresholds', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.precision_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'precision_at_thresholds', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights, includes=('tp', 'fp')) + labels, predictions, thresholds, weights, includes=('tp', 'fp') + ) # Avoid division by zero. epsilon = 1e-7 @@ -2109,11 +2303,13 @@ def compute_precision(tp, fp, name): def precision_across_towers(_, values): return compute_precision(values['tp'], values['fp'], 'value') - prec = _aggregate_across_towers(metrics_collections, - precision_across_towers, values) + prec = _aggregate_across_towers( + metrics_collections, precision_across_towers, values + ) - update_op = compute_precision(update_ops['tp'], update_ops['fp'], - 'update_op') + update_op = compute_precision( + update_ops['tp'], update_ops['fp'], 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2121,12 +2317,14 @@ def precision_across_towers(_, values): @tf_export('metrics.recall') -def recall(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def recall( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the recall of the predictions with respect to the labels. The `recall` function creates two local variables, `true_positives` @@ -2169,44 +2367,53 @@ def recall(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall is not supported is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'recall', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.recall is not supported is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'recall', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) true_p, true_positives_update_op = true_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None + ) false_n, false_negatives_update_op = false_negatives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None + ) def compute_recall(true_p, false_n, name): return array_ops.where( - math_ops.greater(true_p + false_n, 0), - math_ops.div(true_p, true_p + false_n), 0, name) + math_ops.greater(true_p + false_n, 0), + math_ops.div(true_p, true_p + false_n), 0, name + ) def once_across_towers(_, true_p, false_n): return compute_recall(true_p, false_n, 'value') - rec = _aggregate_across_towers(metrics_collections, once_across_towers, - true_p, false_n) + rec = _aggregate_across_towers( + metrics_collections, once_across_towers, true_p, false_n + ) - update_op = compute_recall(true_positives_update_op, - false_negatives_update_op, 'update_op') + update_op = compute_recall( + true_positives_update_op, false_negatives_update_op, 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2236,8 +2443,9 @@ def _select_class_id(ids, selected_id): """ ids = sparse_tensor.convert_to_tensor_or_sparse_tensor(ids) if isinstance(ids, sparse_tensor.SparseTensor): - return sparse_ops.sparse_retain(ids, math_ops.equal(ids.values, - selected_id)) + return sparse_ops.sparse_retain( + ids, math_ops.equal(ids.values, selected_id) + ) # TODO(ptucker): Make this more efficient, maybe add a sparse version of # tf.equal and tf.reduce_any? @@ -2246,14 +2454,17 @@ def _select_class_id(ids, selected_id): ids_shape = array_ops.shape(ids, out_type=dtypes.int64) ids_last_dim = array_ops.size(ids_shape) - 1 filled_selected_id_shape = math_ops.reduced_shape( - ids_shape, array_ops.reshape(ids_last_dim, [1])) + ids_shape, array_ops.reshape(ids_last_dim, [1]) + ) # Intersect `ids` with the selected ID. - filled_selected_id = array_ops.fill(filled_selected_id_shape, - math_ops.to_int64(selected_id)) + filled_selected_id = array_ops.fill( + filled_selected_id_shape, math_ops.to_int64(selected_id) + ) result = sets.set_intersection(filled_selected_id, ids) return sparse_tensor.SparseTensor( - indices=result.indices, values=result.values, dense_shape=ids_shape) + indices=result.indices, values=result.values, dense_shape=ids_shape + ) def _maybe_select_class_id(labels, predictions_idx, selected_id=None): @@ -2275,15 +2486,15 @@ def _maybe_select_class_id(labels, predictions_idx, selected_id=None): """ if selected_id is None: return labels, predictions_idx - return (_select_class_id(labels, selected_id), - _select_class_id(predictions_idx, selected_id)) + return ( + _select_class_id(labels, selected_id), + _select_class_id(predictions_idx, selected_id) + ) -def _sparse_true_positive_at_k(labels, - predictions_idx, - class_id=None, - weights=None, - name=None): +def _sparse_true_positive_at_k( + labels, predictions_idx, class_id=None, weights=None, name=None +): """Calculates true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2310,26 +2521,26 @@ def _sparse_true_positive_at_k(labels, Returns: A [D1, ... DN] `Tensor` of true positive counts. """ - with ops.name_scope(name, 'true_positives', - (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, - class_id) + with ops.name_scope( + name, 'true_positives', (predictions_idx, labels, weights) + ): + labels, predictions_idx = _maybe_select_class_id( + labels, predictions_idx, class_id + ) tp = sets.set_size(sets.set_intersection(predictions_idx, labels)) tp = math_ops.to_double(tp) if weights is not None: with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, tp),)): + (weights_broadcast_ops.assert_broadcastable(weights, tp), ) + ): weights = math_ops.to_double(weights) tp = math_ops.multiply(tp, weights) return tp -def _streaming_sparse_true_positive_at_k(labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - name=None): +def _streaming_sparse_true_positive_at_k( + labels, predictions_idx, k=None, class_id=None, weights=None, name=None +): """Calculates weighted per step true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2362,24 +2573,27 @@ def _streaming_sparse_true_positive_at_k(labels, Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope(name, _at_k_name('true_positive', k, class_id=class_id), - (predictions_idx, labels, weights)) as scope: + with ops.name_scope( + name, _at_k_name('true_positive', k, class_id=class_id), + (predictions_idx, labels, weights) + ) as scope: tp = _sparse_true_positive_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights + ) batch_total_tp = math_ops.to_double(math_ops.reduce_sum(tp)) var = metric_variable([], dtypes.float64, name=scope) return var, state_ops.assign_add( - var, batch_total_tp, name='update', use_locking=True) + var, batch_total_tp, name='update', use_locking=True + ) -def _sparse_false_negative_at_k(labels, - predictions_idx, - class_id=None, - weights=None): +def _sparse_false_negative_at_k( + labels, predictions_idx, class_id=None, weights=None +): """Calculates false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2405,27 +2619,28 @@ def _sparse_false_negative_at_k(labels, Returns: A [D1, ... DN] `Tensor` of false negative counts. """ - with ops.name_scope(None, 'false_negatives', - (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, - class_id) + with ops.name_scope( + None, 'false_negatives', (predictions_idx, labels, weights) + ): + labels, predictions_idx = _maybe_select_class_id( + labels, predictions_idx, class_id + ) fn = sets.set_size( - sets.set_difference(predictions_idx, labels, aminusb=False)) + sets.set_difference(predictions_idx, labels, aminusb=False) + ) fn = math_ops.to_double(fn) if weights is not None: with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, fn),)): + (weights_broadcast_ops.assert_broadcastable(weights, fn), ) + ): weights = math_ops.to_double(weights) fn = math_ops.multiply(fn, weights) return fn -def _streaming_sparse_false_negative_at_k(labels, - predictions_idx, - k, - class_id=None, - weights=None, - name=None): +def _streaming_sparse_false_negative_at_k( + labels, predictions_idx, k, class_id=None, weights=None, name=None +): """Calculates weighted per step false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2458,29 +2673,35 @@ def _streaming_sparse_false_negative_at_k(labels, Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope(name, _at_k_name('false_negative', k, class_id=class_id), - (predictions_idx, labels, weights)) as scope: + with ops.name_scope( + name, _at_k_name('false_negative', k, class_id=class_id), + (predictions_idx, labels, weights) + ) as scope: fn = _sparse_false_negative_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights + ) batch_total_fn = math_ops.to_double(math_ops.reduce_sum(fn)) var = metric_variable([], dtypes.float64, name=scope) return var, state_ops.assign_add( - var, batch_total_fn, name='update', use_locking=True) + var, batch_total_fn, name='update', use_locking=True + ) @tf_export('metrics.recall_at_k') -def recall_at_k(labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def recall_at_k( + labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes recall@k of the predictions with respect to sparse labels. If `class_id` is specified, we calculate recall by considering only the @@ -2548,32 +2769,39 @@ def recall_at_k(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall_at_k is not ' - 'supported when eager execution is enabled.') - - with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), - (predictions, labels, weights)) as scope: + raise RuntimeError( + 'tf.metrics.recall_at_k is not ' + 'supported when eager execution is enabled.' + ) + + with ops.name_scope( + name, _at_k_name('recall', k, class_id=class_id), + (predictions, labels, weights) + ) as scope: _, top_k_idx = nn.top_k(predictions, k) return recall_at_top_k( - labels=labels, - predictions_idx=top_k_idx, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope) + labels=labels, + predictions_idx=top_k_idx, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope + ) @tf_export('metrics.recall_at_top_k') -def recall_at_top_k(labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def recall_at_top_k( + labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes recall@k of top-k predictions with respect to sparse labels. Differs from `recall_at_k` in that predictions must be in the form of top `k` @@ -2619,44 +2847,52 @@ class indices, whereas `recall_at_k` expects logits. Refer to `recall_at_k` `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), - (predictions_idx, labels, weights)) as scope: + with ops.name_scope( + name, _at_k_name('recall', k, class_id=class_id), + (predictions_idx, labels, weights) + ) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.to_int64(predictions_idx) tp, tp_update = _streaming_sparse_true_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights + ) fn, fn_update = _streaming_sparse_false_negative_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights + ) def compute_recall(_, tp, fn): return math_ops.div(tp, math_ops.add(tp, fn), name=scope) - metric = _aggregate_across_towers(metrics_collections, compute_recall, tp, - fn) + metric = _aggregate_across_towers( + metrics_collections, compute_recall, tp, fn + ) update = math_ops.div( - tp_update, math_ops.add(tp_update, fn_update), name='update') + tp_update, math_ops.add(tp_update, fn_update), name='update' + ) if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @tf_export('metrics.recall_at_thresholds') -def recall_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def recall_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes various recall values for different `thresholds` on `predictions`. The `recall_at_thresholds` function creates four local variables, @@ -2700,13 +2936,17 @@ def recall_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'recall_at_thresholds', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.recall_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'recall_at_thresholds', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights, includes=('tp', 'fn')) + labels, predictions, thresholds, weights, includes=('tp', 'fn') + ) # Avoid division by zero. epsilon = 1e-7 @@ -2717,8 +2957,9 @@ def compute_recall(tp, fn, name): def recall_across_towers(_, values): return compute_recall(values['tp'], values['fn'], 'value') - rec = _aggregate_across_towers(metrics_collections, recall_across_towers, - values) + rec = _aggregate_across_towers( + metrics_collections, recall_across_towers, values + ) update_op = compute_recall(update_ops['tp'], update_ops['fn'], 'update_op') if updates_collections: @@ -2728,12 +2969,14 @@ def recall_across_towers(_, values): @tf_export('metrics.root_mean_squared_error') -def root_mean_squared_error(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def root_mean_squared_error( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the root mean squared error between the labels and predictions. The `root_mean_squared_error` function creates two local variables, @@ -2778,14 +3021,17 @@ def root_mean_squared_error(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.root_mean_squared_error is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.root_mean_squared_error is not ' + 'supported when eager execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) - mse, update_mse_op = mean_squared_error(labels, predictions, weights, None, - None, name or - 'root_mean_squared_error') + predictions=predictions, labels=labels, weights=weights + ) + mse, update_mse_op = mean_squared_error( + labels, predictions, weights, None, None, name or 'root_mean_squared_error' + ) once_across_towers = lambda _, mse: math_ops.sqrt(mse) # noqa: E731 rmse = _aggregate_across_towers(metrics_collections, once_across_towers, mse) @@ -2798,14 +3044,16 @@ def root_mean_squared_error(labels, @tf_export('metrics.sensitivity_at_specificity') -def sensitivity_at_specificity(labels, - predictions, - specificity, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None): +def sensitivity_at_specificity( + labels, + predictions, + specificity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the specificity at a given sensitivity. The `sensitivity_at_specificity` function creates four local @@ -2857,22 +3105,26 @@ def sensitivity_at_specificity(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sensitivity_at_specificity is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.sensitivity_at_specificity is not ' + 'supported when eager execution is enabled.' + ) if specificity < 0 or specificity > 1: raise ValueError('`specificity` must be in the range [0, 1].') - with variable_scope.variable_scope(name, 'sensitivity_at_specificity', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'sensitivity_at_specificity', (predictions, labels, weights) + ): kepsilon = 1e-7 # to account for floating point imprecisions thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights) + labels, predictions, thresholds, weights + ) def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): specificities = math_ops.div(tn, tn + fp + kepsilon) @@ -2880,22 +3132,23 @@ def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the sensitivity: - return math_ops.div(tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, - name) + return math_ops.div( + tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, name + ) def sensitivity_across_towers(_, values): - return compute_sensitivity_at_specificity(values['tp'], values['tn'], - values['fp'], values['fn'], - 'value') - - sensitivity = _aggregate_across_towers(metrics_collections, - sensitivity_across_towers, values) - - update_op = compute_sensitivity_at_specificity(update_ops['tp'], - update_ops['tn'], - update_ops['fp'], - update_ops['fn'], - 'update_op') + return compute_sensitivity_at_specificity( + values['tp'], values['tn'], values['fp'], values['fn'], 'value' + ) + + sensitivity = _aggregate_across_towers( + metrics_collections, sensitivity_across_towers, values + ) + + update_op = compute_sensitivity_at_specificity( + update_ops['tp'], update_ops['tn'], update_ops['fp'], update_ops['fn'], + 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2923,37 +3176,45 @@ def _expand_and_tile(tensor, multiple, dim=0, name=None): """ if multiple < 1: raise ValueError('Invalid multiple %s, must be > 0.' % multiple) - with ops.name_scope(name, 'expand_and_tile', - (tensor, multiple, dim)) as scope: + with ops.name_scope( + name, 'expand_and_tile', (tensor, multiple, dim) + ) as scope: # Sparse. tensor = sparse_tensor.convert_to_tensor_or_sparse_tensor(tensor) if isinstance(tensor, sparse_tensor.SparseTensor): if dim < 0: expand_dims = array_ops.reshape( - array_ops.size(tensor.dense_shape) + dim, [1]) + array_ops.size(tensor.dense_shape) + dim, [1] + ) else: expand_dims = [dim] expanded_shape = array_ops.concat( - (array_ops.slice(tensor.dense_shape, [0], expand_dims), [1], - array_ops.slice(tensor.dense_shape, expand_dims, [-1])), - 0, - name='expanded_shape') + ( + array_ops.slice(tensor.dense_shape, [0], expand_dims), [1], + array_ops.slice(tensor.dense_shape, expand_dims, [-1]) + ), + 0, + name='expanded_shape' + ) expanded = sparse_ops.sparse_reshape( - tensor, shape=expanded_shape, name='expand') + tensor, shape=expanded_shape, name='expand' + ) if multiple == 1: return expanded return sparse_ops.sparse_concat( - dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope) + dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope + ) # Dense. expanded = array_ops.expand_dims( - tensor, dim if (dim >= 0) else (dim - 1), name='expand') + tensor, dim if (dim >= 0) else (dim - 1), name='expand' + ) if multiple == 1: return expanded ones = array_ops.ones_like(array_ops.shape(tensor)) - tile_multiples = array_ops.concat((ones[:dim], (multiple,), ones[dim:]), - 0, - name='multiples') + tile_multiples = array_ops.concat( + (ones[:dim], (multiple, ), ones[dim:]), 0, name='multiples' + ) return array_ops.tile(expanded, tile_multiples, name=scope) @@ -2979,7 +3240,7 @@ def _num_relevant(labels, k): """ if k < 1: raise ValueError('Invalid k=%s.' % k) - with ops.name_scope(None, 'num_relevant', (labels,)) as scope: + with ops.name_scope(None, 'num_relevant', (labels, )) as scope: # For SparseTensor, calculate separate count for each row. labels = sparse_tensor.convert_to_tensor_or_sparse_tensor(labels) if isinstance(labels, sparse_tensor.SparseTensor): @@ -3025,9 +3286,12 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): Raises: ValueError: if the last dimension of predictions_idx is not set. """ - with ops.name_scope(None, 'average_precision', - (predictions_idx, labels)) as scope: - predictions_idx = math_ops.to_int64(predictions_idx, name='predictions_idx') + with ops.name_scope( + None, 'average_precision', (predictions_idx, labels) + ) as scope: + predictions_idx = math_ops.to_int64( + predictions_idx, name='predictions_idx' + ) if predictions_idx.get_shape().ndims == 0: raise ValueError('The rank of predictions_idx must be at least 1.') k = predictions_idx.get_shape().as_list()[-1] @@ -3039,11 +3303,13 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # prediction for each k, so we can calculate separate true positive values # for each k. predictions_idx_per_k = array_ops.expand_dims( - predictions_idx, -1, name='predictions_idx_per_k') + predictions_idx, -1, name='predictions_idx_per_k' + ) # Replicate labels k times to produce [D1, ... DN, k, num_labels] tensor. labels_per_k = _expand_and_tile( - labels, multiple=k, dim=-1, name='labels_per_k') + labels, multiple=k, dim=-1, name='labels_per_k' + ) # The following tensors are all of shape [D1, ... DN, k], containing values # per row, per k value. @@ -3058,22 +3324,27 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # `relevant_precision_per_k` (float64) - Relevant precisions; i.e., # precisions at all k for which relevance indicator is true. relevant_per_k = _sparse_true_positive_at_k( - labels_per_k, predictions_idx_per_k, name='relevant_per_k') + labels_per_k, predictions_idx_per_k, name='relevant_per_k' + ) tp_per_k = math_ops.cumsum(relevant_per_k, axis=-1, name='tp_per_k') retrieved_per_k = math_ops.cumsum( - array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k') + array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k' + ) precision_per_k = math_ops.div( - math_ops.to_double(tp_per_k), - math_ops.to_double(retrieved_per_k), - name='precision_per_k') + math_ops.to_double(tp_per_k), + math_ops.to_double(retrieved_per_k), + name='precision_per_k' + ) relevant_precision_per_k = math_ops.multiply( - precision_per_k, - math_ops.to_double(relevant_per_k), - name='relevant_precision_per_k') + precision_per_k, + math_ops.to_double(relevant_per_k), + name='relevant_precision_per_k' + ) # Reduce along k dimension to get the sum, yielding a [D1, ... DN] tensor. precision_sum = math_ops.reduce_sum( - relevant_precision_per_k, reduction_indices=(-1,), name='precision_sum') + relevant_precision_per_k, reduction_indices=(-1, ), name='precision_sum' + ) # Divide by number of relevant items to get average precision. These are # the "num_relevant_items" and "AveP" terms from the formula above. @@ -3081,12 +3352,14 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): return math_ops.div(precision_sum, num_relevant_items, name=scope) -def _streaming_sparse_average_precision_at_top_k(labels, - predictions_idx, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def _streaming_sparse_average_precision_at_top_k( + labels, + predictions_idx, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes average precision@k of predictions with respect to sparse labels. `sparse_average_precision_at_top_k` creates two local variables, @@ -3131,19 +3404,22 @@ def _streaming_sparse_average_precision_at_top_k(labels, update: `Operation` that increments variables appropriately, and whose value matches `metric`. """ - with ops.name_scope(name, 'average_precision_at_top_k', - (predictions_idx, labels, weights)) as scope: + with ops.name_scope( + name, 'average_precision_at_top_k', (predictions_idx, labels, weights) + ) as scope: # Calculate per-example average precision, and apply weights. average_precision = _sparse_average_precision_at_top_k( - predictions_idx=predictions_idx, labels=labels) + predictions_idx=predictions_idx, labels=labels + ) if weights is not None: weights = weights_broadcast_ops.broadcast_weights( - math_ops.to_double(weights), average_precision) + math_ops.to_double(weights), average_precision + ) average_precision = math_ops.multiply(average_precision, weights) # Create accumulation variables and update ops for max average precision and # total average precision. - with ops.name_scope(None, 'max', (average_precision,)) as max_scope: + with ops.name_scope(None, 'max', (average_precision, )) as max_scope: # `max` is the max possible precision. Since max for any row is 1.0: # - For the unweighted case, this is just the number of rows. # - For the weighted case, it's the sum of the weights broadcast across @@ -3151,24 +3427,27 @@ def _streaming_sparse_average_precision_at_top_k(labels, max_var = metric_variable([], dtypes.float64, name=max_scope) if weights is None: batch_max = math_ops.to_double( - array_ops.size(average_precision, name='batch_max')) + array_ops.size(average_precision, name='batch_max') + ) else: batch_max = math_ops.reduce_sum(weights, name='batch_max') max_update = state_ops.assign_add( - max_var, batch_max, name='update', use_locking=True) - with ops.name_scope(None, 'total', (average_precision,)) as total_scope: + max_var, batch_max, name='update', use_locking=True + ) + with ops.name_scope(None, 'total', (average_precision, )) as total_scope: total_var = metric_variable([], dtypes.float64, name=total_scope) batch_total = math_ops.reduce_sum(average_precision, name='batch_total') total_update = state_ops.assign_add( - total_var, batch_total, name='update', use_locking=True) + total_var, batch_total, name='update', use_locking=True + ) # Divide total by max to get mean, for both vars and the update ops. def precision_across_towers(_, total_var, max_var): return _safe_scalar_div(total_var, max_var, name='mean') - mean_average_precision = _aggregate_across_towers(metrics_collections, - precision_across_towers, - total_var, max_var) + mean_average_precision = _aggregate_across_towers( + metrics_collections, precision_across_towers, total_var, max_var + ) update = _safe_scalar_div(total_update, max_update, name=scope) if updates_collections: @@ -3179,32 +3458,37 @@ def precision_across_towers(_, total_var, max_var): @tf_export('metrics.sparse_average_precision_at_k') @deprecated(None, 'Use average_precision_at_k instead') -def sparse_average_precision_at_k(labels, - predictions, - k, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def sparse_average_precision_at_k( + labels, + predictions, + k, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Renamed to `average_precision_at_k`, please use that method instead.""" return average_precision_at_k( - labels=labels, - predictions=predictions, - k=k, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) + labels=labels, + predictions=predictions, + k=k, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name + ) @tf_export('metrics.average_precision_at_k') -def average_precision_at_k(labels, - predictions, - k, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def average_precision_at_k( + labels, + predictions, + k, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes average precision@k of predictions with respect to sparse labels. `average_precision_at_k` creates two local variables, @@ -3258,28 +3542,31 @@ def average_precision_at_k(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sparse_average_precision_at_k is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.sparse_average_precision_at_k is not ' + 'supported when eager execution is enabled.' + ) if k < 1: raise ValueError('Invalid k=%s.' % k) - with ops.name_scope(name, _at_k_name('average_precision', k), - (predictions, labels, weights)) as scope: + with ops.name_scope( + name, _at_k_name('average_precision', k), (predictions, labels, weights) + ) as scope: # Calculate top k indices to produce [D1, ... DN, k] tensor. _, predictions_idx = nn.top_k(predictions, k) return _streaming_sparse_average_precision_at_top_k( - labels=labels, - predictions_idx=predictions_idx, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope) - - -def _sparse_false_positive_at_k(labels, - predictions_idx, - class_id=None, - weights=None): + labels=labels, + predictions_idx=predictions_idx, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope + ) + + +def _sparse_false_positive_at_k( + labels, predictions_idx, class_id=None, weights=None +): """Calculates false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3305,27 +3592,28 @@ def _sparse_false_positive_at_k(labels, Returns: A [D1, ... DN] `Tensor` of false positive counts. """ - with ops.name_scope(None, 'false_positives', - (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, - class_id) + with ops.name_scope( + None, 'false_positives', (predictions_idx, labels, weights) + ): + labels, predictions_idx = _maybe_select_class_id( + labels, predictions_idx, class_id + ) fp = sets.set_size( - sets.set_difference(predictions_idx, labels, aminusb=True)) + sets.set_difference(predictions_idx, labels, aminusb=True) + ) fp = math_ops.to_double(fp) if weights is not None: with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, fp),)): + (weights_broadcast_ops.assert_broadcastable(weights, fp), ) + ): weights = math_ops.to_double(weights) fp = math_ops.multiply(fp, weights) return fp -def _streaming_sparse_false_positive_at_k(labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - name=None): +def _streaming_sparse_false_positive_at_k( + labels, predictions_idx, k=None, class_id=None, weights=None, name=None +): """Calculates weighted per step false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3358,29 +3646,35 @@ def _streaming_sparse_false_positive_at_k(labels, Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope(name, _at_k_name('false_positive', k, class_id=class_id), - (predictions_idx, labels, weights)) as scope: + with ops.name_scope( + name, _at_k_name('false_positive', k, class_id=class_id), + (predictions_idx, labels, weights) + ) as scope: fp = _sparse_false_positive_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights + ) batch_total_fp = math_ops.to_double(math_ops.reduce_sum(fp)) var = metric_variable([], dtypes.float64, name=scope) return var, state_ops.assign_add( - var, batch_total_fp, name='update', use_locking=True) + var, batch_total_fp, name='update', use_locking=True + ) @tf_export('metrics.precision_at_top_k') -def precision_at_top_k(labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def precision_at_top_k( + labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes precision@k of the predictions with respect to sparse labels. Differs from `sparse_precision_at_k` in that predictions must be in the form @@ -3428,34 +3722,42 @@ def precision_at_top_k(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision_at_top_k is not ' - 'supported when eager execution is enabled.') - - with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), - (predictions_idx, labels, weights)) as scope: + raise RuntimeError( + 'tf.metrics.precision_at_top_k is not ' + 'supported when eager execution is enabled.' + ) + + with ops.name_scope( + name, _at_k_name('precision', k, class_id=class_id), + (predictions_idx, labels, weights) + ) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.to_int64(predictions_idx) tp, tp_update = _streaming_sparse_true_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights + ) fp, fp_update = _streaming_sparse_false_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights + ) def precision_across_towers(_, tp, fp): return math_ops.div(tp, math_ops.add(tp, fp), name=scope) - metric = _aggregate_across_towers(metrics_collections, - precision_across_towers, tp, fp) + metric = _aggregate_across_towers( + metrics_collections, precision_across_towers, tp, fp + ) update = math_ops.div( - tp_update, math_ops.add(tp_update, fp_update), name='update') + tp_update, math_ops.add(tp_update, fp_update), name='update' + ) if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @@ -3463,35 +3765,40 @@ def precision_across_towers(_, tp, fp): @tf_export('metrics.sparse_precision_at_k') @deprecated(None, 'Use precision_at_k instead') -def sparse_precision_at_k(labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def sparse_precision_at_k( + labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Renamed to `precision_at_k`, please use that method instead.""" return precision_at_k( - labels=labels, - predictions=predictions, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) + labels=labels, + predictions=predictions, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name + ) @tf_export('metrics.precision_at_k') -def precision_at_k(labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def precision_at_k( + labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes precision@k of the predictions with respect to sparse labels. If `class_id` is specified, we calculate precision by considering only the @@ -3560,32 +3867,39 @@ def precision_at_k(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sparse_precision_at_k is not ' - 'supported when eager execution is enabled.') - - with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), - (predictions, labels, weights)) as scope: + raise RuntimeError( + 'tf.metrics.sparse_precision_at_k is not ' + 'supported when eager execution is enabled.' + ) + + with ops.name_scope( + name, _at_k_name('precision', k, class_id=class_id), + (predictions, labels, weights) + ) as scope: _, top_k_idx = nn.top_k(predictions, k) return precision_at_top_k( - labels=labels, - predictions_idx=top_k_idx, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope) + labels=labels, + predictions_idx=top_k_idx, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope + ) @tf_export('metrics.specificity_at_sensitivity') -def specificity_at_sensitivity(labels, - predictions, - sensitivity, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None): +def specificity_at_sensitivity( + labels, + predictions, + sensitivity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the specificity at a given sensitivity. The `specificity_at_sensitivity` function creates four local @@ -3637,22 +3951,26 @@ def specificity_at_sensitivity(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.specificity_at_sensitivity is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.specificity_at_sensitivity is not ' + 'supported when eager execution is enabled.' + ) if sensitivity < 0 or sensitivity > 1: raise ValueError('`sensitivity` must be in the range [0, 1].') - with variable_scope.variable_scope(name, 'specificity_at_sensitivity', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'specificity_at_sensitivity', (predictions, labels, weights) + ): kepsilon = 1e-7 # to account for floating point imprecisions thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 - kepsilon] values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights) + labels, predictions, thresholds, weights + ) def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): """Computes the specificity at the given sensitivity. @@ -3673,29 +3991,31 @@ def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): # whether we should use the first or last index in case of ties. min_val = math_ops.reduce_min(math_ops.abs(sensitivities - sensitivity)) indices_at_minval = math_ops.equal( - math_ops.abs(sensitivities - sensitivity), min_val) + math_ops.abs(sensitivities - sensitivity), min_val + ) indices_at_minval = math_ops.to_int64(indices_at_minval) indices_at_minval = math_ops.cumsum(indices_at_minval) tf_index = math_ops.argmax(indices_at_minval, 0) tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the specificity: - return math_ops.div(tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, - name) + return math_ops.div( + tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, name + ) def specificity_across_towers(_, values): - return compute_specificity_at_sensitivity(values['tp'], values['tn'], - values['fp'], values['fn'], - 'value') - - specificity = _aggregate_across_towers(metrics_collections, - specificity_across_towers, values) - - update_op = compute_specificity_at_sensitivity(update_ops['tp'], - update_ops['tn'], - update_ops['fp'], - update_ops['fn'], - 'update_op') + return compute_specificity_at_sensitivity( + values['tp'], values['tn'], values['fp'], values['fn'], 'value' + ) + + specificity = _aggregate_across_towers( + metrics_collections, specificity_across_towers, values + ) + + update_op = compute_specificity_at_sensitivity( + update_ops['tp'], update_ops['tn'], update_ops['fp'], update_ops['fn'], + 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) diff --git a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py index 1756826be..4c2c3a7b1 100644 --- a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py +++ b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py @@ -13,26 +13,12 @@ # ============================================================================== """Implementation of tf.metrics module.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import confusion_matrix -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import sets -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import weights_broadcast_ops +from tensorflow.python.framework import dtypes, ops, sparse_tensor +from tensorflow.python.ops import array_ops, check_ops, confusion_matrix, control_flow_ops, math_ops, nn, sets, sparse_ops, state_ops, variable_scope, weights_broadcast_ops # NOQA from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.tf_export import tf_export @@ -76,15 +62,16 @@ def metric_variable(shape, dtype, validate_shape=True, name=None): """ # Note that synchronization "ON_READ" implies trainable=False. return variable_scope.variable( - lambda: array_ops.zeros(shape, dtype), - trainable=False, - collections=[ - ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES - ], - validate_shape=validate_shape, - synchronization=variable_scope.VariableSynchronization.ON_READ, - aggregation=variable_scope.VariableAggregation.SUM, - name=name) + lambda: array_ops.zeros(shape, dtype), + trainable=False, + collections=[ + ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES + ], + validate_shape=validate_shape, + synchronization=variable_scope.VariableSynchronization.ON_READ, + aggregation=variable_scope.VariableAggregation.SUM, + name=name + ) def _remove_squeezable_dimensions(predictions, labels, weights): @@ -113,7 +100,8 @@ def _remove_squeezable_dimensions(predictions, labels, weights): predictions = ops.convert_to_tensor(predictions) if labels is not None: labels, predictions = confusion_matrix.remove_squeezable_dimensions( - labels, predictions) + labels, predictions + ) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if weights is None: @@ -140,12 +128,15 @@ def _remove_squeezable_dimensions(predictions, labels, weights): def _maybe_expand_weights(): return control_flow_ops.cond( - math_ops.equal(rank_diff, -1), - lambda: array_ops.expand_dims(weights, [-1]), lambda: weights) + math_ops.equal(rank_diff, -1), + lambda: array_ops.expand_dims(weights, [-1]), lambda: weights + ) # Don't attempt squeeze if it will fail based on static check. - if ((weights_rank is not None) and - (not weights_shape.dims[-1].is_compatible_with(1))): + if ( + (weights_rank is not None) + and (not weights_shape.dims[-1].is_compatible_with(1)) + ): maybe_squeeze_weights = lambda: weights # noqa: E731 else: maybe_squeeze_weights = lambda: array_ops.squeeze( # noqa: E731 @@ -153,14 +144,16 @@ def _maybe_expand_weights(): def _maybe_adjust_weights(): return control_flow_ops.cond( - math_ops.equal(rank_diff, 1), maybe_squeeze_weights, - _maybe_expand_weights) + math_ops.equal(rank_diff, 1), maybe_squeeze_weights, + _maybe_expand_weights + ) # If weights are scalar, do nothing. Otherwise, try to add or remove a # dimension to match predictions. weights = control_flow_ops.cond( - math_ops.equal(weights_rank_tensor, 0), lambda: weights, - _maybe_adjust_weights) + math_ops.equal(weights_rank_tensor, 0), lambda: weights, + _maybe_adjust_weights + ) return predictions, labels, weights @@ -186,12 +179,11 @@ def _maybe_expand_labels(labels, predictions): # If sparse, expand sparse shape. if isinstance(labels, sparse_tensor.SparseTensor): return control_flow_ops.cond( - math_ops.equal( - array_ops.rank(predictions), - array_ops.size(labels.dense_shape) + 1), + math_ops.equal(array_ops.rank(predictions), + array_ops.size(labels.dense_shape) + 1), lambda: sparse_ops.sparse_reshape( # pylint: disable=g-long-lambda labels, - shape=array_ops.concat((labels.dense_shape, (1,)), 0), + shape=array_ops.concat((labels.dense_shape, (1, )), 0), name=scope), lambda: labels) @@ -205,14 +197,16 @@ def _maybe_expand_labels(labels, predictions): if predictions_rank == labels_rank + 1: return array_ops.expand_dims(labels, -1, name=scope) raise ValueError( - 'Unexpected labels shape %s for predictions shape %s.' % - (labels.get_shape(), predictions.get_shape())) + 'Unexpected labels shape %s for predictions shape %s.' % + (labels.get_shape(), predictions.get_shape()) + ) # Otherwise, use dynamic shape. return control_flow_ops.cond( - math_ops.equal(array_ops.rank(predictions), - array_ops.rank(labels) + 1), - lambda: array_ops.expand_dims(labels, -1, name=scope), lambda: labels) + math_ops.equal(array_ops.rank(predictions), + array_ops.rank(labels) + 1), + lambda: array_ops.expand_dims(labels, -1, name=scope), lambda: labels + ) def _safe_scalar_div(numerator, denominator, name): @@ -231,7 +225,9 @@ def _safe_scalar_div(numerator, denominator, name): return math_ops.div_no_nan(numerator, denominator, name=name) -def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): +def _streaming_confusion_matrix( + labels, predictions, num_classes, weights=None +): """Calculate a streaming confusion matrix. Calculates a confusion matrix. For estimation over a stream of data, @@ -255,9 +251,9 @@ def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): update_op: An operation that increments the confusion matrix. """ # Local variable to accumulate the predictions in the confusion matrix. - total_cm = metric_variable([num_classes, num_classes], - dtypes.float64, - name='total_confusion_matrix') + total_cm = metric_variable( + [num_classes, num_classes], dtypes.float64, name='total_confusion_matrix' + ) # Cast the type to int64 required by confusion_matrix_ops. predictions = math_ops.cast(predictions, dtypes.int64) @@ -276,7 +272,8 @@ def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): # Accumulate the prediction to current confusion matrix. current_cm = confusion_matrix.confusion_matrix( - labels, predictions, num_classes, weights=weights, dtype=dtypes.float64) + labels, predictions, num_classes, weights=weights, dtype=dtypes.float64 + ) update_op = state_ops.assign_add(total_cm, current_cm, use_locking=True) return total_cm, update_op @@ -314,15 +311,18 @@ def fn(distribution, *a): return metric_value return distribution_strategy_context.get_replica_context().merge_call( - fn, args=args) + fn, args=args + ) @tf_export(v1=['metrics.mean']) -def mean(values, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean( + values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the (weighted) mean of the given values. The `mean` function creates two local variables, `total` and `count` @@ -361,8 +361,10 @@ def mean(values, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean is not supported when eager execution ' - 'is enabled.') + raise RuntimeError( + 'tf.metrics.mean is not supported when eager execution ' + 'is enabled.' + ) with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.cast(values, dtypes.float32) @@ -374,25 +376,31 @@ def mean(values, num_values = math_ops.cast(array_ops.size(values), dtypes.float32) else: values, _, weights = _remove_squeezable_dimensions( - predictions=values, labels=None, weights=weights) + predictions=values, labels=None, weights=weights + ) weights = weights_broadcast_ops.broadcast_weights( - math_ops.cast(weights, dtypes.float32), values) + math_ops.cast(weights, dtypes.float32), values + ) values = math_ops.multiply(values, weights) num_values = math_ops.reduce_sum(weights) update_total_op = state_ops.assign_add( - total, math_ops.reduce_sum(values), use_locking=True) + total, math_ops.reduce_sum(values), use_locking=True + ) with ops.control_dependencies([values]): update_count_op = state_ops.assign_add( - count, num_values, use_locking=True) + count, num_values, use_locking=True + ) def compute_mean(_, t, c): return math_ops.div_no_nan(t, math_ops.maximum(c, 0), name='value') - mean_t = _aggregate_across_replicas(metrics_collections, compute_mean, - total, count) + mean_t = _aggregate_across_replicas( + metrics_collections, compute_mean, total, count + ) update_op = math_ops.div_no_nan( - update_total_op, math_ops.maximum(update_count_op, 0), name='update_op') + update_total_op, math_ops.maximum(update_count_op, 0), name='update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -401,12 +409,14 @@ def compute_mean(_, t, c): @tf_export(v1=['metrics.accuracy']) -def accuracy(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def accuracy( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Calculates how often `predictions` matches `labels`. The `accuracy` function creates two local variables, `total` and @@ -451,25 +461,29 @@ def accuracy(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.accuracy is not supported when eager ' - 'execution is enabled.') + raise RuntimeError( + 'tf.metrics.accuracy is not supported when eager ' + 'execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) + predictions=predictions, labels=labels, weights=weights + ) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if labels.dtype != predictions.dtype: predictions = math_ops.cast(predictions, labels.dtype) is_correct = math_ops.cast( - math_ops.equal(predictions, labels), dtypes.float32) - return mean(is_correct, weights, metrics_collections, updates_collections, - name or 'accuracy') + math_ops.equal(predictions, labels), dtypes.float32 + ) + return mean( + is_correct, weights, metrics_collections, updates_collections, name + or 'accuracy' + ) -def _confusion_matrix_at_thresholds(labels, - predictions, - thresholds, - weights=None, - includes=None): +def _confusion_matrix_at_thresholds( + labels, predictions, thresholds, weights=None, includes=None +): """Computes true_positives, false_negatives, true_negatives, false_positives. This function creates up to four local variables, `true_positives`, @@ -520,27 +534,33 @@ def _confusion_matrix_at_thresholds(labels, if include not in all_includes: raise ValueError('Invalid key: %s.' % include) - with ops.control_dependencies([ + with ops.control_dependencies( + [ check_ops.assert_greater_equal( - predictions, - math_ops.cast(0.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]'), + predictions, + math_ops.cast(0.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]' + ), check_ops.assert_less_equal( - predictions, - math_ops.cast(1.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]') - ]): + predictions, + math_ops.cast(1.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]' + ) + ] + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtypes.float32), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtypes.float32), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) num_thresholds = len(thresholds) # Reshape predictions and labels. predictions_2d = array_ops.reshape(predictions, [-1, 1]) labels_2d = array_ops.reshape( - math_ops.cast(labels, dtype=dtypes.bool), [1, -1]) + math_ops.cast(labels, dtype=dtypes.bool), [1, -1] + ) # Use static shape if known. num_predictions = predictions_2d.get_shape().as_list()[0] @@ -549,13 +569,15 @@ def _confusion_matrix_at_thresholds(labels, if num_predictions is None: num_predictions = array_ops.shape(predictions_2d)[0] thresh_tiled = array_ops.tile( - array_ops.expand_dims(array_ops.constant(thresholds), [1]), - array_ops.stack([1, num_predictions])) + array_ops.expand_dims(array_ops.constant(thresholds), [1]), + array_ops.stack([1, num_predictions]) + ) # Tile the predictions after thresholding them across different thresholds. pred_is_pos = math_ops.greater( - array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]), - thresh_tiled) + array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]), + thresh_tiled + ) if ('fn' in includes) or ('tn' in includes): pred_is_neg = math_ops.logical_not(pred_is_pos) @@ -566,11 +588,14 @@ def _confusion_matrix_at_thresholds(labels, if weights is not None: weights = weights_broadcast_ops.broadcast_weights( - math_ops.cast(weights, dtypes.float32), predictions) + math_ops.cast(weights, dtypes.float32), predictions + ) weights_tiled = array_ops.tile( - array_ops.reshape(weights, [1, -1]), [num_thresholds, 1]) + array_ops.reshape(weights, [1, -1]), [num_thresholds, 1] + ) thresh_tiled.get_shape().assert_is_compatible_with( - weights_tiled.get_shape()) + weights_tiled.get_shape() + ) else: weights_tiled = None @@ -578,51 +603,59 @@ def _confusion_matrix_at_thresholds(labels, update_ops = {} if 'tp' in includes: - true_p = metric_variable([num_thresholds], - dtypes.float32, - name='true_positives') + true_p = metric_variable( + [num_thresholds], dtypes.float32, name='true_positives' + ) is_true_positive = math_ops.cast( - math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32) + math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32 + ) if weights_tiled is not None: is_true_positive *= weights_tiled update_ops['tp'] = state_ops.assign_add( - true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True) + true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True + ) values['tp'] = true_p if 'fn' in includes: - false_n = metric_variable([num_thresholds], - dtypes.float32, - name='false_negatives') + false_n = metric_variable( + [num_thresholds], dtypes.float32, name='false_negatives' + ) is_false_negative = math_ops.cast( - math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32) + math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32 + ) if weights_tiled is not None: is_false_negative *= weights_tiled update_ops['fn'] = state_ops.assign_add( - false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True) + false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True + ) values['fn'] = false_n if 'tn' in includes: - true_n = metric_variable([num_thresholds], - dtypes.float32, - name='true_negatives') + true_n = metric_variable( + [num_thresholds], dtypes.float32, name='true_negatives' + ) is_true_negative = math_ops.cast( - math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32) + math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32 + ) if weights_tiled is not None: is_true_negative *= weights_tiled update_ops['tn'] = state_ops.assign_add( - true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True) + true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True + ) values['tn'] = true_n if 'fp' in includes: - false_p = metric_variable([num_thresholds], - dtypes.float32, - name='false_positives') + false_p = metric_variable( + [num_thresholds], dtypes.float32, name='false_positives' + ) is_false_positive = math_ops.cast( - math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32) + math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32 + ) if weights_tiled is not None: is_false_positive *= weights_tiled update_ops['fp'] = state_ops.assign_add( - false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True) + false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True + ) values['fp'] = false_p return values, update_ops @@ -635,16 +668,18 @@ def _aggregate_variable(v, collections): @tf_export(v1=['metrics.auc']) -def auc(labels, - predictions, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - curve='ROC', - name=None, - summation_method='trapezoidal', - thresholds=None): +def auc( + labels, + predictions, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + curve='ROC', + name=None, + summation_method='trapezoidal', + thresholds=None +): """Computes the approximate AUC via a Riemann sum. The `auc` function creates four local variables, `true_positives`, @@ -724,11 +759,14 @@ def auc(labels, """ print('use_tf_auc') if context.executing_eagerly(): - raise RuntimeError('tf.metrics.auc is not supported when eager execution ' - 'is enabled.') - - with variable_scope.variable_scope(name, 'auc', - (labels, predictions, weights)): + raise RuntimeError( + 'tf.metrics.auc is not supported when eager execution ' + 'is enabled.' + ) + + with variable_scope.variable_scope( + name, 'auc', (labels, predictions, weights) + ): if curve != 'ROC' and curve != 'PR': raise ValueError('curve must be either ROC or PR, %s unknown' % (curve)) @@ -740,15 +778,18 @@ def auc(labels, else: # Otherwise, linearly interpolate (num_thresholds - 2) thresholds in # (0, 1). - thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) - for i in range(num_thresholds - 2)] + thresholds = [ + (i + 1) * 1.0 / (num_thresholds - 1) + for i in range(num_thresholds - 2) + ] # Add an endpoint "threshold" below zero and above one for either threshold # method. thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights) + labels, predictions, thresholds, weights + ) # Add epsilons to avoid dividing by 0. epsilon = 1.0e-6 @@ -787,30 +828,36 @@ def interpolate_pr_auc(tp, fp, fn): dtp = tp[:num_thresholds - 1] - tp[1:] p = tp + fp prec_slope = math_ops.div_no_nan( - dtp, - math_ops.maximum(p[:num_thresholds - 1] - p[1:], 0), - name='prec_slope') + dtp, + math_ops.maximum(p[:num_thresholds - 1] - p[1:], 0), + name='prec_slope' + ) intercept = tp[1:] - math_ops.multiply(prec_slope, p[1:]) safe_p_ratio = array_ops.where( - math_ops.logical_and(p[:num_thresholds - 1] > 0, p[1:] > 0), - math_ops.div_no_nan( - p[:num_thresholds - 1], - math_ops.maximum(p[1:], 0), - name='recall_relative_ratio'), array_ops.ones_like(p[1:])) + math_ops.logical_and(p[:num_thresholds - 1] > 0, p[1:] > 0), + math_ops.div_no_nan( + p[:num_thresholds - 1], + math_ops.maximum(p[1:], 0), + name='recall_relative_ratio' + ), array_ops.ones_like(p[1:]) + ) return math_ops.reduce_sum( - math_ops.div_no_nan( - prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)), - math_ops.maximum(tp[1:] + fn[1:], 0), - name='pr_auc_increment'), - name='interpolate_pr_auc') + math_ops.div_no_nan( + prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)), + math_ops.maximum(tp[1:] + fn[1:], 0), + name='pr_auc_increment' + ), + name='interpolate_pr_auc' + ) def compute_auc(tp, fn, tn, fp, name): """Computes the roc-auc or pr-auc based on confusion counts.""" if curve == 'PR': if summation_method == 'trapezoidal': logging.warning( - 'Trapezoidal rule is known to produce incorrect PR-AUCs; ' - 'please switch to "careful_interpolation" instead.') + 'Trapezoidal rule is known to produce incorrect PR-AUCs; ' + 'please switch to "careful_interpolation" instead.' + ) elif summation_method == 'careful_interpolation': # This one is a bit tricky and is handled separately. return interpolate_pr_auc(tp, fp, fn) @@ -827,31 +874,44 @@ def compute_auc(tp, fn, tn, fp, name): # Note that the case ('PR', 'careful_interpolation') has been handled # above. return math_ops.reduce_sum( - math_ops.multiply(x[:num_thresholds - 1] - x[1:], - (y[:num_thresholds - 1] + y[1:]) / 2.), - name=name) + math_ops.multiply( + x[:num_thresholds - 1] - x[1:], + (y[:num_thresholds - 1] + y[1:]) / 2. + ), + name=name + ) elif summation_method == 'minoring': return math_ops.reduce_sum( - math_ops.multiply(x[:num_thresholds - 1] - x[1:], - math_ops.minimum(y[:num_thresholds - 1], y[1:])), - name=name) + math_ops.multiply( + x[:num_thresholds - 1] - x[1:], + math_ops.minimum(y[:num_thresholds - 1], y[1:]) + ), + name=name + ) elif summation_method == 'majoring': return math_ops.reduce_sum( - math_ops.multiply(x[:num_thresholds - 1] - x[1:], - math_ops.maximum(y[:num_thresholds - 1], y[1:])), - name=name) + math_ops.multiply( + x[:num_thresholds - 1] - x[1:], + math_ops.maximum(y[:num_thresholds - 1], y[1:]) + ), + name=name + ) else: raise ValueError('Invalid summation_method: %s' % summation_method) # sum up the areas of all the trapeziums def compute_auc_value(_, values): - return compute_auc(values['tp'], values['fn'], values['tn'], values['fp'], - 'value') - - auc_value = _aggregate_across_replicas(metrics_collections, - compute_auc_value, values) - update_op = compute_auc(update_ops['tp'], update_ops['fn'], - update_ops['tn'], update_ops['fp'], 'update_op') + return compute_auc( + values['tp'], values['fn'], values['tn'], values['fp'], 'value' + ) + + auc_value = _aggregate_across_replicas( + metrics_collections, compute_auc_value, values + ) + update_op = compute_auc( + update_ops['tp'], update_ops['fn'], update_ops['tn'], update_ops['fp'], + 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -860,12 +920,14 @@ def compute_auc_value(_, values): @tf_export(v1=['metrics.mean_absolute_error']) -def mean_absolute_error(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_absolute_error( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the mean absolute error between the labels and predictions. The `mean_absolute_error` function creates two local variables, @@ -910,24 +972,31 @@ def mean_absolute_error(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_absolute_error is not supported ' - 'when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.mean_absolute_error is not supported ' + 'when eager execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) + predictions=predictions, labels=labels, weights=weights + ) absolute_errors = math_ops.abs(predictions - labels) - return mean(absolute_errors, weights, metrics_collections, - updates_collections, name or 'mean_absolute_error') + return mean( + absolute_errors, weights, metrics_collections, updates_collections, name + or 'mean_absolute_error' + ) @tf_export(v1=['metrics.mean_cosine_distance']) -def mean_cosine_distance(labels, - predictions, - dim, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_cosine_distance( + labels, + predictions, + dim, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the cosine distance between the labels and predictions. The `mean_cosine_distance` function creates two local variables, @@ -970,18 +1039,23 @@ def mean_cosine_distance(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_cosine_distance is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.mean_cosine_distance is not supported when ' + 'eager execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) + predictions=predictions, labels=labels, weights=weights + ) radial_diffs = math_ops.multiply(predictions, labels) radial_diffs = math_ops.reduce_sum( - radial_diffs, axis=[ - dim, - ], keepdims=True) - mean_distance, update_op = mean(radial_diffs, weights, None, None, name or - 'mean_cosine_distance') + radial_diffs, axis=[ + dim, + ], keepdims=True + ) + mean_distance, update_op = mean( + radial_diffs, weights, None, None, name or 'mean_cosine_distance' + ) mean_distance = math_ops.subtract(1.0, mean_distance) update_op = math_ops.subtract(1.0, update_op) @@ -995,13 +1069,15 @@ def mean_cosine_distance(labels, @tf_export(v1=['metrics.mean_per_class_accuracy']) -def mean_per_class_accuracy(labels, - predictions, - num_classes, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_per_class_accuracy( + labels, + predictions, + num_classes, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Calculates the mean of the per-class accuracies. Calculates the accuracy for each class, then takes the mean of that. @@ -1043,11 +1119,14 @@ def mean_per_class_accuracy(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_per_class_accuracy is not supported ' - 'when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'mean_accuracy', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.mean_per_class_accuracy is not supported ' + 'when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'mean_accuracy', (predictions, labels, weights) + ): labels = math_ops.cast(labels, dtypes.int64) # Flatten the input if its rank > 1. @@ -1068,7 +1147,8 @@ def mean_per_class_accuracy(labels, if labels.dtype != predictions.dtype: predictions = math_ops.cast(predictions, labels.dtype) is_correct = math_ops.cast( - math_ops.equal(predictions, labels), dtypes.float32) + math_ops.equal(predictions, labels), dtypes.float32 + ) if weights is not None: if weights.get_shape().ndims > 1: @@ -1083,17 +1163,20 @@ def mean_per_class_accuracy(labels, def compute_mean_accuracy(_, count, total): per_class_accuracy = math_ops.div_no_nan( - count, math_ops.maximum(total, 0), name=None) + count, math_ops.maximum(total, 0), name=None + ) mean_accuracy_v = math_ops.reduce_mean( - per_class_accuracy, name='mean_accuracy') + per_class_accuracy, name='mean_accuracy' + ) return mean_accuracy_v - mean_accuracy_v = _aggregate_across_replicas(metrics_collections, - compute_mean_accuracy, count, - total) + mean_accuracy_v = _aggregate_across_replicas( + metrics_collections, compute_mean_accuracy, count, total + ) update_op = math_ops.div_no_nan( - update_count_op, math_ops.maximum(update_total_op, 0), name='update_op') + update_count_op, math_ops.maximum(update_total_op, 0), name='update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1101,13 +1184,15 @@ def compute_mean_accuracy(_, count, total): @tf_export(v1=['metrics.mean_iou']) -def mean_iou(labels, - predictions, - num_classes, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_iou( + labels, + predictions, + num_classes, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Calculate per-step mean Intersection-Over-Union (mIOU). Mean Intersection-Over-Union is a common evaluation metric for @@ -1153,23 +1238,29 @@ def mean_iou(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_iou is not supported when ' - 'eager execution is enabled.') - - with variable_scope.variable_scope(name, 'mean_iou', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.mean_iou is not supported when ' + 'eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'mean_iou', (predictions, labels, weights) + ): # Check if shape is compatible. predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - total_cm, update_op = _streaming_confusion_matrix(labels, predictions, - num_classes, weights) + total_cm, update_op = _streaming_confusion_matrix( + labels, predictions, num_classes, weights + ) def compute_mean_iou(_, total_cm): """Compute the mean intersection-over-union via the confusion matrix.""" sum_over_row = math_ops.cast( - math_ops.reduce_sum(total_cm, 0), dtypes.float32) + math_ops.reduce_sum(total_cm, 0), dtypes.float32 + ) sum_over_col = math_ops.cast( - math_ops.reduce_sum(total_cm, 1), dtypes.float32) + math_ops.reduce_sum(total_cm, 1), dtypes.float32 + ) cm_diag = math_ops.cast(array_ops.diag_part(total_cm), dtypes.float32) denominator = sum_over_row + sum_over_col - cm_diag @@ -1177,25 +1268,30 @@ def compute_mean_iou(_, total_cm): # label or prediction tensor. If the denominator is 0, we need to # ignore the class. num_valid_entries = math_ops.reduce_sum( - math_ops.cast( - math_ops.not_equal(denominator, 0), dtype=dtypes.float32)) + math_ops.cast( + math_ops.not_equal(denominator, 0), dtype=dtypes.float32 + ) + ) # If the value of the denominator is 0, set it to 1 to avoid # zero division. denominator = array_ops.where( - math_ops.greater(denominator, 0), denominator, - array_ops.ones_like(denominator)) + math_ops.greater(denominator, 0), denominator, + array_ops.ones_like(denominator) + ) iou = math_ops.div(cm_diag, denominator) # If the number of valid entries is 0 (no classes) we return 0. result = array_ops.where( - math_ops.greater(num_valid_entries, 0), - math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, 0) + math_ops.greater(num_valid_entries, 0), + math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, 0 + ) return result # TODO(priyag): Use outside_compilation if in TPU context. - mean_iou_v = _aggregate_across_replicas(metrics_collections, - compute_mean_iou, total_cm) + mean_iou_v = _aggregate_across_replicas( + metrics_collections, compute_mean_iou, total_cm + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1204,13 +1300,15 @@ def compute_mean_iou(_, total_cm): @tf_export(v1=['metrics.mean_relative_error']) -def mean_relative_error(labels, - predictions, - normalizer, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_relative_error( + labels, + predictions, + normalizer, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the mean relative error by normalizing with the given values. The `mean_relative_error` function creates two local variables, @@ -1256,29 +1354,38 @@ def mean_relative_error(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_relative_error is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.mean_relative_error is not supported when ' + 'eager execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) + predictions=predictions, labels=labels, weights=weights + ) predictions, normalizer = confusion_matrix.remove_squeezable_dimensions( - predictions, normalizer) + predictions, normalizer + ) predictions.get_shape().assert_is_compatible_with(normalizer.get_shape()) relative_errors = array_ops.where( - math_ops.equal(normalizer, 0.0), array_ops.zeros_like(labels), - math_ops.div(math_ops.abs(labels - predictions), normalizer)) - return mean(relative_errors, weights, metrics_collections, - updates_collections, name or 'mean_relative_error') + math_ops.equal(normalizer, 0.0), array_ops.zeros_like(labels), + math_ops.div(math_ops.abs(labels - predictions), normalizer) + ) + return mean( + relative_errors, weights, metrics_collections, updates_collections, name + or 'mean_relative_error' + ) @tf_export(v1=['metrics.mean_squared_error']) -def mean_squared_error(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_squared_error( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the mean squared error between the labels and predictions. The `mean_squared_error` function creates two local variables, @@ -1323,22 +1430,29 @@ def mean_squared_error(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_squared_error is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.mean_squared_error is not supported when ' + 'eager execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) + predictions=predictions, labels=labels, weights=weights + ) squared_error = math_ops.squared_difference(labels, predictions) - return mean(squared_error, weights, metrics_collections, updates_collections, - name or 'mean_squared_error') + return mean( + squared_error, weights, metrics_collections, updates_collections, name + or 'mean_squared_error' + ) @tf_export(v1=['metrics.mean_tensor']) -def mean_tensor(values, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def mean_tensor( + values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the element-wise (weighted) mean of the given tensors. In contrast to the `mean` function which returns a scalar with the @@ -1381,38 +1495,47 @@ def mean_tensor(values, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_tensor is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.mean_tensor is not supported when ' + 'eager execution is enabled.' + ) with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.cast(values, dtypes.float32) total = metric_variable( - values.get_shape(), dtypes.float32, name='total_tensor') + values.get_shape(), dtypes.float32, name='total_tensor' + ) count = metric_variable( - values.get_shape(), dtypes.float32, name='count_tensor') + values.get_shape(), dtypes.float32, name='count_tensor' + ) num_values = array_ops.ones_like(values) if weights is not None: values, _, weights = _remove_squeezable_dimensions( - predictions=values, labels=None, weights=weights) + predictions=values, labels=None, weights=weights + ) weights = weights_broadcast_ops.broadcast_weights( - math_ops.cast(weights, dtypes.float32), values) + math_ops.cast(weights, dtypes.float32), values + ) values = math_ops.multiply(values, weights) num_values = math_ops.multiply(num_values, weights) update_total_op = state_ops.assign_add(total, values, use_locking=True) with ops.control_dependencies([values]): update_count_op = state_ops.assign_add( - count, num_values, use_locking=True) + count, num_values, use_locking=True + ) compute_mean = lambda _, t, c: math_ops.div_no_nan( # noqa: E731 t, math_ops.maximum(c, 0), name='value') - mean_t = _aggregate_across_replicas(metrics_collections, compute_mean, - total, count) + mean_t = _aggregate_across_replicas( + metrics_collections, compute_mean, total, count + ) update_op = math_ops.div_no_nan( - update_total_op, math_ops.maximum(update_count_op, 0), name='update_op') + update_total_op, math_ops.maximum(update_count_op, 0), name='update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1420,12 +1543,14 @@ def mean_tensor(values, @tf_export(v1=['metrics.percentage_below']) -def percentage_below(values, - threshold, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def percentage_below( + values, + threshold, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the percentage of values less than the given threshold. The `percentage_below` function creates two local variables, @@ -1465,19 +1590,23 @@ def percentage_below(values, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.percentage_below is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.percentage_below is not supported when ' + 'eager execution is enabled.' + ) is_below_threshold = math_ops.cast( - math_ops.less(values, threshold), dtypes.float32) - return mean(is_below_threshold, weights, metrics_collections, - updates_collections, name or 'percentage_below_threshold') + math_ops.less(values, threshold), dtypes.float32 + ) + return mean( + is_below_threshold, weights, metrics_collections, updates_collections, name + or 'percentage_below_threshold' + ) -def _count_condition(values, - weights=None, - metrics_collections=None, - updates_collections=None): +def _count_condition( + values, weights=None, metrics_collections=None, updates_collections=None +): """Sums the weights of cases where the given values are True. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1507,14 +1636,16 @@ def _count_condition(values, values = math_ops.cast(values, dtypes.float32) if weights is not None: with ops.control_dependencies( - (check_ops.assert_rank_in(weights, (0, array_ops.rank(values))),)): + (check_ops.assert_rank_in(weights, (0, array_ops.rank(values))), ) + ): weights = math_ops.cast(weights, dtypes.float32) values = math_ops.multiply(values, weights) value_tensor = _aggregate_variable(count, metrics_collections) update_op = state_ops.assign_add( - count, math_ops.reduce_sum(values), use_locking=True) + count, math_ops.reduce_sum(values), use_locking=True + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1522,12 +1653,14 @@ def _count_condition(values, @tf_export(v1=['metrics.false_negatives']) -def false_negatives(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def false_negatives( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the total number of false negatives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1557,30 +1690,38 @@ def false_negatives(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_negatives is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.false_negatives is not supported when ' + 'eager execution is enabled.' + ) - with variable_scope.variable_scope(name, 'false_negatives', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'false_negatives', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) is_false_negative = math_ops.logical_and( - math_ops.equal(labels, True), math_ops.equal(predictions, False)) - return _count_condition(is_false_negative, weights, metrics_collections, - updates_collections) + math_ops.equal(labels, True), math_ops.equal(predictions, False) + ) + return _count_condition( + is_false_negative, weights, metrics_collections, updates_collections + ) @tf_export(v1=['metrics.false_negatives_at_thresholds']) -def false_negatives_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def false_negatives_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes false negatives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1613,13 +1754,17 @@ def false_negatives_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_negatives_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'false_negatives', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.false_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'false_negatives', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fn',)) + labels, predictions, thresholds, weights=weights, includes=('fn', ) + ) fn_value = _aggregate_variable(values['fn'], metrics_collections) @@ -1630,12 +1775,14 @@ def false_negatives_at_thresholds(labels, @tf_export(v1=['metrics.false_positives']) -def false_positives(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def false_positives( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Sum the weights of false positives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1666,30 +1813,38 @@ def false_positives(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_positives is not supported when ' - 'eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.false_positives is not supported when ' + 'eager execution is enabled.' + ) - with variable_scope.variable_scope(name, 'false_positives', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'false_positives', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) is_false_positive = math_ops.logical_and( - math_ops.equal(labels, False), math_ops.equal(predictions, True)) - return _count_condition(is_false_positive, weights, metrics_collections, - updates_collections) + math_ops.equal(labels, False), math_ops.equal(predictions, True) + ) + return _count_condition( + is_false_positive, weights, metrics_collections, updates_collections + ) @tf_export(v1=['metrics.false_positives_at_thresholds']) -def false_positives_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def false_positives_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes false positives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1722,13 +1877,17 @@ def false_positives_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_positives_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'false_positives', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.false_positives_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'false_positives', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fp',)) + labels, predictions, thresholds, weights=weights, includes=('fp', ) + ) fp_value = _aggregate_variable(values['fp'], metrics_collections) @@ -1739,12 +1898,14 @@ def false_positives_at_thresholds(labels, @tf_export(v1=['metrics.true_negatives']) -def true_negatives(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def true_negatives( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Sum the weights of true_negatives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1775,30 +1936,38 @@ def true_negatives(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_negatives is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.true_negatives is not ' + 'supported when eager execution is enabled.' + ) - with variable_scope.variable_scope(name, 'true_negatives', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'true_negatives', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) is_true_negative = math_ops.logical_and( - math_ops.equal(labels, False), math_ops.equal(predictions, False)) - return _count_condition(is_true_negative, weights, metrics_collections, - updates_collections) + math_ops.equal(labels, False), math_ops.equal(predictions, False) + ) + return _count_condition( + is_true_negative, weights, metrics_collections, updates_collections + ) @tf_export(v1=['metrics.true_negatives_at_thresholds']) -def true_negatives_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def true_negatives_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes true negatives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1831,13 +2000,17 @@ def true_negatives_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_negatives_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'true_negatives', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.true_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'true_negatives', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tn',)) + labels, predictions, thresholds, weights=weights, includes=('tn', ) + ) tn_value = _aggregate_variable(values['tn'], metrics_collections) @@ -1848,12 +2021,14 @@ def true_negatives_at_thresholds(labels, @tf_export(v1=['metrics.true_positives']) -def true_positives(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def true_positives( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Sum the weights of true_positives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1884,30 +2059,38 @@ def true_positives(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_positives is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.true_positives is not ' + 'supported when eager execution is enabled.' + ) - with variable_scope.variable_scope(name, 'true_positives', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'true_positives', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) is_true_positive = math_ops.logical_and( - math_ops.equal(labels, True), math_ops.equal(predictions, True)) - return _count_condition(is_true_positive, weights, metrics_collections, - updates_collections) + math_ops.equal(labels, True), math_ops.equal(predictions, True) + ) + return _count_condition( + is_true_positive, weights, metrics_collections, updates_collections + ) @tf_export(v1=['metrics.true_positives_at_thresholds']) -def true_positives_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def true_positives_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes true positives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1940,13 +2123,17 @@ def true_positives_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_positives_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'true_positives', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.true_positives_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'true_positives', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tp',)) + labels, predictions, thresholds, weights=weights, includes=('tp', ) + ) tp_value = _aggregate_variable(values['tp'], metrics_collections) @@ -1957,12 +2144,14 @@ def true_positives_at_thresholds(labels, @tf_export(v1=['metrics.precision']) -def precision(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def precision( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the precision of the predictions with respect to the labels. The `precision` function creates two local variables, @@ -2007,44 +2196,53 @@ def precision(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.precision is not ' + 'supported when eager execution is enabled.' + ) - with variable_scope.variable_scope(name, 'precision', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'precision', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) true_p, true_positives_update_op = true_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None + ) false_p, false_positives_update_op = false_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None + ) def compute_precision(tp, fp, name): return array_ops.where( - math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name) + math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name + ) def once_across_replicas(_, true_p, false_p): return compute_precision(true_p, false_p, 'value') - p = _aggregate_across_replicas(metrics_collections, once_across_replicas, - true_p, false_p) + p = _aggregate_across_replicas( + metrics_collections, once_across_replicas, true_p, false_p + ) - update_op = compute_precision(true_positives_update_op, - false_positives_update_op, 'update_op') + update_op = compute_precision( + true_positives_update_op, false_positives_update_op, 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2052,13 +2250,15 @@ def once_across_replicas(_, true_p, false_p): @tf_export(v1=['metrics.precision_at_thresholds']) -def precision_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def precision_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes precision values for different `thresholds` on `predictions`. The `precision_at_thresholds` function creates four local variables, @@ -2104,13 +2304,17 @@ def precision_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'precision_at_thresholds', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.precision_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'precision_at_thresholds', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights, includes=('tp', 'fp')) + labels, predictions, thresholds, weights, includes=('tp', 'fp') + ) # Avoid division by zero. epsilon = 1e-7 @@ -2121,11 +2325,13 @@ def compute_precision(tp, fp, name): def precision_across_replicas(_, values): return compute_precision(values['tp'], values['fp'], 'value') - prec = _aggregate_across_replicas(metrics_collections, - precision_across_replicas, values) + prec = _aggregate_across_replicas( + metrics_collections, precision_across_replicas, values + ) - update_op = compute_precision(update_ops['tp'], update_ops['fp'], - 'update_op') + update_op = compute_precision( + update_ops['tp'], update_ops['fp'], 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2133,12 +2339,14 @@ def precision_across_replicas(_, values): @tf_export(v1=['metrics.recall']) -def recall(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def recall( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the recall of the predictions with respect to the labels. The `recall` function creates two local variables, `true_positives` @@ -2181,44 +2389,53 @@ def recall(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall is not supported is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'recall', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.recall is not supported is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'recall', (predictions, labels, weights) + ): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights + ) true_p, true_positives_update_op = true_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None + ) false_n, false_negatives_update_op = false_negatives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None + ) def compute_recall(true_p, false_n, name): return array_ops.where( - math_ops.greater(true_p + false_n, 0), - math_ops.div(true_p, true_p + false_n), 0, name) + math_ops.greater(true_p + false_n, 0), + math_ops.div(true_p, true_p + false_n), 0, name + ) def once_across_replicas(_, true_p, false_n): return compute_recall(true_p, false_n, 'value') - rec = _aggregate_across_replicas(metrics_collections, once_across_replicas, - true_p, false_n) + rec = _aggregate_across_replicas( + metrics_collections, once_across_replicas, true_p, false_n + ) - update_op = compute_recall(true_positives_update_op, - false_negatives_update_op, 'update_op') + update_op = compute_recall( + true_positives_update_op, false_negatives_update_op, 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2248,8 +2465,9 @@ def _select_class_id(ids, selected_id): """ ids = sparse_tensor.convert_to_tensor_or_sparse_tensor(ids) if isinstance(ids, sparse_tensor.SparseTensor): - return sparse_ops.sparse_retain(ids, math_ops.equal(ids.values, - selected_id)) + return sparse_ops.sparse_retain( + ids, math_ops.equal(ids.values, selected_id) + ) # TODO(ptucker): Make this more efficient, maybe add a sparse version of # tf.equal and tf.reduce_any? @@ -2258,14 +2476,17 @@ def _select_class_id(ids, selected_id): ids_shape = array_ops.shape(ids, out_type=dtypes.int64) ids_last_dim = array_ops.size(ids_shape) - 1 filled_selected_id_shape = math_ops.reduced_shape( - ids_shape, array_ops.reshape(ids_last_dim, [1])) + ids_shape, array_ops.reshape(ids_last_dim, [1]) + ) # Intersect `ids` with the selected ID. - filled_selected_id = array_ops.fill(filled_selected_id_shape, - math_ops.cast(selected_id, dtypes.int64)) + filled_selected_id = array_ops.fill( + filled_selected_id_shape, math_ops.cast(selected_id, dtypes.int64) + ) result = sets.set_intersection(filled_selected_id, ids) return sparse_tensor.SparseTensor( - indices=result.indices, values=result.values, dense_shape=ids_shape) + indices=result.indices, values=result.values, dense_shape=ids_shape + ) def _maybe_select_class_id(labels, predictions_idx, selected_id=None): @@ -2287,15 +2508,15 @@ def _maybe_select_class_id(labels, predictions_idx, selected_id=None): """ if selected_id is None: return labels, predictions_idx - return (_select_class_id(labels, selected_id), - _select_class_id(predictions_idx, selected_id)) + return ( + _select_class_id(labels, selected_id), + _select_class_id(predictions_idx, selected_id) + ) -def _sparse_true_positive_at_k(labels, - predictions_idx, - class_id=None, - weights=None, - name=None): +def _sparse_true_positive_at_k( + labels, predictions_idx, class_id=None, weights=None, name=None +): """Calculates true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2322,26 +2543,26 @@ def _sparse_true_positive_at_k(labels, Returns: A [D1, ... DN] `Tensor` of true positive counts. """ - with ops.name_scope(name, 'true_positives', - (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, - class_id) + with ops.name_scope( + name, 'true_positives', (predictions_idx, labels, weights) + ): + labels, predictions_idx = _maybe_select_class_id( + labels, predictions_idx, class_id + ) tp = sets.set_size(sets.set_intersection(predictions_idx, labels)) tp = math_ops.cast(tp, dtypes.float64) if weights is not None: with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, tp),)): + (weights_broadcast_ops.assert_broadcastable(weights, tp), ) + ): weights = math_ops.cast(weights, dtypes.float64) tp = math_ops.multiply(tp, weights) return tp -def _streaming_sparse_true_positive_at_k(labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - name=None): +def _streaming_sparse_true_positive_at_k( + labels, predictions_idx, k=None, class_id=None, weights=None, name=None +): """Calculates weighted per step true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2374,24 +2595,27 @@ def _streaming_sparse_true_positive_at_k(labels, Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope(name, _at_k_name('true_positive', k, class_id=class_id), - (predictions_idx, labels, weights)) as scope: + with ops.name_scope( + name, _at_k_name('true_positive', k, class_id=class_id), + (predictions_idx, labels, weights) + ) as scope: tp = _sparse_true_positive_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights + ) batch_total_tp = math_ops.cast(math_ops.reduce_sum(tp), dtypes.float64) var = metric_variable([], dtypes.float64, name=scope) return var, state_ops.assign_add( - var, batch_total_tp, name='update', use_locking=True) + var, batch_total_tp, name='update', use_locking=True + ) -def _sparse_false_negative_at_k(labels, - predictions_idx, - class_id=None, - weights=None): +def _sparse_false_negative_at_k( + labels, predictions_idx, class_id=None, weights=None +): """Calculates false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2417,27 +2641,28 @@ def _sparse_false_negative_at_k(labels, Returns: A [D1, ... DN] `Tensor` of false negative counts. """ - with ops.name_scope(None, 'false_negatives', - (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, - class_id) + with ops.name_scope( + None, 'false_negatives', (predictions_idx, labels, weights) + ): + labels, predictions_idx = _maybe_select_class_id( + labels, predictions_idx, class_id + ) fn = sets.set_size( - sets.set_difference(predictions_idx, labels, aminusb=False)) + sets.set_difference(predictions_idx, labels, aminusb=False) + ) fn = math_ops.cast(fn, dtypes.float64) if weights is not None: with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, fn),)): + (weights_broadcast_ops.assert_broadcastable(weights, fn), ) + ): weights = math_ops.cast(weights, dtypes.float64) fn = math_ops.multiply(fn, weights) return fn -def _streaming_sparse_false_negative_at_k(labels, - predictions_idx, - k, - class_id=None, - weights=None, - name=None): +def _streaming_sparse_false_negative_at_k( + labels, predictions_idx, k, class_id=None, weights=None, name=None +): """Calculates weighted per step false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2470,29 +2695,35 @@ def _streaming_sparse_false_negative_at_k(labels, Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope(name, _at_k_name('false_negative', k, class_id=class_id), - (predictions_idx, labels, weights)) as scope: + with ops.name_scope( + name, _at_k_name('false_negative', k, class_id=class_id), + (predictions_idx, labels, weights) + ) as scope: fn = _sparse_false_negative_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights + ) batch_total_fn = math_ops.cast(math_ops.reduce_sum(fn), dtypes.float64) var = metric_variable([], dtypes.float64, name=scope) return var, state_ops.assign_add( - var, batch_total_fn, name='update', use_locking=True) + var, batch_total_fn, name='update', use_locking=True + ) @tf_export(v1=['metrics.recall_at_k']) -def recall_at_k(labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def recall_at_k( + labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes recall@k of the predictions with respect to sparse labels. If `class_id` is specified, we calculate recall by considering only the @@ -2560,32 +2791,39 @@ def recall_at_k(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall_at_k is not ' - 'supported when eager execution is enabled.') - - with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), - (predictions, labels, weights)) as scope: + raise RuntimeError( + 'tf.metrics.recall_at_k is not ' + 'supported when eager execution is enabled.' + ) + + with ops.name_scope( + name, _at_k_name('recall', k, class_id=class_id), + (predictions, labels, weights) + ) as scope: _, top_k_idx = nn.top_k(predictions, k) return recall_at_top_k( - labels=labels, - predictions_idx=top_k_idx, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope) + labels=labels, + predictions_idx=top_k_idx, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope + ) @tf_export(v1=['metrics.recall_at_top_k']) -def recall_at_top_k(labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def recall_at_top_k( + labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes recall@k of top-k predictions with respect to sparse labels. Differs from `recall_at_k` in that predictions must be in the form of top `k` @@ -2631,44 +2869,52 @@ class indices, whereas `recall_at_k` expects logits. Refer to `recall_at_k` `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), - (predictions_idx, labels, weights)) as scope: + with ops.name_scope( + name, _at_k_name('recall', k, class_id=class_id), + (predictions_idx, labels, weights) + ) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.cast(predictions_idx, dtypes.int64) tp, tp_update = _streaming_sparse_true_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights + ) fn, fn_update = _streaming_sparse_false_negative_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights + ) def compute_recall(_, tp, fn): return math_ops.div(tp, math_ops.add(tp, fn), name=scope) - metric = _aggregate_across_replicas(metrics_collections, compute_recall, tp, - fn) + metric = _aggregate_across_replicas( + metrics_collections, compute_recall, tp, fn + ) update = math_ops.div( - tp_update, math_ops.add(tp_update, fn_update), name='update') + tp_update, math_ops.add(tp_update, fn_update), name='update' + ) if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @tf_export(v1=['metrics.recall_at_thresholds']) -def recall_at_thresholds(labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def recall_at_thresholds( + labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes various recall values for different `thresholds` on `predictions`. The `recall_at_thresholds` function creates four local variables, @@ -2712,13 +2958,17 @@ def recall_at_thresholds(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall_at_thresholds is not ' - 'supported when eager execution is enabled.') - - with variable_scope.variable_scope(name, 'recall_at_thresholds', - (predictions, labels, weights)): + raise RuntimeError( + 'tf.metrics.recall_at_thresholds is not ' + 'supported when eager execution is enabled.' + ) + + with variable_scope.variable_scope( + name, 'recall_at_thresholds', (predictions, labels, weights) + ): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights, includes=('tp', 'fn')) + labels, predictions, thresholds, weights, includes=('tp', 'fn') + ) # Avoid division by zero. epsilon = 1e-7 @@ -2729,8 +2979,9 @@ def compute_recall(tp, fn, name): def recall_across_replicas(_, values): return compute_recall(values['tp'], values['fn'], 'value') - rec = _aggregate_across_replicas(metrics_collections, - recall_across_replicas, values) + rec = _aggregate_across_replicas( + metrics_collections, recall_across_replicas, values + ) update_op = compute_recall(update_ops['tp'], update_ops['fn'], 'update_op') if updates_collections: @@ -2740,12 +2991,14 @@ def recall_across_replicas(_, values): @tf_export(v1=['metrics.root_mean_squared_error']) -def root_mean_squared_error(labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def root_mean_squared_error( + labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the root mean squared error between the labels and predictions. The `root_mean_squared_error` function creates two local variables, @@ -2790,18 +3043,22 @@ def root_mean_squared_error(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.root_mean_squared_error is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.root_mean_squared_error is not ' + 'supported when eager execution is enabled.' + ) predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) - mse, update_mse_op = mean_squared_error(labels, predictions, weights, None, - None, name or - 'root_mean_squared_error') + predictions=predictions, labels=labels, weights=weights + ) + mse, update_mse_op = mean_squared_error( + labels, predictions, weights, None, None, name or 'root_mean_squared_error' + ) once_across_replicas = lambda _, mse: math_ops.sqrt(mse) # noqa: E731 - rmse = _aggregate_across_replicas(metrics_collections, once_across_replicas, - mse) + rmse = _aggregate_across_replicas( + metrics_collections, once_across_replicas, mse + ) update_rmse_op = math_ops.sqrt(update_mse_op) if updates_collections: @@ -2811,14 +3068,16 @@ def root_mean_squared_error(labels, @tf_export(v1=['metrics.sensitivity_at_specificity']) -def sensitivity_at_specificity(labels, - predictions, - specificity, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None): +def sensitivity_at_specificity( + labels, + predictions, + specificity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the specificity at a given sensitivity. The `sensitivity_at_specificity` function creates four local @@ -2870,22 +3129,26 @@ def sensitivity_at_specificity(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sensitivity_at_specificity is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.sensitivity_at_specificity is not ' + 'supported when eager execution is enabled.' + ) if specificity < 0 or specificity > 1: raise ValueError('`specificity` must be in the range [0, 1].') - with variable_scope.variable_scope(name, 'sensitivity_at_specificity', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'sensitivity_at_specificity', (predictions, labels, weights) + ): kepsilon = 1e-7 # to account for floating point imprecisions thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights) + labels, predictions, thresholds, weights + ) def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): specificities = math_ops.div(tn, tn + fp + kepsilon) @@ -2893,23 +3156,23 @@ def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the sensitivity: - return math_ops.div(tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, - name) + return math_ops.div( + tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, name + ) def sensitivity_across_replicas(_, values): - return compute_sensitivity_at_specificity(values['tp'], values['tn'], - values['fp'], values['fn'], - 'value') - - sensitivity = _aggregate_across_replicas(metrics_collections, - sensitivity_across_replicas, - values) - - update_op = compute_sensitivity_at_specificity(update_ops['tp'], - update_ops['tn'], - update_ops['fp'], - update_ops['fn'], - 'update_op') + return compute_sensitivity_at_specificity( + values['tp'], values['tn'], values['fp'], values['fn'], 'value' + ) + + sensitivity = _aggregate_across_replicas( + metrics_collections, sensitivity_across_replicas, values + ) + + update_op = compute_sensitivity_at_specificity( + update_ops['tp'], update_ops['tn'], update_ops['fp'], update_ops['fn'], + 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2937,37 +3200,45 @@ def _expand_and_tile(tensor, multiple, dim=0, name=None): """ if multiple < 1: raise ValueError('Invalid multiple %s, must be > 0.' % multiple) - with ops.name_scope(name, 'expand_and_tile', - (tensor, multiple, dim)) as scope: + with ops.name_scope( + name, 'expand_and_tile', (tensor, multiple, dim) + ) as scope: # Sparse. tensor = sparse_tensor.convert_to_tensor_or_sparse_tensor(tensor) if isinstance(tensor, sparse_tensor.SparseTensor): if dim < 0: expand_dims = array_ops.reshape( - array_ops.size(tensor.dense_shape) + dim, [1]) + array_ops.size(tensor.dense_shape) + dim, [1] + ) else: expand_dims = [dim] expanded_shape = array_ops.concat( - (array_ops.slice(tensor.dense_shape, [0], expand_dims), [1], - array_ops.slice(tensor.dense_shape, expand_dims, [-1])), - 0, - name='expanded_shape') + ( + array_ops.slice(tensor.dense_shape, [0], expand_dims), [1], + array_ops.slice(tensor.dense_shape, expand_dims, [-1]) + ), + 0, + name='expanded_shape' + ) expanded = sparse_ops.sparse_reshape( - tensor, shape=expanded_shape, name='expand') + tensor, shape=expanded_shape, name='expand' + ) if multiple == 1: return expanded return sparse_ops.sparse_concat( - dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope) + dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope + ) # Dense. expanded = array_ops.expand_dims( - tensor, dim if (dim >= 0) else (dim - 1), name='expand') + tensor, dim if (dim >= 0) else (dim - 1), name='expand' + ) if multiple == 1: return expanded ones = array_ops.ones_like(array_ops.shape(tensor)) - tile_multiples = array_ops.concat((ones[:dim], (multiple,), ones[dim:]), - 0, - name='multiples') + tile_multiples = array_ops.concat( + (ones[:dim], (multiple, ), ones[dim:]), 0, name='multiples' + ) return array_ops.tile(expanded, tile_multiples, name=scope) @@ -2993,7 +3264,7 @@ def _num_relevant(labels, k): """ if k < 1: raise ValueError('Invalid k=%s.' % k) - with ops.name_scope(None, 'num_relevant', (labels,)) as scope: + with ops.name_scope(None, 'num_relevant', (labels, )) as scope: # For SparseTensor, calculate separate count for each row. labels = sparse_tensor.convert_to_tensor_or_sparse_tensor(labels) if isinstance(labels, sparse_tensor.SparseTensor): @@ -3002,10 +3273,12 @@ def _num_relevant(labels, k): # The relevant values for each (d1, ... dN) is the minimum of k and the # number of labels along the last dimension that are non-negative. num_labels = math_ops.reduce_sum( - array_ops.where_v2( - math_ops.greater_equal(labels, 0), array_ops.ones_like(labels), - array_ops.zeros_like(labels)), - axis=-1) + array_ops.where_v2( + math_ops.greater_equal(labels, 0), array_ops.ones_like(labels), + array_ops.zeros_like(labels) + ), + axis=-1 + ) return math_ops.minimum(num_labels, k, name=scope) @@ -3041,10 +3314,12 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): Raises: ValueError: if the last dimension of predictions_idx is not set. """ - with ops.name_scope(None, 'average_precision', - (predictions_idx, labels)) as scope: + with ops.name_scope( + None, 'average_precision', (predictions_idx, labels) + ) as scope: predictions_idx = math_ops.cast( - predictions_idx, dtypes.int64, name='predictions_idx') + predictions_idx, dtypes.int64, name='predictions_idx' + ) if predictions_idx.get_shape().ndims == 0: raise ValueError('The rank of predictions_idx must be at least 1.') k = predictions_idx.get_shape().as_list()[-1] @@ -3056,11 +3331,13 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # prediction for each k, so we can calculate separate true positive values # for each k. predictions_idx_per_k = array_ops.expand_dims( - predictions_idx, -1, name='predictions_idx_per_k') + predictions_idx, -1, name='predictions_idx_per_k' + ) # Replicate labels k times to produce [D1, ... DN, k, num_labels] tensor. labels_per_k = _expand_and_tile( - labels, multiple=k, dim=-1, name='labels_per_k') + labels, multiple=k, dim=-1, name='labels_per_k' + ) # The following tensors are all of shape [D1, ... DN, k], containing values # per row, per k value. @@ -3075,35 +3352,44 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # `relevant_precision_per_k` (float64) - Relevant precisions; i.e., # precisions at all k for which relevance indicator is true. relevant_per_k = _sparse_true_positive_at_k( - labels_per_k, predictions_idx_per_k, name='relevant_per_k') + labels_per_k, predictions_idx_per_k, name='relevant_per_k' + ) tp_per_k = math_ops.cumsum(relevant_per_k, axis=-1, name='tp_per_k') retrieved_per_k = math_ops.cumsum( - array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k') + array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k' + ) precision_per_k = math_ops.div( - math_ops.cast(tp_per_k, dtypes.float64), - math_ops.cast(retrieved_per_k, dtypes.float64), - name='precision_per_k') + math_ops.cast(tp_per_k, dtypes.float64), + math_ops.cast(retrieved_per_k, dtypes.float64), + name='precision_per_k' + ) relevant_precision_per_k = math_ops.multiply( - precision_per_k, - math_ops.cast(relevant_per_k, dtypes.float64), - name='relevant_precision_per_k') + precision_per_k, + math_ops.cast(relevant_per_k, dtypes.float64), + name='relevant_precision_per_k' + ) # Reduce along k dimension to get the sum, yielding a [D1, ... DN] tensor. precision_sum = math_ops.reduce_sum( - relevant_precision_per_k, axis=(-1,), name='precision_sum') + relevant_precision_per_k, axis=(-1, ), name='precision_sum' + ) # Divide by number of relevant items to get average precision. These are # the "num_relevant_items" and "AveP" terms from the formula above. - num_relevant_items = math_ops.cast(_num_relevant(labels, k), dtypes.float64) + num_relevant_items = math_ops.cast( + _num_relevant(labels, k), dtypes.float64 + ) return math_ops.div(precision_sum, num_relevant_items, name=scope) -def _streaming_sparse_average_precision_at_top_k(labels, - predictions_idx, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def _streaming_sparse_average_precision_at_top_k( + labels, + predictions_idx, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes average precision@k of predictions with respect to sparse labels. `sparse_average_precision_at_top_k` creates two local variables, @@ -3148,19 +3434,22 @@ def _streaming_sparse_average_precision_at_top_k(labels, update: `Operation` that increments variables appropriately, and whose value matches `metric`. """ - with ops.name_scope(name, 'average_precision_at_top_k', - (predictions_idx, labels, weights)) as scope: + with ops.name_scope( + name, 'average_precision_at_top_k', (predictions_idx, labels, weights) + ) as scope: # Calculate per-example average precision, and apply weights. average_precision = _sparse_average_precision_at_top_k( - predictions_idx=predictions_idx, labels=labels) + predictions_idx=predictions_idx, labels=labels + ) if weights is not None: weights = weights_broadcast_ops.broadcast_weights( - math_ops.cast(weights, dtypes.float64), average_precision) + math_ops.cast(weights, dtypes.float64), average_precision + ) average_precision = math_ops.multiply(average_precision, weights) # Create accumulation variables and update ops for max average precision and # total average precision. - with ops.name_scope(None, 'max', (average_precision,)) as max_scope: + with ops.name_scope(None, 'max', (average_precision, )) as max_scope: # `max` is the max possible precision. Since max for any row is 1.0: # - For the unweighted case, this is just the number of rows. # - For the weighted case, it's the sum of the weights broadcast across @@ -3168,23 +3457,27 @@ def _streaming_sparse_average_precision_at_top_k(labels, max_var = metric_variable([], dtypes.float64, name=max_scope) if weights is None: batch_max = math_ops.cast( - array_ops.size(average_precision, name='batch_max'), dtypes.float64) + array_ops.size(average_precision, name='batch_max'), dtypes.float64 + ) else: batch_max = math_ops.reduce_sum(weights, name='batch_max') max_update = state_ops.assign_add( - max_var, batch_max, name='update', use_locking=True) - with ops.name_scope(None, 'total', (average_precision,)) as total_scope: + max_var, batch_max, name='update', use_locking=True + ) + with ops.name_scope(None, 'total', (average_precision, )) as total_scope: total_var = metric_variable([], dtypes.float64, name=total_scope) batch_total = math_ops.reduce_sum(average_precision, name='batch_total') total_update = state_ops.assign_add( - total_var, batch_total, name='update', use_locking=True) + total_var, batch_total, name='update', use_locking=True + ) # Divide total by max to get mean, for both vars and the update ops. def precision_across_replicas(_, total_var, max_var): return _safe_scalar_div(total_var, max_var, name='mean') mean_average_precision = _aggregate_across_replicas( - metrics_collections, precision_across_replicas, total_var, max_var) + metrics_collections, precision_across_replicas, total_var, max_var + ) update = _safe_scalar_div(total_update, max_update, name=scope) if updates_collections: @@ -3211,59 +3504,69 @@ def _clean_out_of_range_indices(labels, num_classes): def _labels_is_sparse(): """Returns true is `labels` is a sparse tensor.""" return isinstance( - labels, (sparse_tensor.SparseTensor, sparse_tensor.SparseTensorValue)) + labels, (sparse_tensor.SparseTensor, sparse_tensor.SparseTensorValue) + ) def _clean_out_of_range(values): """Replaces by -1 any large out-of-range `values`.""" return array_ops.where_v2( - math_ops.greater_equal(values, num_classes), - -1 * array_ops.ones_like(values), values) + math_ops.greater_equal(values, num_classes), + -1 * array_ops.ones_like(values), values + ) def _clean_labels_out_of_range(): """Replaces by -1 ane large out-of-range values in `labels`.""" if _labels_is_sparse(): return type(labels)( - indices=labels.indices, - values=_clean_out_of_range(labels.values), - dense_shape=labels.dense_shape) + indices=labels.indices, + values=_clean_out_of_range(labels.values), + dense_shape=labels.dense_shape + ) else: return _clean_out_of_range(labels) max_labels = math_ops.reduce_max( - labels.values if _labels_is_sparse() else labels) + labels.values if _labels_is_sparse() else labels + ) return control_flow_ops.cond( - math_ops.greater_equal(max_labels, num_classes), - _clean_labels_out_of_range, lambda: labels) + math_ops.greater_equal(max_labels, num_classes), + _clean_labels_out_of_range, lambda: labels + ) @tf_export(v1=['metrics.sparse_average_precision_at_k']) @deprecated(None, 'Use average_precision_at_k instead') -def sparse_average_precision_at_k(labels, - predictions, - k, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def sparse_average_precision_at_k( + labels, + predictions, + k, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Renamed to `average_precision_at_k`, please use that method instead.""" return average_precision_at_k( - labels=labels, - predictions=predictions, - k=k, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) + labels=labels, + predictions=predictions, + k=k, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name + ) @tf_export(v1=['metrics.average_precision_at_k']) -def average_precision_at_k(labels, - predictions, - k, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def average_precision_at_k( + labels, + predictions, + k, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes average precision@k of predictions with respect to sparse labels. `average_precision_at_k` creates two local variables, @@ -3317,13 +3620,16 @@ def average_precision_at_k(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sparse_average_precision_at_k is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.sparse_average_precision_at_k is not ' + 'supported when eager execution is enabled.' + ) if k < 1: raise ValueError('Invalid k=%s.' % k) - with ops.name_scope(name, _at_k_name('average_precision', k), - (predictions, labels, weights)) as scope: + with ops.name_scope( + name, _at_k_name('average_precision', k), (predictions, labels, weights) + ) as scope: # Calculate top k indices to produce [D1, ... DN, k] tensor. _, predictions_idx = nn.top_k(predictions, k) # The documentation states that labels should be in [0, ..., num_classes), @@ -3331,20 +3637,21 @@ def average_precision_at_k(labels, # For conformity with the documentation, any label >= num_classes, which is # ignored, is replaced by -1. labels = _clean_out_of_range_indices( - labels, math_ops.cast(array_ops.shape(predictions)[-1], dtypes.int64)) + labels, math_ops.cast(array_ops.shape(predictions)[-1], dtypes.int64) + ) return _streaming_sparse_average_precision_at_top_k( - labels=labels, - predictions_idx=predictions_idx, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope) - - -def _sparse_false_positive_at_k(labels, - predictions_idx, - class_id=None, - weights=None): + labels=labels, + predictions_idx=predictions_idx, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope + ) + + +def _sparse_false_positive_at_k( + labels, predictions_idx, class_id=None, weights=None +): """Calculates false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3370,27 +3677,28 @@ def _sparse_false_positive_at_k(labels, Returns: A [D1, ... DN] `Tensor` of false positive counts. """ - with ops.name_scope(None, 'false_positives', - (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, - class_id) + with ops.name_scope( + None, 'false_positives', (predictions_idx, labels, weights) + ): + labels, predictions_idx = _maybe_select_class_id( + labels, predictions_idx, class_id + ) fp = sets.set_size( - sets.set_difference(predictions_idx, labels, aminusb=True)) + sets.set_difference(predictions_idx, labels, aminusb=True) + ) fp = math_ops.cast(fp, dtypes.float64) if weights is not None: with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, fp),)): + (weights_broadcast_ops.assert_broadcastable(weights, fp), ) + ): weights = math_ops.cast(weights, dtypes.float64) fp = math_ops.multiply(fp, weights) return fp -def _streaming_sparse_false_positive_at_k(labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - name=None): +def _streaming_sparse_false_positive_at_k( + labels, predictions_idx, k=None, class_id=None, weights=None, name=None +): """Calculates weighted per step false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3423,29 +3731,35 @@ def _streaming_sparse_false_positive_at_k(labels, Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope(name, _at_k_name('false_positive', k, class_id=class_id), - (predictions_idx, labels, weights)) as scope: + with ops.name_scope( + name, _at_k_name('false_positive', k, class_id=class_id), + (predictions_idx, labels, weights) + ) as scope: fp = _sparse_false_positive_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights + ) batch_total_fp = math_ops.cast(math_ops.reduce_sum(fp), dtypes.float64) var = metric_variable([], dtypes.float64, name=scope) return var, state_ops.assign_add( - var, batch_total_fp, name='update', use_locking=True) + var, batch_total_fp, name='update', use_locking=True + ) @tf_export(v1=['metrics.precision_at_top_k']) -def precision_at_top_k(labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def precision_at_top_k( + labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes precision@k of the predictions with respect to sparse labels. Differs from `sparse_precision_at_k` in that predictions must be in the form @@ -3493,34 +3807,42 @@ def precision_at_top_k(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision_at_top_k is not ' - 'supported when eager execution is enabled.') - - with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), - (predictions_idx, labels, weights)) as scope: + raise RuntimeError( + 'tf.metrics.precision_at_top_k is not ' + 'supported when eager execution is enabled.' + ) + + with ops.name_scope( + name, _at_k_name('precision', k, class_id=class_id), + (predictions_idx, labels, weights) + ) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.cast(predictions_idx, dtypes.int64) tp, tp_update = _streaming_sparse_true_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights + ) fp, fp_update = _streaming_sparse_false_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights + ) def precision_across_replicas(_, tp, fp): return math_ops.div(tp, math_ops.add(tp, fp), name=scope) - metric = _aggregate_across_replicas(metrics_collections, - precision_across_replicas, tp, fp) + metric = _aggregate_across_replicas( + metrics_collections, precision_across_replicas, tp, fp + ) update = math_ops.div( - tp_update, math_ops.add(tp_update, fp_update), name='update') + tp_update, math_ops.add(tp_update, fp_update), name='update' + ) if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @@ -3528,35 +3850,40 @@ def precision_across_replicas(_, tp, fp): @tf_export(v1=['metrics.sparse_precision_at_k']) @deprecated(None, 'Use precision_at_k instead') -def sparse_precision_at_k(labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def sparse_precision_at_k( + labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Renamed to `precision_at_k`, please use that method instead.""" return precision_at_k( - labels=labels, - predictions=predictions, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) + labels=labels, + predictions=predictions, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name + ) @tf_export(v1=['metrics.precision_at_k']) -def precision_at_k(labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def precision_at_k( + labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes precision@k of the predictions with respect to sparse labels. If `class_id` is specified, we calculate precision by considering only the @@ -3625,32 +3952,39 @@ def precision_at_k(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sparse_precision_at_k is not ' - 'supported when eager execution is enabled.') - - with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), - (predictions, labels, weights)) as scope: + raise RuntimeError( + 'tf.metrics.sparse_precision_at_k is not ' + 'supported when eager execution is enabled.' + ) + + with ops.name_scope( + name, _at_k_name('precision', k, class_id=class_id), + (predictions, labels, weights) + ) as scope: _, top_k_idx = nn.top_k(predictions, k) return precision_at_top_k( - labels=labels, - predictions_idx=top_k_idx, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope) + labels=labels, + predictions_idx=top_k_idx, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope + ) @tf_export(v1=['metrics.specificity_at_sensitivity']) -def specificity_at_sensitivity(labels, - predictions, - sensitivity, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None): +def specificity_at_sensitivity( + labels, + predictions, + sensitivity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None +): """Computes the specificity at a given sensitivity. The `specificity_at_sensitivity` function creates four local @@ -3702,22 +4036,26 @@ def specificity_at_sensitivity(labels, RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.specificity_at_sensitivity is not ' - 'supported when eager execution is enabled.') + raise RuntimeError( + 'tf.metrics.specificity_at_sensitivity is not ' + 'supported when eager execution is enabled.' + ) if sensitivity < 0 or sensitivity > 1: raise ValueError('`sensitivity` must be in the range [0, 1].') - with variable_scope.variable_scope(name, 'specificity_at_sensitivity', - (predictions, labels, weights)): + with variable_scope.variable_scope( + name, 'specificity_at_sensitivity', (predictions, labels, weights) + ): kepsilon = 1e-7 # to account for floating point imprecisions thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 - kepsilon] values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights) + labels, predictions, thresholds, weights + ) def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): """Computes the specificity at the given sensitivity. @@ -3738,30 +4076,31 @@ def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): # whether we should use the first or last index in case of ties. min_val = math_ops.reduce_min(math_ops.abs(sensitivities - sensitivity)) indices_at_minval = math_ops.equal( - math_ops.abs(sensitivities - sensitivity), min_val) + math_ops.abs(sensitivities - sensitivity), min_val + ) indices_at_minval = math_ops.cast(indices_at_minval, dtypes.int64) indices_at_minval = math_ops.cumsum(indices_at_minval) tf_index = math_ops.argmax(indices_at_minval, 0) tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the specificity: - return math_ops.div(tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, - name) + return math_ops.div( + tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, name + ) def specificity_across_replicas(_, values): - return compute_specificity_at_sensitivity(values['tp'], values['tn'], - values['fp'], values['fn'], - 'value') - - specificity = _aggregate_across_replicas(metrics_collections, - specificity_across_replicas, - values) - - update_op = compute_specificity_at_sensitivity(update_ops['tp'], - update_ops['tn'], - update_ops['fp'], - update_ops['fn'], - 'update_op') + return compute_specificity_at_sensitivity( + values['tp'], values['tn'], values['fp'], values['fn'], 'value' + ) + + specificity = _aggregate_across_replicas( + metrics_collections, specificity_across_replicas, values + ) + + update_op = compute_specificity_at_sensitivity( + update_ops['tp'], update_ops['tn'], update_ops['fp'], update_ops['fn'], + 'update_op' + ) if updates_collections: ops.add_to_collections(updates_collections, update_op) diff --git a/easy_rec/python/core/learning_schedules.py b/easy_rec/python/core/learning_schedules.py index fcd1446e8..d8944fc27 100644 --- a/easy_rec/python/core/learning_schedules.py +++ b/easy_rec/python/core/learning_schedules.py @@ -22,14 +22,16 @@ tf = tf.compat.v1 -def exponential_decay_with_burnin(global_step, - learning_rate_base, - learning_rate_decay_steps, - learning_rate_decay_factor, - burnin_learning_rate=0.0, - burnin_steps=0, - min_learning_rate=0.0, - staircase=True): +def exponential_decay_with_burnin( + global_step, + learning_rate_base, + learning_rate_decay_steps, + learning_rate_decay_factor, + burnin_learning_rate=0.0, + burnin_steps=0, + min_learning_rate=0.0, + staircase=True +): """Exponential decay schedule with burn-in period. In this schedule, learning rate is fixed at burnin_learning_rate @@ -57,28 +59,34 @@ def exponential_decay_with_burnin(global_step, burnin_rate = learning_rate_base else: slope = (learning_rate_base - burnin_learning_rate) / burnin_steps - burnin_rate = slope * tf.cast(global_step, - tf.float32) + burnin_learning_rate + burnin_rate = slope * tf.cast( + global_step, tf.float32 + ) + burnin_learning_rate post_burnin_learning_rate = tf.train.exponential_decay( - learning_rate_base, - global_step - burnin_steps, - learning_rate_decay_steps, - learning_rate_decay_factor, - staircase=staircase) + learning_rate_base, + global_step - burnin_steps, + learning_rate_decay_steps, + learning_rate_decay_factor, + staircase=staircase + ) return tf.maximum( - tf.where( - tf.less(tf.cast(global_step, tf.int32), tf.constant(burnin_steps)), - burnin_rate, post_burnin_learning_rate), - min_learning_rate, - name='learning_rate') - - -def cosine_decay_with_warmup(global_step, - learning_rate_base, - total_steps, - warmup_learning_rate=0.0, - warmup_steps=0, - hold_base_rate_steps=0): + tf.where( + tf.less(tf.cast(global_step, tf.int32), tf.constant(burnin_steps)), + burnin_rate, post_burnin_learning_rate + ), + min_learning_rate, + name='learning_rate' + ) + + +def cosine_decay_with_warmup( + global_step, + learning_rate_base, + total_steps, + warmup_learning_rate=0.0, + warmup_steps=0, + hold_base_rate_steps=0 +): """Cosine decay schedule with warm up period. Cosine annealing learning rate as described in: @@ -105,25 +113,38 @@ def cosine_decay_with_warmup(global_step, or if warmup_steps is larger than total_steps. """ if learning_rate_base < warmup_learning_rate: - raise ValueError('learning_rate_base must be larger ' - 'or equal to warmup_learning_rate.') + raise ValueError( + 'learning_rate_base must be larger ' + 'or equal to warmup_learning_rate.' + ) if total_steps < warmup_steps: - raise ValueError('total_steps must be larger or equal to ' 'warmup_steps.') - learning_rate = 0.5 * learning_rate_base * (1 + tf.cos( + raise ValueError( + 'total_steps must be larger or equal to ' + 'warmup_steps.' + ) + learning_rate = 0.5 * learning_rate_base * ( + 1 + tf.cos( np.pi * - (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps) / - float(total_steps - warmup_steps - hold_base_rate_steps))) + (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps) + / float(total_steps - warmup_steps - hold_base_rate_steps) + ) + ) if hold_base_rate_steps > 0: - learning_rate = tf.where(global_step > warmup_steps + hold_base_rate_steps, - learning_rate, learning_rate_base) + learning_rate = tf.where( + global_step > warmup_steps + hold_base_rate_steps, learning_rate, + learning_rate_base + ) if warmup_steps > 0: slope = (learning_rate_base - warmup_learning_rate) / warmup_steps - warmup_rate = slope * tf.cast(global_step, - tf.float32) + warmup_learning_rate - learning_rate = tf.where(global_step < warmup_steps, warmup_rate, - learning_rate) + warmup_rate = slope * tf.cast( + global_step, tf.float32 + ) + warmup_learning_rate + learning_rate = tf.where( + global_step < warmup_steps, warmup_rate, learning_rate + ) return tf.where( - global_step > total_steps, 0.0, learning_rate, name='learning_rate') + global_step > total_steps, 0.0, learning_rate, name='learning_rate' + ) def manual_stepping(global_step, boundaries, rates, warmup=False): @@ -154,16 +175,18 @@ def manual_stepping(global_step, boundaries, rates, warmup=False): 2. len(rates) == len(boundaries) + 1 3. boundaries[0] != 0 """ - if any([b < 0 for b in boundaries]) or any( - [not isinstance(b, int) for b in boundaries]): + if any([b < 0 for b in boundaries] + ) or any([not isinstance(b, int) for b in boundaries]): raise ValueError('boundaries must be a list of positive integers') if any([bnext <= b for bnext, b in zip(boundaries[1:], boundaries[:-1])]): raise ValueError('Entries in boundaries must be strictly increasing.') if any([not isinstance(r, float) for r in rates]): raise ValueError('Learning rates must be floats') if len(rates) != len(boundaries) + 1: - raise ValueError('Number of provided learning rates must exceed ' - 'number of boundary points by exactly 1.') + raise ValueError( + 'Number of provided learning rates must exceed ' + 'number of boundary points by exactly 1.' + ) if boundaries and boundaries[0] == 0: raise ValueError('First step cannot be zero.') @@ -178,22 +201,26 @@ def manual_stepping(global_step, boundaries, rates, warmup=False): boundaries = [0] + boundaries num_boundaries = len(boundaries) rate_index = tf.reduce_max( - tf.where( - tf.greater_equal(global_step, boundaries), - list(range(num_boundaries)), [0] * num_boundaries)) + tf.where( + tf.greater_equal(global_step, boundaries), list(range(num_boundaries)), + [0] * num_boundaries + ) + ) return tf.reduce_sum( - rates * tf.one_hot(rate_index, depth=num_boundaries), - name='learning_rate') - - -def transformer_policy(global_step, - learning_rate, - d_model, - warmup_steps, - step_scaling_rate=1.0, - max_lr=None, - coefficient=1.0, - dtype=tf.float32): + rates * tf.one_hot(rate_index, depth=num_boundaries), name='learning_rate' + ) + + +def transformer_policy( + global_step, + learning_rate, + d_model, + warmup_steps, + step_scaling_rate=1.0, + max_lr=None, + coefficient=1.0, + dtype=tf.float32 +): """Transformer's learning rate schedule. Transformer's learning rate policy from @@ -219,8 +246,9 @@ def transformer_policy(global_step, step_num *= step_scaling_rate ws *= step_scaling_rate - decay = coefficient * d_model**-0.5 * tf.minimum((step_num + 1) * ws**-1.5, - (step_num + 1)**-0.5) + decay = coefficient * d_model**-0.5 * tf.minimum( + (step_num + 1) * ws**-1.5, (step_num + 1)**-0.5 + ) new_lr = decay * learning_rate if max_lr is not None: diff --git a/easy_rec/python/core/metrics.py b/easy_rec/python/core/metrics.py index bd7cb0976..2169c9960 100644 --- a/easy_rec/python/core/metrics.py +++ b/easy_rec/python/core/metrics.py @@ -2,20 +2,15 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import json import logging -import os -from collections import defaultdict - import numpy as np +import os import tensorflow as tf +from collections import defaultdict from sklearn import metrics as sklearn_metrics -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import array_ops, math_ops, state_ops, variable_scope # NOQA from easy_rec.python.utils.estimator_utils import get_task_index_and_num -from easy_rec.python.utils.io_util import read_data_from_json_path -from easy_rec.python.utils.io_util import save_data_to_json_path +from easy_rec.python.utils.io_util import read_data_from_json_path, save_data_to_json_path # NOQA from easy_rec.python.utils.shape_utils import get_shape_list if tf.__version__ >= '2.0': @@ -33,7 +28,7 @@ def max_f1(label, predictions): num_thresholds = 200 kepsilon = 1e-7 thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] @@ -43,9 +38,11 @@ def max_f1(label, predictions): for threshold in thresholds: pred = (predictions > threshold) precision, precision_update_op = metrics_tf.precision( - labels=label, predictions=pred, name='precision_%s' % threshold) + labels=label, predictions=pred, name='precision_%s' % threshold + ) recall, recall_update_op = metrics_tf.recall( - labels=label, predictions=pred, name='recall_%s' % threshold) + labels=label, predictions=pred, name='recall_%s' % threshold + ) f1_score = (2 * precision * recall) / (precision + recall + 1e-12) precision_update_ops.append(precision_update_op) recall_update_ops.append(recall_update_op) @@ -121,32 +118,37 @@ def value_pyfunc(pos_neg_arr, total_pos_neg): auc += (total_pos - partial_sum_pos) * pos_neg_arr[0][i] * 2 auc += pos_neg_arr[0][i] * pos_neg_arr[1][i] auc = np.double(auc) / np.double(total_pos * total_neg * 2) - logging.info('fast_auc[%s]: total_pos=%d total_neg=%d total=%d' % - (name, total_pos, total_neg, total_pos + total_neg)) + logging.info( + 'fast_auc[%s]: total_pos=%d total_neg=%d total=%d' % + (name, total_pos, total_neg, total_pos + total_neg) + ) return np.float32(auc) with variable_scope.variable_scope(name_or_scope=name), tf.name_scope(name): neg_pos_var = variable_scope.get_variable( - name='neg_pos_cnt', - shape=[2, num_thresholds + 1], - trainable=False, - collections=[tf.GraphKeys.METRIC_VARIABLES], - initializer=tf.zeros_initializer(), - dtype=tf.int64) + name='neg_pos_cnt', + shape=[2, num_thresholds + 1], + trainable=False, + collections=[tf.GraphKeys.METRIC_VARIABLES], + initializer=tf.zeros_initializer(), + dtype=tf.int64 + ) total_var = variable_scope.get_variable( - name='total_cnt', - shape=[2], - trainable=False, - collections=[tf.GraphKeys.METRIC_VARIABLES], - initializer=tf.zeros_initializer(), - dtype=tf.int64) + name='total_cnt', + shape=[2], + trainable=False, + collections=[tf.GraphKeys.METRIC_VARIABLES], + initializer=tf.zeros_initializer(), + dtype=tf.int64 + ) pred_bins = math_ops.cast(predictions * num_thresholds, dtype=tf.int32) labels = math_ops.cast(labels, dtype=tf.int32) labels = array_ops.reshape(labels, [-1, 1]) pred_bins = array_ops.reshape(pred_bins, [-1, 1]) update_op0 = state_ops.scatter_nd_add( - neg_pos_var, tf.concat([labels, pred_bins], axis=1), - array_ops.ones(tf.shape(labels)[0], dtype=tf.int64)) + neg_pos_var, tf.concat([labels, pred_bins], axis=1), + array_ops.ones(tf.shape(labels)[0], dtype=tf.int64) + ) total_pos = math_ops.reduce_sum(labels) total_neg = array_ops.shape(labels)[0] - total_pos total_add = math_ops.cast(tf.stack([total_neg, total_pos]), dtype=tf.int64) @@ -155,11 +157,9 @@ def value_pyfunc(pos_neg_arr, total_pos_neg): tf.float32), tf.group([update_op0, update_op1]) -def _distribute_separated_auc_impl(labels, - predictions, - keys, - reduction='mean', - metric_name='sepatated_auc'): +def _distribute_separated_auc_impl( + labels, predictions, keys, reduction='mean', metric_name='sepatated_auc' +): """Computes the AUC group by the key separately. Args: @@ -182,10 +182,13 @@ def _distribute_separated_auc_impl(labels, tf_config = json.loads(os.environ['TF_CONFIG']) cur_job_name = tf_config['task']['type'] cur_task_index, task_num = get_task_index_and_num() - cur_work_device = 'job_' + cur_job_name + '__' + 'task_' + str(cur_task_index) + cur_work_device = 'job_' + cur_job_name + '__' + 'task_' + str( + cur_task_index + ) eval_tmp_results_dir = os.environ['eval_tmp_results_dir'] assert tf.gfile.IsDirectory( - eval_tmp_results_dir), 'eval_tmp_results_dir not exists' + eval_tmp_results_dir + ), 'eval_tmp_results_dir not exists' def update_pyfunc(labels, predictions, keys): for label, prediction, key in zip(labels, predictions, keys): @@ -199,8 +202,9 @@ def update_pyfunc(labels, predictions, keys): elif reduction == 'mean_by_positive_num': separated_weights[key] += label.item() for name, data in zip( - ['separated_label', 'separated_prediction', 'separated_weights'], - [separated_label, separated_prediction, separated_weights]): + ['separated_label', 'separated_prediction', 'separated_weights'], + [separated_label, separated_prediction, separated_weights] + ): cur_json_name = metric_name + '__' + cur_work_device + '__' + name + '.json' cur_json_path = os.path.join(eval_tmp_results_dir, cur_json_name) save_data_to_json_path(cur_json_path, data) @@ -209,32 +213,38 @@ def value_pyfunc(): for task_i in range(1, task_num): work_device_i = 'job_worker__task_' + str(task_i) for name in [ - 'separated_label', 'separated_prediction', 'separated_weights' + 'separated_label', 'separated_prediction', 'separated_weights' ]: json_name_i = metric_name + '__' + work_device_i + '__' + name + '.json' json_path_i = os.path.join(eval_tmp_results_dir, json_name_i) data_i = read_data_from_json_path(json_path_i) if (name == 'separated_label'): - separated_label.update({ + separated_label.update( + { key: separated_label.get(key, []) + data_i.get(key, []) - for key in set( - list(separated_label.keys()) + list(data_i.keys())) - }) + for key in + set(list(separated_label.keys()) + list(data_i.keys())) + } + ) elif (name == 'separated_prediction'): - separated_prediction.update({ + separated_prediction.update( + { key: separated_prediction.get(key, []) + data_i.get(key, []) - for key in set( - list(separated_prediction.keys()) + list(data_i.keys())) - }) + for key in + set(list(separated_prediction.keys()) + list(data_i.keys())) + } + ) elif (name == 'separated_weights'): if reduction == 'mean': separated_weights.update(data_i) else: - separated_weights.update({ + separated_weights.update( + { key: separated_weights.get(key, 0) + data_i.get(key, 0) - for key in set( - list(separated_weights.keys()) + list(data_i.keys())) - }) + for key in + set(list(separated_weights.keys()) + list(data_i.keys())) + } + ) else: assert False, 'Not supported name {}'.format(name) metrics = [] @@ -273,7 +283,8 @@ def gauc(labels, predictions, uids, reduction='mean'): """ if os.environ.get('distribute_eval') == 'True': return _distribute_separated_auc_impl( - labels, predictions, uids, reduction, metric_name='gauc') + labels, predictions, uids, reduction, metric_name='gauc' + ) return _separated_auc_impl(labels, predictions, uids, reduction) @@ -293,15 +304,14 @@ def session_auc(labels, predictions, session_ids, reduction='mean'): """ if os.environ.get('distribute_eval') == 'True': return _distribute_separated_auc_impl( - labels, predictions, session_ids, reduction, metric_name='session_auc') + labels, predictions, session_ids, reduction, metric_name='session_auc' + ) return _separated_auc_impl(labels, predictions, session_ids, reduction) -def metric_learning_recall_at_k(k, - embeddings, - labels, - session_ids=None, - embed_normed=False): +def metric_learning_recall_at_k( + k, embeddings, labels, session_ids=None, embed_normed=False +): """Computes the recall_at_k metric for metric learning. Args: @@ -312,6 +322,7 @@ def metric_learning_recall_at_k(k, embed_normed: indicator of whether the input embeddings are l2_normalized """ from easy_rec.python.core.easyrec_metrics import metrics_tf + # make sure embedding should be l2-normalized if not embed_normed: embeddings = tf.nn.l2_normalize(embeddings, axis=1) @@ -324,16 +335,18 @@ def metric_learning_recall_at_k(k, labels_equal = tf.equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1)) if session_ids is not None and session_ids is not labels: sessions_equal = tf.equal( - tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1)) + tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1) + ) labels_equal = tf.logical_and(sessions_equal, labels_equal) mask = tf.logical_and(indices_not_equal, labels_equal) mask_pos = tf.where( - mask, sim_mat, - -array_ops.ones_like(sim_mat)) # shape: (batch_size, batch_size) + mask, sim_mat, -array_ops.ones_like(sim_mat) + ) # shape: (batch_size, batch_size) if isinstance(k, int): _, pos_top_k_idx = tf.nn.top_k(mask_pos, k) # shape: (batch_size, k) return metrics_tf.recall_at_k( - labels=tf.to_int64(pos_top_k_idx), predictions=sim_mat, k=k) + labels=tf.to_int64(pos_top_k_idx), predictions=sim_mat, k=k + ) if any((isinstance(k, list), isinstance(k, tuple), isinstance(k, set))): metrics = {} for kk in k: @@ -341,18 +354,18 @@ def metric_learning_recall_at_k(k, continue _, pos_top_k_idx = tf.nn.top_k(mask_pos, kk) metrics['recall@' + str(kk)] = metrics_tf.recall_at_k( - labels=tf.to_int64(pos_top_k_idx), predictions=sim_mat, k=kk) + labels=tf.to_int64(pos_top_k_idx), predictions=sim_mat, k=kk + ) return metrics else: raise ValueError('k should be a `int` or a list/tuple/set of int.') -def metric_learning_average_precision_at_k(k, - embeddings, - labels, - session_ids=None, - embed_normed=False): +def metric_learning_average_precision_at_k( + k, embeddings, labels, session_ids=None, embed_normed=False +): from easy_rec.python.core.easyrec_metrics import metrics_tf + # make sure embedding should be l2-normalized if not embed_normed: embeddings = tf.nn.l2_normalize(embeddings, axis=1) @@ -363,7 +376,8 @@ def metric_learning_average_precision_at_k(k, mask = tf.equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1)) if session_ids is not None and session_ids is not labels: sessions_equal = tf.equal( - tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1)) + tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1) + ) mask = tf.logical_and(sessions_equal, mask) label_indices = _get_matrix_mask_indices(mask) if isinstance(k, int): @@ -374,7 +388,8 @@ def metric_learning_average_precision_at_k(k, if kk < 1: continue metrics['MAP@' + str(kk)] = metrics_tf.average_precision_at_k( - label_indices, sim_mat, kk) + label_indices, sim_mat, kk + ) return metrics else: raise ValueError('k should be a `int` or a list/tuple/set of int.') @@ -386,7 +401,8 @@ def _get_matrix_mask_indices(matrix, num_rows=None): indices = tf.where(matrix) num_indices = tf.shape(indices)[0] elem_per_row = tf.bincount( - tf.cast(indices[:, 0], tf.int32), minlength=num_rows) + tf.cast(indices[:, 0], tf.int32), minlength=num_rows + ) max_elem_per_row = tf.reduce_max(elem_per_row) row_start = tf.concat([[0], tf.cumsum(elem_per_row[:-1])], axis=0) r = tf.range(max_elem_per_row) @@ -395,7 +411,8 @@ def _get_matrix_mask_indices(matrix, num_rows=None): result = tf.gather(indices[:, 1], idx) # replace invalid elements with -1 result = tf.where( - tf.expand_dims(elem_per_row, 1) > r, result, -array_ops.ones_like(result)) + tf.expand_dims(elem_per_row, 1) > r, result, -array_ops.ones_like(result) + ) max_index_per_row = tf.reduce_max(result, axis=1, keepdims=True) max_index_per_row = tf.tile(max_index_per_row, [1, max_elem_per_row]) result = tf.where(result >= 0, result, max_index_per_row) diff --git a/easy_rec/python/core/sampler.py b/easy_rec/python/core/sampler.py index dcdcd44f0..59fb82655 100644 --- a/easy_rec/python/core/sampler.py +++ b/easy_rec/python/core/sampler.py @@ -1,18 +1,16 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function import json import logging import math -import os -import sys -import threading - import numpy as np +import os import six +import sys import tensorflow as tf +import threading from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils import ds_util @@ -46,7 +44,7 @@ def string_attrs(self, string_attrs): # NOQA Values.string_attrs = string_attrs except Exception: logging.info( - 'GraphLearn is not installed. You can install it by "pip install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-0.7-cp27-cp27mu-linux_x86_64.whl"' # noqa: E501 + 'GraphLearn is not installed. You can install it by "pip install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-0.7-cp27-cp27mu-linux_x86_64.whl"' # noqa: E501 ) if tf.__version__ >= '2.0': @@ -55,12 +53,12 @@ def string_attrs(self, string_attrs): # NOQA def _get_gl_type(field_type): type_map = { - DatasetConfig.INT32: 'int', - DatasetConfig.INT64: 'int', - DatasetConfig.STRING: 'string', - DatasetConfig.BOOL: 'int', - DatasetConfig.FLOAT: 'float', - DatasetConfig.DOUBLE: 'float' + DatasetConfig.INT32: 'int', + DatasetConfig.INT64: 'int', + DatasetConfig.STRING: 'string', + DatasetConfig.BOOL: 'int', + DatasetConfig.FLOAT: 'float', + DatasetConfig.DOUBLE: 'float' } assert field_type in type_map, 'invalid type: %s' % field_type return type_map[field_type] @@ -68,12 +66,12 @@ def _get_gl_type(field_type): def _get_np_type(field_type): type_map = { - DatasetConfig.INT32: np.int32, - DatasetConfig.INT64: np.int64, - DatasetConfig.STRING: str, - DatasetConfig.BOOL: bool, - DatasetConfig.FLOAT: np.float32, - DatasetConfig.DOUBLE: np.double + DatasetConfig.INT32: np.int32, + DatasetConfig.INT64: np.int64, + DatasetConfig.STRING: str, + DatasetConfig.BOOL: bool, + DatasetConfig.FLOAT: np.float32, + DatasetConfig.DOUBLE: np.double } assert field_type in type_map, 'invalid type: %s' % field_type return type_map[field_type] @@ -92,8 +90,9 @@ def __init__(self, fields, num_sample, num_eval_sample=None): self._is_on_ds = ds_util.is_on_ds() def set_eval_num_sample(self): - print('set_eval_num_sample: %d %d' % - (self._num_sample, self._num_eval_sample)) + print( + 'set_eval_num_sample: %d %d' % (self._num_sample, self._num_eval_sample) + ) self._num_sample = self._num_eval_sample def _init_graph(self): @@ -109,12 +108,12 @@ def _init_graph(self): if self._is_on_ds: gl.set_tracker_mode(0) server_hosts = [ - host.split(':')[0] + ':888' + str(i) - for i, host in enumerate(tf_config['cluster']['ps']) + host.split(':')[0] + ':888' + str(i) + for i, host in enumerate(tf_config['cluster']['ps']) ] cluster = { - 'server': ','.join(server_hosts), - 'client_count': task_count + 'server': ','.join(server_hosts), + 'client_count': task_count } else: ps_count = len(tf_config['cluster']['ps']) @@ -123,22 +122,25 @@ def _init_graph(self): self._g.init(cluster=cluster, job_name='client', task_index=0) elif tf_config['task']['type'] == 'worker': self._g.init( - cluster=cluster, - job_name='client', - task_index=tf_config['task']['index'] + 2) + cluster=cluster, + job_name='client', + task_index=tf_config['task']['index'] + 2 + ) # TODO(hongsheng.jhs): check cluster has evaluator or not? elif tf_config['task']['type'] == 'evaluator': self._g.init( - cluster=cluster, - job_name='client', - task_index=tf_config['task']['index'] + 1) + cluster=cluster, + job_name='client', + task_index=tf_config['task']['index'] + 1 + ) if self._num_eval_sample is not None and self._num_eval_sample > 0: self._num_sample = self._num_eval_sample elif tf_config['task']['type'] == 'ps': self._g.init( - cluster=cluster, - job_name='server', - task_index=tf_config['task']['index']) + cluster=cluster, + job_name='server', + task_index=tf_config['task']['index'] + ) else: # worker mode task_count = len(tf_config['cluster']['worker']) + 1 @@ -147,31 +149,33 @@ def _init_graph(self): self._g.init(task_index=0, task_count=task_count) elif tf_config['task']['type'] == 'worker': self._g.init( - task_index=tf_config['task']['index'] + 1, - task_count=task_count) + task_index=tf_config['task']['index'] + 1, task_count=task_count + ) else: gl.set_tracker_mode(0) if tf_config['cluster'].get('chief', ''): - chief_host = tf_config['cluster']['chief'][0].split( - ':')[0] + ':8880' + chief_host = tf_config['cluster']['chief'][0].split(':' + )[0] + ':8880' else: - chief_host = tf_config['cluster']['master'][0].split( - ':')[0] + ':8880' + chief_host = tf_config['cluster']['master'][0].split(':' + )[0] + ':8880' worker_hosts = chief_host + [ - host.split(':')[0] + ':888' + str(i) - for i, host in enumerate(tf_config['cluster']['worker']) + host.split(':')[0] + ':888' + str(i) + for i, host in enumerate(tf_config['cluster']['worker']) ] if tf_config['task']['type'] in ['chief', 'master']: self._g.init( - task_index=0, - task_count=task_count, - hosts=','.join(worker_hosts)) + task_index=0, + task_count=task_count, + hosts=','.join(worker_hosts) + ) elif tf_config['task']['type'] == 'worker': self._g.init( - task_index=tf_config['task']['index'] + 1, - task_count=task_count, - hosts=worker_hosts) + task_index=tf_config['task']['index'] + 1, + task_count=task_count, + hosts=worker_hosts + ) # TODO(hongsheng.jhs): check cluster has evaluator or not? else: @@ -204,15 +208,18 @@ def __del__(self): def _parse_nodes(self, nodes): if self._log_first_n > 0: - logging.info('num_example=%d num_eval_example=%d node_num=%d' % - (self._num_sample, self._num_eval_sample, len(nodes.ids))) + logging.info( + 'num_example=%d num_eval_example=%d node_num=%d' % + (self._num_sample, self._num_eval_sample, len(nodes.ids)) + ) self._log_first_n -= 1 features = [] int_idx = 0 float_idx = 0 string_idx = 0 - for attr_gl_type, attr_np_type in zip(self._attr_gl_types, - self._attr_np_types): + for attr_gl_type, attr_np_type in zip( + self._attr_gl_types, self._attr_np_types + ): if attr_gl_type == 'int': feature = nodes.int_attrs[:, :, int_idx] int_idx += 1 @@ -238,8 +245,9 @@ def _parse_sparse_nodes(self, nodes): int_idx = 0 float_idx = 0 string_idx = 0 - for attr_gl_type, attr_np_type in zip(self._attr_gl_types, - self._attr_np_types): + for attr_gl_type, attr_np_type in zip( + self._attr_gl_types, self._attr_np_types + ): if attr_gl_type == 'int': feature = nodes.int_attrs[:, int_idx] int_idx += 1 @@ -272,27 +280,32 @@ class NegativeSampler(BaseSampler): num_eval_sample: number of negative samples for evaluator. """ - def __init__(self, - data_path, - fields, - num_sample, - batch_size, - attr_delimiter=':', - num_eval_sample=None): + def __init__( + self, + data_path, + fields, + num_sample, + batch_size, + attr_delimiter=':', + num_eval_sample=None + ): super(NegativeSampler, self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size self._g = gl.Graph().node( - tf.compat.as_str(data_path), - node_type='item', - decoder=gl.Decoder( - attr_types=self._attr_gl_types, - weighted=True, - attr_delimiter=attr_delimiter)) + tf.compat.as_str(data_path), + node_type='item', + decoder=gl.Decoder( + attr_types=self._attr_gl_types, + weighted=True, + attr_delimiter=attr_delimiter + ) + ) self._init_graph() expand_factor = int(math.ceil(self._num_sample / batch_size)) self._sampler = self._g.negative_sampler( - 'item', expand_factor, strategy='node_weight') + 'item', expand_factor, strategy='node_weight' + ) def _get_impl(self, ids): ids = np.array(ids, dtype=np.int64) @@ -332,15 +345,17 @@ class NegativeSamplerInMemory(BaseSampler): num_eval_sample: number of negative samples for evaluator. """ - def __init__(self, - data_path, - fields, - num_sample, - batch_size, - attr_delimiter=':', - num_eval_sample=None): - super(NegativeSamplerInMemory, self).__init__(fields, num_sample, - num_eval_sample) + def __init__( + self, + data_path, + fields, + num_sample, + batch_size, + attr_delimiter=':', + num_eval_sample=None + ): + super(NegativeSamplerInMemory, + self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size self._item_ids = [] @@ -361,7 +376,8 @@ def __init__(self, self._cols[col_id] = np.array(self._cols[col_id], dtype=np_type) else: self._cols[col_id] = np.asarray( - self._cols[col_id], order='C', dtype=object) + self._cols[col_id], order='C', dtype=object + ) def _load_table(self, data_path, attr_delimiter): import common_io @@ -377,8 +393,10 @@ def _load_table(self, data_path, attr_delimiter): if schema[tid][0].startswith('id'): item_id_col = tid break - print('NegativeSamplerInMemory: feature_id_col = %d, item_id_col = %d' % - (fea_id_col, item_id_col)) + print( + 'NegativeSamplerInMemory: feature_id_col = %d, item_id_col = %d' % + (fea_id_col, item_id_col) + ) while True: try: row_arr = reader.read(num_records=1024, allow_smaller_final_batch=True) @@ -387,8 +405,10 @@ def _load_table(self, data_path, attr_delimiter): self._item_ids.append(int(row[item_id_col])) col_vals = row[fea_id_col].split(attr_delimiter) assert len(col_vals) == len( - self._cols), 'invalid row[%d %d]: %s %s' % (len( - col_vals), len(self._cols), row[item_id_col], row[fea_id_col]) + self._cols + ), 'invalid row[%d %d]: %s %s' % ( + len(col_vals), len(self._cols), row[item_id_col], row[fea_id_col] + ) for col_id in range(len(col_vals)): self._cols[col_id].append(col_vals[col_id]) except common_io.exception.OutOfRangeException: @@ -410,15 +430,18 @@ def _load_data(self, data_path, attr_delimiter): item_id_col = tid if schema[tid][0].startswith('feature'): fea_id_col = tid - print('feature_id_col = %d, item_id_col = %d' % - (fea_id_col, item_id_col)) + print( + 'feature_id_col = %d, item_id_col = %d' % + (fea_id_col, item_id_col) + ) else: self._item_ids.append(int(cols[item_id_col])) fea_vals = cols[fea_id_col].split(attr_delimiter) - assert len(fea_vals) == len( - self._cols), 'invalid row[%d][%d %d]:%s %s' % ( - line_id, len(fea_vals), len( - self._cols), cols[item_id_col], cols[fea_id_col]) + assert len(fea_vals) == len(self._cols + ), 'invalid row[%d][%d %d]:%s %s' % ( + line_id, len(fea_vals), len(self._cols), + cols[item_id_col], cols[fea_id_col] + ) for col_id in range(len(fea_vals)): self._cols[col_id].append(fea_vals[col_id]) @@ -429,9 +452,10 @@ def _get_impl(self, ids): assert self._num_sample > 0, 'invalid num_sample: %d' % self._num_sample indices = np.random.choice( - len(self._item_ids), - size=self._num_sample + self._batch_size, - replace=False) + len(self._item_ids), + size=self._num_sample + self._batch_size, + replace=False + ) sel_ids = [] for tid in indices: @@ -450,7 +474,8 @@ def _get_impl(self, ids): features.append(sel_feas) else: features.append( - np.asarray([tmp_col[x] for x in sel_ids], order='C', dtype=object)) + np.asarray([tmp_col[x] for x in sel_ids], order='C', dtype=object) + ) return features def get(self, ids): @@ -488,16 +513,19 @@ class NegativeSamplerV2(BaseSampler): num_eval_sample: number of negative samples for evaluator. """ - def __init__(self, - user_data_path, - item_data_path, - edge_data_path, - fields, - num_sample, - batch_size, - attr_delimiter=':', - num_eval_sample=None): - super(NegativeSamplerV2, self).__init__(fields, num_sample, num_eval_sample) + def __init__( + self, + user_data_path, + item_data_path, + edge_data_path, + fields, + num_sample, + batch_size, + attr_delimiter=':', + num_eval_sample=None + ): + super(NegativeSamplerV2, + self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size self._g = gl.Graph() \ .node(tf.compat.as_str(user_data_path), @@ -516,7 +544,8 @@ def __init__(self, expand_factor = int(math.ceil(self._num_sample / batch_size)) self._sampler = self._g.negative_sampler( - 'edge', expand_factor, strategy='random', conditional=True) + 'edge', expand_factor, strategy='random', conditional=True + ) def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) @@ -537,8 +566,9 @@ def get(self, src_ids, dst_ids): Returns: Negative sampled feature dict. """ - sampled_values = tf.py_func(self._get_impl, [src_ids, dst_ids], - self._attr_tf_types) + sampled_values = tf.py_func( + self._get_impl, [src_ids, dst_ids], self._attr_tf_types + ) result_dict = {} for k, t, v in zip(self._attr_names, self._attr_tf_types, sampled_values): v.set_shape([self._num_sample]) @@ -564,18 +594,20 @@ class HardNegativeSampler(BaseSampler): num_eval_sample: number of negative samples for evaluator. """ - def __init__(self, - user_data_path, - item_data_path, - hard_neg_edge_data_path, - fields, - num_sample, - num_hard_sample, - batch_size, - attr_delimiter=':', - num_eval_sample=None): - super(HardNegativeSampler, self).__init__(fields, num_sample, - num_eval_sample) + def __init__( + self, + user_data_path, + item_data_path, + hard_neg_edge_data_path, + fields, + num_sample, + num_hard_sample, + batch_size, + attr_delimiter=':', + num_eval_sample=None + ): + super(HardNegativeSampler, + self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size self._g = gl.Graph() \ .node(tf.compat.as_str(user_data_path), @@ -594,10 +626,11 @@ def __init__(self, expand_factor = int(math.ceil(self._num_sample / batch_size)) self._neg_sampler = self._g.negative_sampler( - 'item', expand_factor, strategy='node_weight') - self._hard_neg_sampler = self._g.neighbor_sampler(['hard_neg_edge'], - num_hard_sample, - strategy='full') + 'item', expand_factor, strategy='node_weight' + ) + self._hard_neg_sampler = self._g.neighbor_sampler( + ['hard_neg_edge'], num_hard_sample, strategy='full' + ) def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) @@ -606,12 +639,16 @@ def _get_impl(self, src_ids, dst_ids): nodes = self._neg_sampler.get(dst_ids) neg_features = self._parse_nodes(nodes) sparse_nodes = self._hard_neg_sampler.get(src_ids).layer_nodes(1) - hard_neg_features, hard_neg_indices = self._parse_sparse_nodes(sparse_nodes) + hard_neg_features, hard_neg_indices = self._parse_sparse_nodes( + sparse_nodes + ) results = [] for i, v in enumerate(hard_neg_features): if type(v) == list: - results.append(np.asarray(neg_features[i] + v, order='C', dtype=object)) + results.append( + np.asarray(neg_features[i] + v, order='C', dtype=object) + ) else: results.append(np.concatenate([neg_features[i], v], axis=0)) results.append(hard_neg_indices) @@ -628,10 +665,13 @@ def get(self, src_ids, dst_ids): Sampled feature dict. The first batch_size is negative samples, remainder is hard negative samples """ output_types = self._attr_tf_types + [tf.int64] - output_values = tf.py_func(self._get_impl, [src_ids, dst_ids], output_types) + output_values = tf.py_func( + self._get_impl, [src_ids, dst_ids], output_types + ) result_dict = {} - for k, t, v in zip(self._attr_names, self._attr_tf_types, - output_values[:-1]): + for k, t, v in zip( + self._attr_names, self._attr_tf_types, output_values[:-1] + ): v.set_shape([None]) result_dict[k] = v @@ -660,19 +700,21 @@ class HardNegativeSamplerV2(BaseSampler): num_eval_sample: number of negative samples for evaluator. """ - def __init__(self, - user_data_path, - item_data_path, - edge_data_path, - hard_neg_edge_data_path, - fields, - num_sample, - num_hard_sample, - batch_size, - attr_delimiter=':', - num_eval_sample=None): - super(HardNegativeSamplerV2, self).__init__(fields, num_sample, - num_eval_sample) + def __init__( + self, + user_data_path, + item_data_path, + edge_data_path, + hard_neg_edge_data_path, + fields, + num_sample, + num_hard_sample, + batch_size, + attr_delimiter=':', + num_eval_sample=None + ): + super(HardNegativeSamplerV2, + self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size self._g = gl.Graph() \ .node(tf.compat.as_str(user_data_path), @@ -694,26 +736,32 @@ def __init__(self, expand_factor = int(math.ceil(self._num_sample / batch_size)) self._neg_sampler = self._g.negative_sampler( - 'edge', expand_factor, strategy='random', conditional=True) - self._hard_neg_sampler = self._g.neighbor_sampler(['hard_neg_edge'], - num_hard_sample, - strategy='full') + 'edge', expand_factor, strategy='random', conditional=True + ) + self._hard_neg_sampler = self._g.neighbor_sampler( + ['hard_neg_edge'], num_hard_sample, strategy='full' + ) def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) - src_ids_padded = np.pad(src_ids, (0, self._batch_size - len(src_ids)), - 'edge') + src_ids_padded = np.pad( + src_ids, (0, self._batch_size - len(src_ids)), 'edge' + ) dst_ids = np.array(dst_ids, dtype=np.int64) dst_ids = np.pad(dst_ids, (0, self._batch_size - len(dst_ids)), 'edge') nodes = self._neg_sampler.get(src_ids_padded, dst_ids) neg_features = self._parse_nodes(nodes) sparse_nodes = self._hard_neg_sampler.get(src_ids).layer_nodes(1) - hard_neg_features, hard_neg_indices = self._parse_sparse_nodes(sparse_nodes) + hard_neg_features, hard_neg_indices = self._parse_sparse_nodes( + sparse_nodes + ) results = [] for i, v in enumerate(hard_neg_features): if type(v) == list: - results.append(np.asarray(neg_features[i] + v, order='C', dtype=object)) + results.append( + np.asarray(neg_features[i] + v, order='C', dtype=object) + ) else: results.append(np.concatenate([neg_features[i], v], axis=0)) results.append(hard_neg_indices) @@ -730,10 +778,13 @@ def get(self, src_ids, dst_ids): Sampled feature dict. The first batch_size is negative samples, remainder is hard negative samples """ output_types = self._attr_tf_types + [tf.int64] - output_values = tf.py_func(self._get_impl, [src_ids, dst_ids], output_types) + output_values = tf.py_func( + self._get_impl, [src_ids, dst_ids], output_types + ) result_dict = {} - for k, t, v in zip(self._attr_names, self._attr_tf_types, - output_values[:-1]): + for k, t, v in zip( + self._attr_names, self._attr_tf_types, output_values[:-1] + ): v.set_shape([None]) result_dict[k] = v @@ -760,85 +811,100 @@ def build(data_config): input_path = process_multi_file_input_path(sampler_config.input_path) return NegativeSampler.instance( - data_path=input_path, - fields=attr_fields, - num_sample=sampler_config.num_sample, - batch_size=data_config.batch_size, - attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample) + data_path=input_path, + fields=attr_fields, + num_sample=sampler_config.num_sample, + batch_size=data_config.batch_size, + attr_delimiter=sampler_config.attr_delimiter, + num_eval_sample=sampler_config.num_eval_sample + ) elif sampler_type == 'negative_sampler_in_memory': input_fields = {f.input_name: f for f in data_config.input_fields} attr_fields = [input_fields[name] for name in sampler_config.attr_fields] input_path = process_multi_file_input_path(sampler_config.input_path) return NegativeSamplerInMemory.instance( - data_path=input_path, - fields=attr_fields, - num_sample=sampler_config.num_sample, - batch_size=data_config.batch_size, - attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample) + data_path=input_path, + fields=attr_fields, + num_sample=sampler_config.num_sample, + batch_size=data_config.batch_size, + attr_delimiter=sampler_config.attr_delimiter, + num_eval_sample=sampler_config.num_eval_sample + ) elif sampler_type == 'negative_sampler_v2': input_fields = {f.input_name: f for f in data_config.input_fields} attr_fields = [input_fields[name] for name in sampler_config.attr_fields] user_input_path = process_multi_file_input_path( - sampler_config.user_input_path) + sampler_config.user_input_path + ) item_input_path = process_multi_file_input_path( - sampler_config.item_input_path) + sampler_config.item_input_path + ) pos_edge_input_path = process_multi_file_input_path( - sampler_config.pos_edge_input_path) + sampler_config.pos_edge_input_path + ) return NegativeSamplerV2.instance( - user_data_path=user_input_path, - item_data_path=item_input_path, - edge_data_path=pos_edge_input_path, - fields=attr_fields, - num_sample=sampler_config.num_sample, - batch_size=data_config.batch_size, - attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample) + user_data_path=user_input_path, + item_data_path=item_input_path, + edge_data_path=pos_edge_input_path, + fields=attr_fields, + num_sample=sampler_config.num_sample, + batch_size=data_config.batch_size, + attr_delimiter=sampler_config.attr_delimiter, + num_eval_sample=sampler_config.num_eval_sample + ) elif sampler_type == 'hard_negative_sampler': input_fields = {f.input_name: f for f in data_config.input_fields} attr_fields = [input_fields[name] for name in sampler_config.attr_fields] user_input_path = process_multi_file_input_path( - sampler_config.user_input_path) + sampler_config.user_input_path + ) item_input_path = process_multi_file_input_path( - sampler_config.item_input_path) + sampler_config.item_input_path + ) hard_neg_edge_input_path = process_multi_file_input_path( - sampler_config.hard_neg_edge_input_path) + sampler_config.hard_neg_edge_input_path + ) return HardNegativeSampler.instance( - user_data_path=user_input_path, - item_data_path=item_input_path, - hard_neg_edge_data_path=hard_neg_edge_input_path, - fields=attr_fields, - num_sample=sampler_config.num_sample, - num_hard_sample=sampler_config.num_hard_sample, - batch_size=data_config.batch_size, - attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample) + user_data_path=user_input_path, + item_data_path=item_input_path, + hard_neg_edge_data_path=hard_neg_edge_input_path, + fields=attr_fields, + num_sample=sampler_config.num_sample, + num_hard_sample=sampler_config.num_hard_sample, + batch_size=data_config.batch_size, + attr_delimiter=sampler_config.attr_delimiter, + num_eval_sample=sampler_config.num_eval_sample + ) elif sampler_type == 'hard_negative_sampler_v2': input_fields = {f.input_name: f for f in data_config.input_fields} attr_fields = [input_fields[name] for name in sampler_config.attr_fields] user_input_path = process_multi_file_input_path( - sampler_config.user_input_path) + sampler_config.user_input_path + ) item_input_path = process_multi_file_input_path( - sampler_config.item_input_path) + sampler_config.item_input_path + ) pos_edge_input_path = process_multi_file_input_path( - sampler_config.pos_edge_input_path) + sampler_config.pos_edge_input_path + ) hard_neg_edge_input_path = process_multi_file_input_path( - sampler_config.hard_neg_edge_input_path) + sampler_config.hard_neg_edge_input_path + ) return HardNegativeSamplerV2.instance( - user_data_path=user_input_path, - item_data_path=item_input_path, - edge_data_path=pos_edge_input_path, - hard_neg_edge_data_path=hard_neg_edge_input_path, - fields=attr_fields, - num_sample=sampler_config.num_sample, - num_hard_sample=sampler_config.num_hard_sample, - batch_size=data_config.batch_size, - attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample) + user_data_path=user_input_path, + item_data_path=item_input_path, + edge_data_path=pos_edge_input_path, + hard_neg_edge_data_path=hard_neg_edge_input_path, + fields=attr_fields, + num_sample=sampler_config.num_sample, + num_hard_sample=sampler_config.num_hard_sample, + batch_size=data_config.batch_size, + attr_delimiter=sampler_config.attr_delimiter, + num_eval_sample=sampler_config.num_eval_sample + ) else: raise ValueError('Unknown sampler %s' % sampler_type) diff --git a/easy_rec/python/eval.py b/easy_rec/python/eval.py index d41c3d7ae..f0b80abc5 100644 --- a/easy_rec/python/eval.py +++ b/easy_rec/python/eval.py @@ -3,42 +3,45 @@ import logging import os - import six import tensorflow as tf from tensorflow.python.lib.io import file_io -from easy_rec.python.main import distribute_evaluate -from easy_rec.python.main import evaluate +from easy_rec.python.main import distribute_evaluate, evaluate from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.utils import config_util -from easy_rec.python.utils import ds_util -from easy_rec.python.utils import estimator_utils - +from easy_rec.python.utils import config_util, ds_util, estimator_utils from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_distribute_eval_worker_num_on_ds # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) -tf.app.flags.DEFINE_string('pipeline_config_path', None, - 'Path to pipeline config ' - 'file.') tf.app.flags.DEFINE_string( - 'checkpoint_path', None, 'checkpoint to be evaled ' - ' if not specified, use the latest checkpoint in ' - 'train_config.model_dir') + 'pipeline_config_path', None, 'Path to pipeline config ' + 'file.' +) +tf.app.flags.DEFINE_string( + 'checkpoint_path', None, 'checkpoint to be evaled ' + ' if not specified, use the latest checkpoint in ' + 'train_config.model_dir' +) tf.app.flags.DEFINE_multi_string( - 'eval_input_path', None, 'eval data path, if specified will ' - 'override pipeline_config.eval_input_path') + 'eval_input_path', None, 'eval data path, if specified will ' + 'override pipeline_config.eval_input_path' +) tf.app.flags.DEFINE_string('model_dir', None, help='will update the model_dir') tf.app.flags.DEFINE_string('odps_config', None, help='odps config path') -tf.app.flags.DEFINE_string('eval_result_path', 'eval_result.txt', - 'eval result metric file') -tf.app.flags.DEFINE_bool('distribute_eval', False, - 'use distribute parameter server for train and eval.') +tf.app.flags.DEFINE_string( + 'eval_result_path', 'eval_result.txt', 'eval result metric file' +) +tf.app.flags.DEFINE_bool( + 'distribute_eval', False, + 'use distribute parameter server for train and eval.' +) tf.app.flags.DEFINE_bool('is_on_ds', False, help='is on ds') FLAGS = tf.app.flags.FLAGS @@ -63,30 +66,34 @@ def main(argv): pipeline_config_path = FLAGS.pipeline_config_path pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path) + pipeline_config_path + ) if FLAGS.model_dir: pipeline_config.model_dir = FLAGS.model_dir if pipeline_config.train_config.train_distribute in [ - DistributionStrategy.HorovodStrategy, + DistributionStrategy.HorovodStrategy, ]: estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ - DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy ]: estimator_utils.init_hvd() estimator_utils.init_sok() if FLAGS.distribute_eval: os.environ['distribute_eval'] = 'True' - eval_result = distribute_evaluate(pipeline_config, FLAGS.checkpoint_path, - FLAGS.eval_input_path, - FLAGS.eval_result_path) + eval_result = distribute_evaluate( + pipeline_config, FLAGS.checkpoint_path, FLAGS.eval_input_path, + FLAGS.eval_result_path + ) else: os.environ['distribute_eval'] = 'False' - eval_result = evaluate(pipeline_config, FLAGS.checkpoint_path, - FLAGS.eval_input_path, FLAGS.eval_result_path) + eval_result = evaluate( + pipeline_config, FLAGS.checkpoint_path, FLAGS.eval_input_path, + FLAGS.eval_result_path + ) if eval_result is not None: # when distribute evaluate, only master has eval_result. for key in sorted(eval_result): diff --git a/easy_rec/python/export.py b/easy_rec/python/export.py index c1a8ce670..0768cd4bd 100644 --- a/easy_rec/python/export.py +++ b/easy_rec/python/export.py @@ -2,14 +2,12 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os - import tensorflow as tf from tensorflow.python.lib.io import file_io from easy_rec.python.main import export from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.utils import config_util -from easy_rec.python.utils import estimator_utils +from easy_rec.python.utils import config_util, estimator_utils if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile @@ -20,42 +18,56 @@ tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) -tf.app.flags.DEFINE_string('pipeline_config_path', None, - 'Path to pipeline config ' - 'file.') + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) +tf.app.flags.DEFINE_string( + 'pipeline_config_path', None, 'Path to pipeline config ' + 'file.' +) tf.app.flags.DEFINE_string('checkpoint_path', '', 'checkpoint to be exported') -tf.app.flags.DEFINE_string('export_dir', None, - 'directory where model should be exported to') +tf.app.flags.DEFINE_string( + 'export_dir', None, 'directory where model should be exported to' +) tf.app.flags.DEFINE_string('redis_url', None, 'export to redis url, host:port') tf.app.flags.DEFINE_string('redis_passwd', None, 'export to redis passwd') tf.app.flags.DEFINE_integer('redis_threads', 0, 'export to redis threads') -tf.app.flags.DEFINE_integer('redis_batch_size', 256, - 'export to redis batch_size') -tf.app.flags.DEFINE_integer('redis_timeout', 600, - 'export to redis time_out in seconds') -tf.app.flags.DEFINE_integer('redis_expire', 24, - 'export to redis expire time in hour') -tf.app.flags.DEFINE_string('redis_embedding_version', '', - 'redis embedding version') -tf.app.flags.DEFINE_integer('redis_write_kv', 1, - 'whether to write embedding to redis') +tf.app.flags.DEFINE_integer( + 'redis_batch_size', 256, 'export to redis batch_size' +) +tf.app.flags.DEFINE_integer( + 'redis_timeout', 600, 'export to redis time_out in seconds' +) +tf.app.flags.DEFINE_integer( + 'redis_expire', 24, 'export to redis expire time in hour' +) +tf.app.flags.DEFINE_string( + 'redis_embedding_version', '', 'redis embedding version' +) +tf.app.flags.DEFINE_integer( + 'redis_write_kv', 1, 'whether to write embedding to redis' +) tf.app.flags.DEFINE_string( - 'oss_path', None, 'write embed objects to oss folder, oss://bucket/folder') + 'oss_path', None, 'write embed objects to oss folder, oss://bucket/folder' +) tf.app.flags.DEFINE_string('oss_endpoint', None, 'oss endpoint') tf.app.flags.DEFINE_string('oss_ak', None, 'oss ak') tf.app.flags.DEFINE_string('oss_sk', None, 'oss sk') -tf.app.flags.DEFINE_integer('oss_threads', 10, - '# threads access oss at the same time') -tf.app.flags.DEFINE_integer('oss_timeout', 10, - 'connect to oss, time_out in seconds') +tf.app.flags.DEFINE_integer( + 'oss_threads', 10, '# threads access oss at the same time' +) +tf.app.flags.DEFINE_integer( + 'oss_timeout', 10, 'connect to oss, time_out in seconds' +) tf.app.flags.DEFINE_integer('oss_expire', 24, 'oss expire time in hours') -tf.app.flags.DEFINE_integer('oss_write_kv', 1, - 'whether to write embedding to oss') -tf.app.flags.DEFINE_string('oss_embedding_version', '', 'oss embedding version') +tf.app.flags.DEFINE_integer( + 'oss_write_kv', 1, 'whether to write embedding to oss' +) +tf.app.flags.DEFINE_string( + 'oss_embedding_version', '', 'oss embedding version' +) tf.app.flags.DEFINE_string('asset_files', '', 'more files to add to asset') tf.app.flags.DEFINE_bool('verbose', False, 'print more debug information') @@ -64,8 +76,9 @@ tf.app.flags.mark_flag_as_required('export_dir') tf.app.flags.DEFINE_bool('clear_export', False, 'remove export_dir if exists') -tf.app.flags.DEFINE_string('export_done_file', '', - 'a flag file to signal that export model is done') +tf.app.flags.DEFINE_string( + 'export_done_file', '', 'a flag file to signal that export model is done' +) FLAGS = tf.app.flags.FLAGS @@ -118,14 +131,15 @@ def main(argv): extra_params['oss_embedding_version'] = FLAGS.oss_embedding_version pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path) + pipeline_config_path + ) if pipeline_config.train_config.train_distribute in [ - DistributionStrategy.HorovodStrategy, + DistributionStrategy.HorovodStrategy, ]: estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ - DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy ]: estimator_utils.init_hvd() estimator_utils.init_sok() @@ -135,9 +149,10 @@ def main(argv): if gfile.IsDirectory(FLAGS.export_dir): gfile.DeleteRecursively(FLAGS.export_dir) - export_out_dir = export(FLAGS.export_dir, pipeline_config_path, - FLAGS.checkpoint_path, FLAGS.asset_files, - FLAGS.verbose, **extra_params) + export_out_dir = export( + FLAGS.export_dir, pipeline_config_path, FLAGS.checkpoint_path, + FLAGS.asset_files, FLAGS.verbose, **extra_params + ) if FLAGS.export_done_file: flag_file = os.path.join(export_out_dir, FLAGS.export_done_file) diff --git a/easy_rec/python/feature_column/feature_column.py b/easy_rec/python/feature_column/feature_column.py index 8701b55fc..0bcc595c8 100644 --- a/easy_rec/python/feature_column/feature_column.py +++ b/easy_rec/python/feature_column/feature_column.py @@ -3,19 +3,16 @@ import collections import logging import sys - import tensorflow as tf from tensorflow.python.ops import partitioned_variables from tensorflow.python.platform import gfile from easy_rec.python.builders import hyperparams_builder +from easy_rec.python.compat.feature_column import feature_column_v2 as feature_column # NOQA from easy_rec.python.compat.feature_column import sequence_feature_column -from easy_rec.python.protos.feature_config_pb2 import FeatureConfig -from easy_rec.python.protos.feature_config_pb2 import WideOrDeep +from easy_rec.python.protos.feature_config_pb2 import FeatureConfig, WideOrDeep from easy_rec.python.utils.proto_util import copy_obj -from easy_rec.python.compat.feature_column import feature_column_v2 as feature_column # NOQA - MAX_HASH_BUCKET_SIZE = 9223372036854775807 @@ -33,19 +30,24 @@ def __init__(self, embedding_name, index, sequence_combiner=None): self.sequence_combiner = sequence_combiner -EVParams = collections.namedtuple('EVParams', [ - 'filter_freq', 'steps_to_live', 'use_cache', 'init_capacity', 'max_capacity' -]) +EVParams = collections.namedtuple( + 'EVParams', [ + 'filter_freq', 'steps_to_live', 'use_cache', 'init_capacity', + 'max_capacity' + ] +) class FeatureColumnParser(object): """Parse and generate feature columns.""" - def __init__(self, - feature_configs, - wide_deep_dict={}, - wide_output_dim=-1, - ev_params=None): + def __init__( + self, + feature_configs, + wide_deep_dict={}, + wide_output_dim=-1, + ev_params=None + ): """Initializes a `FeatureColumnParser`. Args: @@ -97,7 +99,7 @@ def _cmp_embed_config(a, b): # remove not shared embedding names not_shared = [ - x for x in self._share_embed_names if self._share_embed_names[x] == 1 + x for x in self._share_embed_names if self._share_embed_names[x] == 1 ] for embed_name in not_shared: del self._share_embed_names[embed_name] @@ -105,14 +107,19 @@ def _cmp_embed_config(a, b): logging.info('shared embeddings[num=%d]' % len(self._share_embed_names)) for embed_name in self._share_embed_names: - logging.info('\t%s: share_num[%d], share_info[%s]' % - (embed_name, self._share_embed_names[embed_name], - self._share_embed_infos[embed_name])) + logging.info( + '\t%s: share_num[%d], share_info[%s]' % ( + embed_name, self._share_embed_names[embed_name], + self._share_embed_infos[embed_name] + ) + ) self._deep_share_embed_columns = { - embed_name: [] for embed_name in self._share_embed_names + embed_name: [] + for embed_name in self._share_embed_names } self._wide_share_embed_columns = { - embed_name: [] for embed_name in self._share_embed_names + embed_name: [] + for embed_name in self._share_embed_names } self._feature_vocab_size = {} @@ -142,29 +149,35 @@ def _cmp_embed_config(a, b): initializer = None if self._share_embed_infos[embed_name].HasField('initializer'): initializer = hyperparams_builder.build_initializer( - self._share_embed_infos[embed_name].initializer) + self._share_embed_infos[embed_name].initializer + ) - partitioner = self._build_partitioner(self._share_embed_infos[embed_name]) + partitioner = self._build_partitioner( + self._share_embed_infos[embed_name] + ) if self._share_embed_infos[embed_name].HasField('ev_params'): ev_params = self._build_ev_params( - self._share_embed_infos[embed_name].ev_params) + self._share_embed_infos[embed_name].ev_params + ) else: ev_params = self._global_ev_params # for handling share embedding columns if len(self._deep_share_embed_columns[embed_name]) > 0: share_embed_fcs = feature_column.shared_embedding_columns( - self._deep_share_embed_columns[embed_name], - self._share_embed_infos[embed_name].embedding_dim, - initializer=initializer, - shared_embedding_collection_name=embed_name, - combiner=self._share_embed_infos[embed_name].combiner, - partitioner=partitioner, - ev_params=ev_params) + self._deep_share_embed_columns[embed_name], + self._share_embed_infos[embed_name].embedding_dim, + initializer=initializer, + shared_embedding_collection_name=embed_name, + combiner=self._share_embed_infos[embed_name].combiner, + partitioner=partitioner, + ev_params=ev_params + ) config = self._share_embed_infos[embed_name] max_seq_len = config.max_seq_len if config.HasField( - 'max_seq_len') else -1 + 'max_seq_len' + ) else -1 for fc in share_embed_fcs: fc.max_seq_length = max_seq_len self._deep_share_embed_columns[embed_name] = share_embed_fcs @@ -172,16 +185,18 @@ def _cmp_embed_config(a, b): # for handling wide share embedding columns if len(self._wide_share_embed_columns[embed_name]) > 0: share_embed_fcs = feature_column.shared_embedding_columns( - self._wide_share_embed_columns[embed_name], - self._wide_output_dim, - initializer=initializer, - shared_embedding_collection_name=embed_name + '_wide', - combiner='sum', - partitioner=partitioner, - ev_params=ev_params) + self._wide_share_embed_columns[embed_name], + self._wide_output_dim, + initializer=initializer, + shared_embedding_collection_name=embed_name + '_wide', + combiner='sum', + partitioner=partitioner, + ev_params=ev_params + ) config = self._share_embed_infos[embed_name] max_seq_len = config.max_seq_len if config.HasField( - 'max_seq_len') else -1 + 'max_seq_len' + ) else -1 for fc in share_embed_fcs: fc.max_seq_length = max_seq_len self._wide_share_embed_columns[embed_name] = share_embed_fcs @@ -195,7 +210,8 @@ def _cmp_embed_config(a, b): fc = self._wide_columns[fc_name] if isinstance(fc, SharedEmbedding): self._wide_columns[fc_name] = self._get_shared_embedding_column( - fc, deep=False) + fc, deep=False + ) for fc_name in self._sequence_columns: fc = self._sequence_columns[fc_name] @@ -222,7 +238,7 @@ def is_wide(self, config): if feature_name not in self._wide_deep_dict: raise FeatureKeyError(feature_name) return self._wide_deep_dict[feature_name] in [ - WideOrDeep.WIDE, WideOrDeep.WIDE_AND_DEEP + WideOrDeep.WIDE, WideOrDeep.WIDE_AND_DEEP ] def is_deep(self, config): @@ -234,7 +250,7 @@ def is_deep(self, config): if feature_name not in self._wide_deep_dict: raise FeatureKeyError(feature_name) return self._wide_deep_dict[feature_name] in [ - WideOrDeep.DEEP, WideOrDeep.WIDE_AND_DEEP + WideOrDeep.DEEP, WideOrDeep.WIDE_AND_DEEP ] def get_feature_vocab_size(self, feature): @@ -271,27 +287,31 @@ def parse_id_feature(self, config): hash_bucket_size = self._get_hash_bucket_size(config) if hash_bucket_size > 0: fc = feature_column.categorical_column_with_hash_bucket( - feature_name, - hash_bucket_size=hash_bucket_size, - feature_name=feature_name) + feature_name, + hash_bucket_size=hash_bucket_size, + feature_name=feature_name + ) elif config.vocab_list: fc = feature_column.categorical_column_with_vocabulary_list( - feature_name, - default_value=0, - vocabulary_list=config.vocab_list, - feature_name=feature_name) + feature_name, + default_value=0, + vocabulary_list=config.vocab_list, + feature_name=feature_name + ) elif config.vocab_file: fc = feature_column.categorical_column_with_vocabulary_file( - feature_name, - default_value=0, - vocabulary_file=config.vocab_file, - vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name) + feature_name, + default_value=0, + vocabulary_file=config.vocab_file, + vocabulary_size=self._get_vocab_size(config.vocab_file), + feature_name=feature_name + ) else: use_ev = self._global_ev_params or config.HasField('ev_params') num_buckets = sys.maxsize if use_ev else config.num_buckets fc = feature_column.categorical_column_with_identity( - feature_name, num_buckets, default_value=0, feature_name=feature_name) + feature_name, num_buckets, default_value=0, feature_name=feature_name + ) if self.is_wide(config): self._add_wide_embedding_column(fc, config) @@ -313,35 +333,41 @@ def parse_tag_feature(self, config): hash_bucket_size = self._get_hash_bucket_size(config) if hash_bucket_size > 0: tag_fc = feature_column.categorical_column_with_hash_bucket( - feature_name, - hash_bucket_size, - dtype=tf.string, - feature_name=feature_name) + feature_name, + hash_bucket_size, + dtype=tf.string, + feature_name=feature_name + ) elif config.vocab_list: tag_fc = feature_column.categorical_column_with_vocabulary_list( - feature_name, - default_value=0, - vocabulary_list=config.vocab_list, - feature_name=feature_name) + feature_name, + default_value=0, + vocabulary_list=config.vocab_list, + feature_name=feature_name + ) elif config.vocab_file: tag_fc = feature_column.categorical_column_with_vocabulary_file( - feature_name, - default_value=0, - vocabulary_file=config.vocab_file, - vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name) + feature_name, + default_value=0, + vocabulary_file=config.vocab_file, + vocabulary_size=self._get_vocab_size(config.vocab_file), + feature_name=feature_name + ) else: use_ev = self._global_ev_params or config.HasField('ev_params') num_buckets = sys.maxsize if use_ev else config.num_buckets tag_fc = feature_column.categorical_column_with_identity( - feature_name, num_buckets, default_value=0, feature_name=feature_name) + feature_name, num_buckets, default_value=0, feature_name=feature_name + ) if len(config.input_names) > 1: tag_fc = feature_column.weighted_categorical_column( - tag_fc, weight_feature_key=feature_name + '_w', dtype=tf.float32) + tag_fc, weight_feature_key=feature_name + '_w', dtype=tf.float32 + ) elif config.HasField('kv_separator'): tag_fc = feature_column.weighted_categorical_column( - tag_fc, weight_feature_key=feature_name + '_w', dtype=tf.float32) + tag_fc, weight_feature_key=feature_name + '_w', dtype=tf.float32 + ) if self.is_wide(config): self._add_wide_embedding_column(tag_fc, config) @@ -359,9 +385,10 @@ def parse_raw_feature(self, config): feature_name = config.feature_name if config.HasField('feature_name') \ else config.input_names[0] fc = feature_column.numeric_column( - key=feature_name, - shape=(config.raw_input_dim,), - feature_name=feature_name) + key=feature_name, + shape=(config.raw_input_dim, ), + feature_name=feature_name + ) bounds = None if config.boundaries: @@ -370,17 +397,20 @@ def parse_raw_feature(self, config): elif config.num_buckets > 1 and config.max_val > config.min_val: # the feature values are already normalized into [0, 1] bounds = [ - x / float(config.num_buckets) for x in range(0, config.num_buckets) + x / float(config.num_buckets) for x in range(0, config.num_buckets) ] - logging.info('discrete %s into %d buckets' % - (feature_name, config.num_buckets)) + logging.info( + 'discrete %s into %d buckets' % (feature_name, config.num_buckets) + ) if bounds: try: fc = feature_column.bucketized_column(fc, bounds) except Exception as e: - logging.error('bucketized_column [%s] with bounds %s error' % - (fc.name, str(bounds))) + logging.error( + 'bucketized_column [%s] with bounds %s error' % + (fc.name, str(bounds)) + ) raise e if self.is_wide(config): self._add_wide_embedding_column(fc, config) @@ -388,14 +418,16 @@ def parse_raw_feature(self, config): self._add_deep_embedding_column(fc, config) else: tmp_id_col = feature_column.categorical_column_with_identity( - feature_name + '_raw_proj_id', - config.raw_input_dim, - default_value=0, - feature_name=feature_name) + feature_name + '_raw_proj_id', + config.raw_input_dim, + default_value=0, + feature_name=feature_name + ) wgt_fc = feature_column.weighted_categorical_column( - tmp_id_col, - weight_feature_key=feature_name + '_raw_proj_val', - dtype=tf.float32) + tmp_id_col, + weight_feature_key=feature_name + '_raw_proj_val', + dtype=tf.float32 + ) if self.is_wide(config): self._add_wide_embedding_column(wgt_fc, config) if self.is_deep(config): @@ -415,7 +447,8 @@ def parse_expr_feature(self, config): feature_name = config.feature_name if config.HasField('feature_name') \ else config.input_names[0] fc = feature_column.numeric_column( - feature_name, shape=(1,), feature_name=feature_name) + feature_name, shape=(1, ), feature_name=feature_name + ) if self.is_wide(config): self._add_wide_embedding_column(fc, config) if self.is_deep(config): @@ -439,15 +472,17 @@ def parse_combo_feature(self, config): else: input_names.append(feature_name + '_' + str(input_id)) fc = feature_column.crossed_column( - input_names, - self._get_hash_bucket_size(config), - hash_key=None, - feature_name=feature_name) + input_names, + self._get_hash_bucket_size(config), + hash_key=None, + feature_name=feature_name + ) else: fc = feature_column.categorical_column_with_hash_bucket( - feature_name, - hash_bucket_size=self._get_hash_bucket_size(config), - feature_name=feature_name) + feature_name, + hash_bucket_size=self._get_hash_bucket_size(config), + feature_name=feature_name + ) if self.is_wide(config): self._add_wide_embedding_column(fc, config) @@ -465,10 +500,11 @@ def parse_lookup_feature(self, config): assert config.HasField('hash_bucket_size') hash_bucket_size = self._get_hash_bucket_size(config) fc = feature_column.categorical_column_with_hash_bucket( - feature_name, - hash_bucket_size, - dtype=tf.string, - feature_name=feature_name) + feature_name, + hash_bucket_size, + dtype=tf.string, + feature_name=feature_name + ) if self.is_wide(config): self._add_wide_embedding_column(fc, config) @@ -490,35 +526,40 @@ def parse_sequence_feature(self, config): if config.HasField('hash_bucket_size'): hash_bucket_size = self._get_hash_bucket_size(config) fc = sequence_feature_column.sequence_categorical_column_with_hash_bucket( - feature_name, - hash_bucket_size, - dtype=tf.string, - feature_name=feature_name) + feature_name, + hash_bucket_size, + dtype=tf.string, + feature_name=feature_name + ) elif config.vocab_list: fc = sequence_feature_column.sequence_categorical_column_with_vocabulary_list( - feature_name, - default_value=0, - vocabulary_list=config.vocab_list, - feature_name=feature_name) + feature_name, + default_value=0, + vocabulary_list=config.vocab_list, + feature_name=feature_name + ) elif config.vocab_file: fc = sequence_feature_column.sequence_categorical_column_with_vocabulary_file( - feature_name, - default_value=0, - vocabulary_file=config.vocab_file, - vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name) + feature_name, + default_value=0, + vocabulary_file=config.vocab_file, + vocabulary_size=self._get_vocab_size(config.vocab_file), + feature_name=feature_name + ) else: use_ev = self._global_ev_params or config.HasField('ev_params') num_buckets = sys.maxsize if use_ev else config.num_buckets fc = sequence_feature_column.sequence_categorical_column_with_identity( - feature_name, - num_buckets, - default_value=0, - feature_name=feature_name) + feature_name, + num_buckets, + default_value=0, + feature_name=feature_name + ) else: # raw feature bounds = None fc = sequence_feature_column.sequence_numeric_column( - feature_name, shape=(1,), feature_name=feature_name) + feature_name, shape=(1, ), feature_name=feature_name + ) if config.hash_bucket_size > 0: hash_bucket_size = self._get_hash_bucket_size(config) assert sub_feature_type == config.IdFeature, \ @@ -529,34 +570,41 @@ def parse_sequence_feature(self, config): elif config.num_buckets > 1 and config.max_val > config.min_val: # the feature values are already normalized into [0, 1] bounds = [ - x / float(config.num_buckets) for x in range(0, config.num_buckets) + x / float(config.num_buckets) for x in range(0, config.num_buckets) ] - logging.info('sequence feature discrete %s into %d buckets' % - (feature_name, config.num_buckets)) + logging.info( + 'sequence feature discrete %s into %d buckets' % + (feature_name, config.num_buckets) + ) if bounds: try: fc = sequence_feature_column.sequence_numeric_column_with_bucketized_column( - fc, bounds) + fc, bounds + ) except Exception as e: logging.error( - 'sequence features bucketized_column [%s] with bounds %s error' % - (feature_name, str(bounds))) + 'sequence features bucketized_column [%s] with bounds %s error' % + (feature_name, str(bounds)) + ) raise e elif config.hash_bucket_size <= 0: if config.embedding_dim > 0: tmp_id_col = sequence_feature_column.sequence_categorical_column_with_identity( - feature_name + '_raw_proj_id', - config.raw_input_dim, - default_value=0, - feature_name=feature_name) + feature_name + '_raw_proj_id', + config.raw_input_dim, + default_value=0, + feature_name=feature_name + ) wgt_fc = sequence_feature_column.sequence_weighted_categorical_column( - tmp_id_col, - weight_feature_key=feature_name + '_raw_proj_val', - dtype=tf.float32) + tmp_id_col, + weight_feature_key=feature_name + '_raw_proj_val', + dtype=tf.float32 + ) fc = wgt_fc else: fc = sequence_feature_column.sequence_numeric_column_with_raw_column( - fc, config.sequence_length) + fc, config.sequence_length + ) if config.embedding_dim > 0: self._add_deep_embedding_column(fc, config) @@ -568,10 +616,12 @@ def _build_partitioner(self, config): if self._global_ev_params is not None or config.HasField('ev_params'): # pai embedding_variable should use fixed_size_partitioner return partitioned_variables.fixed_size_partitioner( - num_shards=config.max_partitions) + num_shards=config.max_partitions + ) else: return partitioned_variables.min_max_variable_partitioner( - max_partitions=config.max_partitions) + max_partitions=config.max_partitions + ) else: return None @@ -604,7 +654,8 @@ def _add_wide_embedding_column(self, fc, config): assert self._wide_output_dim > 0, 'wide_output_dim is not set' if config.embedding_name in self._wide_share_embed_columns: wide_fc = self._add_shared_embedding_column( - config.embedding_name, fc, deep=False) + config.embedding_name, fc, deep=False + ) else: initializer = None if config.HasField('initializer'): @@ -614,12 +665,13 @@ def _add_wide_embedding_column(self, fc, config): else: ev_params = self._global_ev_params wide_fc = feature_column.embedding_column( - fc, - self._wide_output_dim, - combiner='sum', - initializer=initializer, - partitioner=self._build_partitioner(config), - ev_params=ev_params) + fc, + self._wide_output_dim, + combiner='sum', + initializer=initializer, + partitioner=self._build_partitioner(config), + ev_params=ev_params + ) self._wide_columns[feature_name] = wide_fc def _add_deep_embedding_column(self, fc, config): @@ -639,14 +691,16 @@ def _add_deep_embedding_column(self, fc, config): else: ev_params = self._global_ev_params fc = feature_column.embedding_column( - fc, - config.embedding_dim, - combiner=config.combiner, - initializer=initializer, - partitioner=self._build_partitioner(config), - ev_params=ev_params) + fc, + config.embedding_dim, + combiner=config.combiner, + initializer=initializer, + partitioner=self._build_partitioner(config), + ev_params=ev_params + ) fc.max_seq_length = config.max_seq_len if config.HasField( - 'max_seq_len') else -1 + 'max_seq_len' + ) else -1 if config.feature_type != config.SequenceFeature: self._deep_columns[feature_name] = fc @@ -658,7 +712,8 @@ def _add_deep_embedding_column(self, fc, config): def _build_ev_params(self, ev_params): """Build embedding_variables params.""" ev_params = EVParams( - ev_params.filter_freq, - ev_params.steps_to_live if ev_params.steps_to_live > 0 else None, - ev_params.use_cache, ev_params.init_capacity, ev_params.max_capacity) + ev_params.filter_freq, + ev_params.steps_to_live if ev_params.steps_to_live > 0 else None, + ev_params.use_cache, ev_params.init_capacity, ev_params.max_capacity + ) return ev_params diff --git a/easy_rec/python/feature_column/feature_group.py b/easy_rec/python/feature_column/feature_group.py index b04a635ad..a30cd820d 100644 --- a/easy_rec/python/feature_column/feature_group.py +++ b/easy_rec/python/feature_column/feature_group.py @@ -2,8 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import re -from easy_rec.python.protos.feature_config_pb2 import FeatureGroupConfig -from easy_rec.python.protos.feature_config_pb2 import WideOrDeep +from easy_rec.python.protos.feature_config_pb2 import FeatureGroupConfig, WideOrDeep # NOQA class FeatureGroup(object): diff --git a/easy_rec/python/hpo/emr_hpo.py b/easy_rec/python/hpo/emr_hpo.py index 9dd6e4dbd..fca116c64 100644 --- a/easy_rec/python/hpo/emr_hpo.py +++ b/easy_rec/python/hpo/emr_hpo.py @@ -7,27 +7,28 @@ import os import shutil import time - from pai.automl.hpo.autotuner import AutoTuner from easy_rec.python.utils import hpo_util file_dir, _ = os.path.split(os.path.abspath(__file__)) logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) -def hpo_config(config_path, hyperparams, exp_dir, metric_name, - el_submit_params): +def hpo_config( + config_path, hyperparams, exp_dir, metric_name, el_submit_params +): earlystop = { - 'type': 'large_is_better', - 'threshold': 0.99, - 'max_runtime': 2400 + 'type': 'large_is_better', + 'threshold': 0.99, + 'max_runtime': 2400 } algorithm = { - 'type': 'gp', - 'initial_trials_num': 4, - 'stop_when_exception': True + 'type': 'gp', + 'initial_trials_num': 4, + 'stop_when_exception': True } tmp_dir = '/tmp/emr_easy_rec_hpo_%d' % time.time() @@ -40,48 +41,50 @@ def hpo_config(config_path, hyperparams, exp_dir, metric_name, metric_path = os.path.join(model_path, 'res.metric') pre_task = { - 'type': 'BashTask', - 'cmd': ['hadoop', 'fs', '-mkdir', '-p', model_path] + 'type': 'BashTask', + 'cmd': ['hadoop', 'fs', '-mkdir', '-p', model_path] } adapter_task = { - 'type': 'localadaptertask', - # hpo_param_path for easy_rec - 'param_file': param_path, + 'type': 'localadaptertask', + # hpo_param_path for easy_rec + 'param_file': param_path, } el_params = [ - x.strip() for x in el_submit_params.split(' ') if x.strip() != '' + x.strip() for x in el_submit_params.split(' ') if x.strip() != '' ] - assert len( - el_params) % 2 == 0, 'invalid number of el_submit params: %d[%s]' % ( - len(el_params), str(el_params)) + assert len(el_params + ) % 2 == 0, 'invalid number of el_submit params: %d[%s]' % ( + len(el_params), str(el_params) + ) for i in range(0, len(el_params), 2): assert el_params[i] in [ - '-t', '-m', '-pn', '-pc', '-pg', '-pm', '-wn', '-wc', '-wm', '-wg' + '-t', '-m', '-pn', '-pc', '-pg', '-pm', '-wn', '-wc', '-wm', '-wg' ] cmd = ['el_submit'] + el_params + [ - '-a', 'easy_rec_hpo', '-m', 'local', '-f', '{},train_eval.py,{}'.format( - config_path, param_path), '--interact', 'INTERACT', '-c', - 'python -m easy_rec.python.train_eval --hpo_metric_save_path {} ' - '--hpo_param_path {} --pipeline_config_path {} --model_dir {}'.format( - metric_path, param_path_file, config_path, model_path) + '-a', 'easy_rec_hpo', '-m', 'local', '-f', + '{},train_eval.py,{}'.format(config_path, + param_path), '--interact', 'INTERACT', '-c', + 'python -m easy_rec.python.train_eval --hpo_metric_save_path {} ' + '--hpo_param_path {} --pipeline_config_path {} --model_dir {}'. + format(metric_path, param_path_file, config_path, model_path) ] train_task = { - 'type': 'BashTask', - 'cmd': cmd, - 'metric_reader': { - 'type': 'hdfs_reader', - 'location': metric_path, - 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name - } + 'type': 'BashTask', + 'cmd': cmd, + 'metric_reader': { + 'type': 'hdfs_reader', + 'location': metric_path, + 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name + } } tasks = [pre_task, adapter_task, train_task] data = { - 'earlystop': earlystop, - 'algorithm': algorithm, - 'hyperparams': hyperparams, - 'tasks': tasks + 'earlystop': earlystop, + 'algorithm': algorithm, + 'hyperparams': hyperparams, + 'tasks': tasks } return data, tmp_dir @@ -89,32 +92,41 @@ def hpo_config(config_path, hyperparams, exp_dir, metric_name, if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--hyperparams', type=str, help='hyper parameters', default=None) + '--hyperparams', type=str, help='hyper parameters', default=None + ) parser.add_argument( - '--config_path', type=str, help='pipeline config', default=None) + '--config_path', type=str, help='pipeline config', default=None + ) parser.add_argument( - '--exp_dir', type=str, help='hpo experiment directory', default=None) + '--exp_dir', type=str, help='hpo experiment directory', default=None + ) parser.add_argument( - '--el_submit_params', - type=str, - help='el_submit parameters(-t x -m x [-pn x -pc x -pm x] -wn x -wc x -wm x -wg x)', - default='-t standalone -m local -wn 1 -wc 6 -wm 20000 -wg 1') + '--el_submit_params', + type=str, + help= + 'el_submit parameters(-t x -m x [-pn x -pc x -pm x] -wn x -wc x -wm x -wg x)', + default='-t standalone -m local -wn 1 -wc 6 -wm 20000 -wg 1' + ) parser.add_argument( - '--metric_name', type=str, help='metric_name', default='auc') + '--metric_name', type=str, help='metric_name', default='auc' + ) parser.add_argument( - '--max_parallel', - type=int, - help='max number of trials run at the same time', - default=4) + '--max_parallel', + type=int, + help='max number of trials run at the same time', + default=4 + ) parser.add_argument( - '--total_trial_num', - type=int, - help='total number of trials will run', - default=6) + '--total_trial_num', + type=int, + help='total number of trials will run', + default=6 + ) parser.add_argument( - '--debug', - action='store_true', - help='debug mode, will keep the temporary folder') + '--debug', + action='store_true', + help='debug mode, will keep the temporary folder' + ) args = parser.parse_args() @@ -125,13 +137,16 @@ def hpo_config(config_path, hyperparams, exp_dir, metric_name, with open(args.hyperparams, 'r') as fin: hyperparams = json.load(fin) - data, tmp_dir = hpo_config(args.config_path, hyperparams, args.exp_dir, - args.metric_name, args.el_submit_params) + data, tmp_dir = hpo_config( + args.config_path, hyperparams, args.exp_dir, args.metric_name, + args.el_submit_params + ) hpo_util.kill_old_proc(tmp_dir, platform='emr') tuner = AutoTuner.create_tuner( - data, max_parallel=args.max_parallel, max_trial_num=args.total_trial_num) + data, max_parallel=args.max_parallel, max_trial_num=args.total_trial_num + ) tuner.fit(synchronize=True) if not args.debug: diff --git a/easy_rec/python/hpo/generate_hpo_sql.py b/easy_rec/python/hpo/generate_hpo_sql.py index 648be7c38..c91e0df9c 100644 --- a/easy_rec/python/hpo/generate_hpo_sql.py +++ b/easy_rec/python/hpo/generate_hpo_sql.py @@ -7,40 +7,53 @@ parser = argparse.ArgumentParser() parser.add_argument( - '--sql_path', type=str, help='output sql path', default=None) + '--sql_path', type=str, help='output sql path', default=None + ) parser.add_argument( - '--config_path', type=str, help='config path', default=None) + '--config_path', type=str, help='config path', default=None + ) parser.add_argument( - '--tables', type=str, help='train_table and test_table', default=None) + '--tables', type=str, help='train_table and test_table', default=None + ) parser.add_argument( - '--train_tables', type=str, help='train_tables', default=None) + '--train_tables', type=str, help='train_tables', default=None + ) parser.add_argument( - '--eval_tables', type=str, help='eval_tables', default=None) + '--eval_tables', type=str, help='eval_tables', default=None + ) parser.add_argument( - '--cluster', - type=str, - help='specify tensorflow train jobs cluster parameter', - default=None) + '--cluster', + type=str, + help='specify tensorflow train jobs cluster parameter', + default=None + ) parser.add_argument('--bucket', type=str, help='oss bucket', default=None) parser.add_argument( - '--hpo_param_path', type=str, help='hpo param path', default=None) + '--hpo_param_path', type=str, help='hpo param path', default=None + ) parser.add_argument( - '--hpo_metric_save_path', - type=str, - help='hpo metric save path', - default=None) + '--hpo_metric_save_path', + type=str, + help='hpo metric save path', + default=None + ) parser.add_argument('--model_dir', type=str, help='model_dir', default=None) - parser.add_argument('--oss_host', type=str, help='oss endpoint', default=None) + parser.add_argument( + '--oss_host', type=str, help='oss endpoint', default=None + ) parser.add_argument('--role_arn', type=str, help='role arn', default=None) parser.add_argument( - '--algo_proj_name', - type=str, - help='algorithm project name', - default='algo_public') + '--algo_proj_name', + type=str, + help='algorithm project name', + default='algo_public' + ) parser.add_argument( - '--algo_res_proj', type=str, help='algo resource project', default=None) + '--algo_res_proj', type=str, help='algo resource project', default=None + ) parser.add_argument( - '--algo_version', type=str, help='algo version', default=None) + '--algo_version', type=str, help='algo version', default=None + ) args = parser.parse_args() diff --git a/easy_rec/python/hpo/pai_hpo.py b/easy_rec/python/hpo/pai_hpo.py index 8d6edaa19..11fb5c6ab 100644 --- a/easy_rec/python/hpo/pai_hpo.py +++ b/easy_rec/python/hpo/pai_hpo.py @@ -6,14 +6,14 @@ import os import shutil import time - from pai.automl import hpo from easy_rec.python.utils import hpo_util file_dir, _ = os.path.split(os.path.abspath(__file__)) logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) try: import subprocess @@ -21,7 +21,7 @@ subprocess.check_output('which odpscmd', shell=True) except Exception: logging.error( - 'odpscmd is not in path, please install from https://help.aliyun.com/document_detail/27971.html' + 'odpscmd is not in path, please install from https://help.aliyun.com/document_detail/27971.html' ) @@ -59,25 +59,28 @@ def get_tuner(data, max_parallel, max_trial_num): algo = hpo.algorithm.create(**param_dict['algorithm']) tuner = hpo.autotuner.AutoTuner( - earlystop=early_stop, - algorithm=algo, - hyperparams=params, - task_list=tasks, - max_parallel=max_parallel, - max_trial_num=max_trial_num, - mode='local', - user_id='your_cloud_id') + earlystop=early_stop, + algorithm=algo, + hyperparams=params, + task_list=tasks, + max_parallel=max_parallel, + max_trial_num=max_trial_num, + mode='local', + user_id='your_cloud_id' + ) return tuner -def hpo_config(config_path, hyperparams, environment, exp_dir, tables, - train_tables, eval_tables, cluster, algo_proj_name, - algo_res_proj, algo_version, metric_name, odps_config_path): +def hpo_config( + config_path, hyperparams, environment, exp_dir, tables, train_tables, + eval_tables, cluster, algo_proj_name, algo_res_proj, algo_version, + metric_name, odps_config_path +): earlystop = {'type': 'large_is_better', 'max_runtime': 3600 * 12} algorithm = { - 'type': 'gp', - 'initial_trials_num': 4, - 'stop_when_exception': True + 'type': 'gp', + 'initial_trials_num': 4, + 'stop_when_exception': True } if exp_dir.startswith('oss://'): @@ -90,9 +93,9 @@ def hpo_config(config_path, hyperparams, environment, exp_dir, tables, bucket = 'oss://' + environment['bucket'].strip('/') + '/' adapter_task = { - 'type': 'ossadaptertask', - # hpo_param_path for easy_rec - 'param_file': param_path, + 'type': 'ossadaptertask', + # hpo_param_path for easy_rec + 'param_file': param_path, } tmp_dir = '/tmp/pai_easy_rec_hpo_%d' % time.time() @@ -118,21 +121,22 @@ def _add_prefix(table_name): sql_path = '%s/train_ext_hpo_{{ trial.id }}.sql' % tmp_dir cmd_args = [ - 'python', '-m', 'easy_rec.python.hpo.generate_hpo_sql', '--sql_path', - sql_path, '--config_path', config_path, '--cluster', cluster, '--bucket', - bucket, '--hpo_param_path', - os.path.join(bucket, param_path), '--hpo_metric_save_path', - os.path.join(bucket, metric_path), '--model_dir', - os.path.join(bucket, - model_path), '--oss_host', environment['oss_endpoint'], - '--role_arn', environment['role_arn'], '--algo_proj_name', algo_proj_name + 'python', '-m', 'easy_rec.python.hpo.generate_hpo_sql', '--sql_path', + sql_path, '--config_path', config_path, '--cluster', cluster, '--bucket', + bucket, '--hpo_param_path', + os.path.join(bucket, param_path), '--hpo_metric_save_path', + os.path.join(bucket, metric_path), '--model_dir', + os.path.join(bucket, + model_path), '--oss_host', environment['oss_endpoint'], + '--role_arn', environment['role_arn'], '--algo_proj_name', algo_proj_name ] if tables: cmd_args.extend(['--tables', tables]) if train_tables and eval_tables: cmd_args.extend( - ['--train_tables', train_tables, '--eval_tables', eval_tables]) + ['--train_tables', train_tables, '--eval_tables', eval_tables] + ) if algo_res_proj: cmd_args.extend(['--algo_res_proj', algo_res_proj]) @@ -141,23 +145,23 @@ def _add_prefix(table_name): prepare_sql_task = {'type': 'BashTask', 'cmd': cmd_args} train_task = { - 'type': 'BashTask', - 'cmd': ['odpscmd', - '--config=%s' % odps_config_path, '-f', sql_path], - 'metric_reader': { - 'type': 'oss_reader', - 'location': metric_path, - 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name - } + 'type': 'BashTask', + 'cmd': ['odpscmd', + '--config=%s' % odps_config_path, '-f', sql_path], + 'metric_reader': { + 'type': 'oss_reader', + 'location': metric_path, + 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name + } } tasks = [adapter_task, prepare_sql_task, train_task] data = { - 'earlystop': earlystop, - 'algorithm': algorithm, - 'hyperparams': hyperparams, - 'tasks': tasks, - 'environment': environment + 'earlystop': earlystop, + 'algorithm': algorithm, + 'hyperparams': hyperparams, + 'tasks': tasks, + 'environment': environment } return data, tmp_dir @@ -167,54 +171,70 @@ def _add_prefix(table_name): parser = argparse.ArgumentParser() parser.add_argument( - '--odps_config', type=str, help='odps_config.ini', default=None) + '--odps_config', type=str, help='odps_config.ini', default=None + ) parser.add_argument( - '--oss_config', type=str, help='excel config path', default='') + '--oss_config', type=str, help='excel config path', default='' + ) parser.add_argument('--bucket', type=str, help='bucket name', default=None) parser.add_argument('--role_arn', type=str, help='role arn', default=None) parser.add_argument( - '--hyperparams', type=str, help='hyper parameters', default=None) + '--hyperparams', type=str, help='hyper parameters', default=None + ) parser.add_argument( - '--config_path', type=str, help='pipeline config', default=None) + '--config_path', type=str, help='pipeline config', default=None + ) parser.add_argument( - '--tables', type=str, help='train table and test table', default=None) + '--tables', type=str, help='train table and test table', default=None + ) parser.add_argument( - '--train_tables', type=str, help='train tables', default=None) + '--train_tables', type=str, help='train tables', default=None + ) parser.add_argument( - '--eval_tables', type=str, help='eval tables', default=None) + '--eval_tables', type=str, help='eval tables', default=None + ) parser.add_argument( - '--exp_dir', type=str, help='hpo experiment directory', default=None) + '--exp_dir', type=str, help='hpo experiment directory', default=None + ) parser.add_argument( - '--cluster', - type=str, - help='cluster spec', - default='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}' + '--cluster', + type=str, + help='cluster spec', + default= + '{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}' ) parser.add_argument( - '--algo_proj_name', - type=str, - help='algo project name', - default='algo_public') + '--algo_proj_name', + type=str, + help='algo project name', + default='algo_public' + ) parser.add_argument( - '--algo_version', type=str, help='algo version', default=None) + '--algo_version', type=str, help='algo version', default=None + ) parser.add_argument( - '--algo_res_proj', type=str, help='algo resource project', default=None) + '--algo_res_proj', type=str, help='algo resource project', default=None + ) parser.add_argument( - '--metric_name', type=str, help='evaluate metric name', default='auc') + '--metric_name', type=str, help='evaluate metric name', default='auc' + ) parser.add_argument( - '--max_parallel', - type=int, - help='max number of trials run at the same time', - default=4) + '--max_parallel', + type=int, + help='max number of trials run at the same time', + default=4 + ) parser.add_argument( - '--total_trial_num', - type=int, - help='total number of trials will run', - default=6) + '--total_trial_num', + type=int, + help='total number of trials will run', + default=6 + ) parser.add_argument( - '--debug', - action='store_true', - help='debug mode, will keep the temporary folder') + '--debug', + action='store_true', + help='debug mode, will keep the temporary folder' + ) args = parser.parse_args() @@ -258,16 +278,16 @@ def _add_prefix(table_name): args.bucket = args.bucket.strip('/') environment = { - 'access_id': odps_config['access_id'], - 'access_key': odps_config['access_key'], - 'oss_access_id': oss_config['accessKeyID'], - 'oss_access_key': oss_config['accessKeySecret'], - 'project': odps_config['project_name'], - 'odps_endpoint': odps_config['end_point'], - 'biz_id': '147331^paistudio^xxxxxxx^2020-03-18', - 'role_arn': args.role_arn, - 'bucket': args.bucket, - 'oss_endpoint': oss_config['endpoint'] + 'access_id': odps_config['access_id'], + 'access_key': odps_config['access_key'], + 'oss_access_id': oss_config['accessKeyID'], + 'oss_access_key': oss_config['accessKeySecret'], + 'project': odps_config['project_name'], + 'odps_endpoint': odps_config['end_point'], + 'biz_id': '147331^paistudio^xxxxxxx^2020-03-18', + 'role_arn': args.role_arn, + 'bucket': args.bucket, + 'oss_endpoint': oss_config['endpoint'] } assert args.hyperparams is not None @@ -276,15 +296,15 @@ def _add_prefix(table_name): assert args.config_path is not None assert args.exp_dir is not None - assert args.tables is not None or (args.train_tables is not None and - args.eval_tables is not None) - - data, tmp_dir = hpo_config(args.config_path, hyperparams, environment, - args.exp_dir, args.tables, args.train_tables, - args.eval_tables, args.cluster, - args.algo_proj_name, args.algo_res_proj, - args.algo_version, args.metric_name, - args.odps_config) + assert args.tables is not None or ( + args.train_tables is not None and args.eval_tables is not None + ) + + data, tmp_dir = hpo_config( + args.config_path, hyperparams, environment, args.exp_dir, args.tables, + args.train_tables, args.eval_tables, args.cluster, args.algo_proj_name, + args.algo_res_proj, args.algo_version, args.metric_name, args.odps_config + ) hpo_util.kill_old_proc(tmp_dir, platform='pai') data_json = json.dumps(data) diff --git a/easy_rec/python/inference/client/client_demo.py b/easy_rec/python/inference/client/client_demo.py index 9464b1073..13e5e9726 100644 --- a/easy_rec/python/inference/client/client_demo.py +++ b/easy_rec/python/inference/client/client_demo.py @@ -6,11 +6,11 @@ import traceback from easy_rec.python.inference.client.easyrec_request import EasyrecRequest -from easy_rec.python.protos.predict_pb2 import PBFeature -from easy_rec.python.protos.predict_pb2 import PBRequest +from easy_rec.python.protos.predict_pb2 import PBFeature, PBRequest logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) try: from eas_prediction import PredictClient # TFRequest @@ -71,24 +71,29 @@ def send_request(req_pb, client, debug_level=0): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--endpoint', - type=str, - default=None, - help='eas endpoint, such as 12345.cn-beijing.pai-eas.aliyuncs.com') + '--endpoint', + type=str, + default=None, + help='eas endpoint, such as 12345.cn-beijing.pai-eas.aliyuncs.com' + ) parser.add_argument( - '--service_name', type=str, default=None, help='eas service name') + '--service_name', type=str, default=None, help='eas service name' + ) parser.add_argument( - '--token', type=str, default=None, help='eas service token') + '--token', type=str, default=None, help='eas service token' + ) parser.add_argument( - '--table_schema', - type=str, - default=None, - help='user feature table schema path') + '--table_schema', + type=str, + default=None, + help='user feature table schema path' + ) parser.add_argument( - '--table_data', - type=str, - default=None, - help='user feature table data path') + '--table_data', + type=str, + default=None, + help='user feature table data path' + ) parser.add_argument('--item_lst', type=str, default=None, help='item list') args, _ = parser.parse_known_args() diff --git a/easy_rec/python/inference/client/easyrec_request.py b/easy_rec/python/inference/client/easyrec_request.py index 4980b5064..ba44446d9 100644 --- a/easy_rec/python/inference/client/easyrec_request.py +++ b/easy_rec/python/inference/client/easyrec_request.py @@ -2,8 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from eas_prediction.request import Request -from easy_rec.python.protos.predict_pb2 import PBRequest -from easy_rec.python.protos.predict_pb2 import PBResponse +from easy_rec.python.protos.predict_pb2 import PBRequest, PBResponse # from eas_prediction.request import Response diff --git a/easy_rec/python/inference/csv_predictor.py b/easy_rec/python/inference/csv_predictor.py index c5e154d2e..9053ff3f2 100644 --- a/easy_rec/python/inference/csv_predictor.py +++ b/easy_rec/python/inference/csv_predictor.py @@ -1,17 +1,13 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import logging import os - import tensorflow as tf from tensorflow.python.platform import gfile -from easy_rec.python.inference.predictor import SINGLE_PLACEHOLDER_FEATURE_KEY -from easy_rec.python.inference.predictor import Predictor +from easy_rec.python.inference.predictor import SINGLE_PLACEHOLDER_FEATURE_KEY, Predictor # NOQA from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils.check_utils import check_split @@ -21,16 +17,19 @@ class CSVPredictor(Predictor): - def __init__(self, - model_path, - data_config, - with_header=False, - ds_vector_recall=False, - fg_json_path=None, - profiling_file=None, - selected_cols=None, - output_sep=chr(1)): - super(CSVPredictor, self).__init__(model_path, profiling_file, fg_json_path) + def __init__( + self, + model_path, + data_config, + with_header=False, + ds_vector_recall=False, + fg_json_path=None, + profiling_file=None, + selected_cols=None, + output_sep=chr(1) + ): + super(CSVPredictor, + self).__init__(model_path, profiling_file, fg_json_path) self._output_sep = output_sep self._ds_vector_recall = ds_vector_recall input_type = DatasetConfig.InputType.Name(data_config.input_type).lower() @@ -73,17 +72,19 @@ def _get_reserved_cols(self, reserved_cols): def _parse_line(self, line): check_list = [ - tf.py_func( - check_split, [line, self._input_sep, - len(self._record_defaults)], - Tout=tf.bool) + tf.py_func( + check_split, [line, self._input_sep, + len(self._record_defaults)], + Tout=tf.bool + ) ] with tf.control_dependencies(check_list): fields = tf.decode_csv( - line, - field_delim=self._input_sep, - record_defaults=self._record_defaults, - name='decode_csv') + line, + field_delim=self._input_sep, + record_defaults=self._record_defaults, + name='decode_csv' + ) if self._is_rtp: if self._with_header: inputs = dict(zip(self._all_fields, fields)) @@ -110,8 +111,9 @@ def _get_num_cols(self, file_paths): line_tok = line_str.strip().split(self._input_sep) if num_cols != -1: assert num_cols == len(line_tok), ( - 'num selected cols is %d, not equal to %d, current line is: %s, please check input_sep and data.' - % (num_cols, len(line_tok), line_str)) + 'num selected cols is %d, not equal to %d, current line is: %s, please check input_sep and data.' + % (num_cols, len(line_tok), line_str) + ) num_cols = len(line_tok) num_lines += 1 if num_lines > 10: @@ -119,8 +121,9 @@ def _get_num_cols(self, file_paths): logging.info('num selected cols = %d' % num_cols) return num_cols - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, - slice_id): + def _get_dataset( + self, input_path, num_parallel_calls, batch_size, slice_num, slice_id + ): file_paths = [] for path in input_path.split(','): for x in gfile.Glob(path): @@ -151,15 +154,16 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, self._record_defaults[col_idx] = default_val else: self._record_defaults = [ - self._get_defaults(col_name) for col_name in self._all_fields + self._get_defaults(col_name) for col_name in self._all_fields ] dataset = tf.data.Dataset.from_tensor_slices(file_paths) parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset(x).skip(int(self._with_header)), - cycle_length=parallel_num, - num_parallel_calls=parallel_num) + lambda x: tf.data.TextLineDataset(x).skip(int(self._with_header)), + cycle_length=parallel_num, + num_parallel_calls=parallel_num + ) dataset = dataset.shard(slice_num, slice_id) dataset = dataset.batch(batch_size) dataset = dataset.prefetch(buffer_size=64) @@ -171,12 +175,14 @@ def _get_writer(self, output_path, slice_id): res_path = os.path.join(output_path, 'part-%d.csv' % slice_id) table_writer = gfile.GFile(res_path, 'w') table_writer.write( - self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') + self._output_sep.join(self._output_cols + self._reserved_cols) + '\n' + ) return table_writer def _write_lines(self, table_writer, outputs): outputs = '\n'.join( - [self._output_sep.join([str(i) for i in output]) for output in outputs]) + [self._output_sep.join([str(i) for i in output]) for output in outputs] + ) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): diff --git a/easy_rec/python/inference/hive_parquet_predictor.py b/easy_rec/python/inference/hive_parquet_predictor.py index bd5178fe7..e6c878309 100644 --- a/easy_rec/python/inference/hive_parquet_predictor.py +++ b/easy_rec/python/inference/hive_parquet_predictor.py @@ -1,15 +1,12 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import time +from __future__ import absolute_import, division, print_function import numpy as np +import os import pandas as pd import tensorflow as tf +import time from tensorflow.python.platform import gfile from easy_rec.python.inference.predictor import Predictor @@ -24,17 +21,19 @@ class HiveParquetPredictor(Predictor): - def __init__(self, - model_path, - data_config, - hive_config, - fg_json_path=None, - profiling_file=None, - output_sep=chr(1), - all_cols=None, - all_col_types=None): - super(HiveParquetPredictor, self).__init__(model_path, profiling_file, - fg_json_path) + def __init__( + self, + model_path, + data_config, + hive_config, + fg_json_path=None, + profiling_file=None, + output_sep=chr(1), + all_cols=None, + all_col_types=None + ): + super(HiveParquetPredictor, + self).__init__(model_path, profiling_file, fg_json_path) self._data_config = data_config self._hive_config = hive_config @@ -47,8 +46,8 @@ def __init__(self, self._all_cols = [x.strip() for x in all_cols if x != ''] self._all_col_types = [x.strip() for x in all_col_types if x != ''] self._record_defaults = [ - self._get_defaults(col_name, col_type) - for col_name, col_type in zip(self._all_cols, self._all_col_types) + self._get_defaults(col_name, col_type) + for col_name, col_type in zip(self._all_cols, self._all_col_types) ] def _get_reserved_cols(self, reserved_cols): @@ -63,24 +62,29 @@ def _parse_line(self, *fields): field_dict = {self._all_cols[i]: fields[i] for i in range(len(fields))} return field_dict - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, - slice_id): + def _get_dataset( + self, input_path, num_parallel_calls, batch_size, slice_num, slice_id + ): self._hive_util = HiveUtils( - data_config=self._data_config, hive_config=self._hive_config) + data_config=self._data_config, hive_config=self._hive_config + ) hdfs_path = self._hive_util.get_table_location(input_path) self._input_hdfs_path = gfile.Glob(os.path.join(hdfs_path, '*')) - assert len(self._input_hdfs_path) > 0, 'match no files with %s' % input_path + assert len( + self._input_hdfs_path + ) > 0, 'match no files with %s' % input_path list_type = [] input_field_type_map = { - x.input_name: x.input_type for x in self._data_config.input_fields + x.input_name: x.input_type + for x in self._data_config.input_fields } type_2_tftype = { - 'string': tf.string, - 'double': tf.double, - 'float': tf.float32, - 'bigint': tf.int32, - 'boolean': tf.bool + 'string': tf.string, + 'double': tf.double, + 'float': tf.float32, + 'bigint': tf.int32, + 'boolean': tf.bool } for col_name, col_type in zip(self._all_cols, self._all_col_types): if col_name in input_field_type_map: @@ -113,7 +117,8 @@ def parquet_read(): yield tuple(inputs) dataset = tf.data.Dataset.from_generator( - parquet_read, output_types=list_type, output_shapes=list_shapes) + parquet_read, output_types=list_type, output_shapes=list_shapes + ) dataset = dataset.shard(slice_num, slice_id) dataset = dataset.prefetch(buffer_size=64) return dataset @@ -128,14 +133,18 @@ def get_table_info(self, output_path): return table_name, partition_name, partition_val def _get_writer(self, output_path, slice_id): - table_name, partition_name, partition_val = self.get_table_info(output_path) + table_name, partition_name, partition_val = self.get_table_info( + output_path + ) is_exist = self._hive_util.is_table_or_partition_exist( - table_name, partition_name, partition_val) + table_name, partition_name, partition_val + ) assert not is_exist, '%s is already exists. Please drop it.' % output_path output_path = output_path.replace('.', '/') self._hdfs_path = 'hdfs://%s:9000/user/easy_rec/%s_tmp' % ( - self._hive_config.host, output_path) + self._hive_config.host, output_path + ) if not gfile.Exists(self._hdfs_path): gfile.MakeDirs(self._hdfs_path) res_path = os.path.join(self._hdfs_path, 'part-%d.csv' % slice_id) @@ -144,7 +153,8 @@ def _get_writer(self, output_path, slice_id): def _write_lines(self, table_writer, outputs): outputs = '\n'.join( - [self._output_sep.join([str(i) for i in output]) for output in outputs]) + [self._output_sep.join([str(i) for i in output]) for output in outputs] + ) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): @@ -166,7 +176,9 @@ def load_to_table(self, output_path, slice_num, slice_id): while not gfile.Exists(res_path): time.sleep(10) - table_name, partition_name, partition_val = self.get_table_info(output_path) + table_name, partition_name, partition_val = self.get_table_info( + output_path + ) schema = '' for output_col_name in self._output_cols: tf_type = self._predictor_impl._outputs_map[output_col_name].dtype diff --git a/easy_rec/python/inference/hive_predictor.py b/easy_rec/python/inference/hive_predictor.py index f2923f8a3..cf21062fc 100644 --- a/easy_rec/python/inference/hive_predictor.py +++ b/easy_rec/python/inference/hive_predictor.py @@ -1,13 +1,10 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os -import time - import tensorflow as tf +import time from tensorflow.python.platform import gfile from easy_rec.python.inference.predictor import Predictor @@ -21,17 +18,19 @@ class HivePredictor(Predictor): - def __init__(self, - model_path, - data_config, - hive_config, - fg_json_path=None, - profiling_file=None, - output_sep=chr(1), - all_cols=None, - all_col_types=None): - super(HivePredictor, self).__init__(model_path, profiling_file, - fg_json_path) + def __init__( + self, + model_path, + data_config, + hive_config, + fg_json_path=None, + profiling_file=None, + output_sep=chr(1), + all_cols=None, + all_col_types=None + ): + super(HivePredictor, + self).__init__(model_path, profiling_file, fg_json_path) self._data_config = data_config self._hive_config = hive_config @@ -44,8 +43,8 @@ def __init__(self, self._all_cols = [x.strip() for x in all_cols if x != ''] self._all_col_types = [x.strip() for x in all_col_types if x != ''] self._record_defaults = [ - self._get_defaults(col_name, col_type) - for col_name, col_type in zip(self._all_cols, self._all_col_types) + self._get_defaults(col_name, col_type) + for col_name, col_type in zip(self._all_cols, self._all_col_types) ] def _get_reserved_cols(self, reserved_cols): @@ -58,17 +57,20 @@ def _get_reserved_cols(self, reserved_cols): def _parse_line(self, line): field_delim = self._data_config.rtp_separator if self._is_rtp else self._data_config.separator fields = tf.decode_csv( - line, - field_delim=field_delim, - record_defaults=self._record_defaults, - name='decode_csv') + line, + field_delim=field_delim, + record_defaults=self._record_defaults, + name='decode_csv' + ) inputs = {self._all_cols[x]: fields[x] for x in range(len(fields))} return inputs - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, - slice_id): + def _get_dataset( + self, input_path, num_parallel_calls, batch_size, slice_num, slice_id + ): self._hive_util = HiveUtils( - data_config=self._data_config, hive_config=self._hive_config) + data_config=self._data_config, hive_config=self._hive_config + ) self._input_hdfs_path = self._hive_util.get_table_location(input_path) file_paths = tf.gfile.Glob(os.path.join(self._input_hdfs_path, '*')) assert len(file_paths) > 0, 'match no files with %s' % input_path @@ -76,9 +78,10 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, dataset = tf.data.Dataset.from_tensor_slices(file_paths) parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - tf.data.TextLineDataset, - cycle_length=parallel_num, - num_parallel_calls=parallel_num) + tf.data.TextLineDataset, + cycle_length=parallel_num, + num_parallel_calls=parallel_num + ) dataset = dataset.shard(slice_num, slice_id) dataset = dataset.batch(batch_size) dataset = dataset.prefetch(buffer_size=64) @@ -94,14 +97,18 @@ def get_table_info(self, output_path): return table_name, partition_name, partition_val def _get_writer(self, output_path, slice_id): - table_name, partition_name, partition_val = self.get_table_info(output_path) + table_name, partition_name, partition_val = self.get_table_info( + output_path + ) is_exist = self._hive_util.is_table_or_partition_exist( - table_name, partition_name, partition_val) + table_name, partition_name, partition_val + ) assert not is_exist, '%s is already exists. Please drop it.' % output_path output_path = output_path.replace('.', '/') self._hdfs_path = 'hdfs://%s:9000/user/easy_rec/%s_tmp' % ( - self._hive_config.host, output_path) + self._hive_config.host, output_path + ) if not gfile.Exists(self._hdfs_path): gfile.MakeDirs(self._hdfs_path) res_path = os.path.join(self._hdfs_path, 'part-%d.csv' % slice_id) @@ -110,7 +117,8 @@ def _get_writer(self, output_path, slice_id): def _write_lines(self, table_writer, outputs): outputs = '\n'.join( - [self._output_sep.join([str(i) for i in output]) for output in outputs]) + [self._output_sep.join([str(i) for i in output]) for output in outputs] + ) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): @@ -132,7 +140,9 @@ def load_to_table(self, output_path, slice_num, slice_id): while not gfile.Exists(res_path): time.sleep(10) - table_name, partition_name, partition_val = self.get_table_info(output_path) + table_name, partition_name, partition_val = self.get_table_info( + output_path + ) schema = '' for output_col_name in self._output_cols: tf_type = self._predictor_impl._outputs_map[output_col_name].dtype diff --git a/easy_rec/python/inference/odps_predictor.py b/easy_rec/python/inference/odps_predictor.py index 183fc4d13..e38ff7ff7 100644 --- a/easy_rec/python/inference/odps_predictor.py +++ b/easy_rec/python/inference/odps_predictor.py @@ -1,8 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf @@ -11,21 +9,23 @@ class ODPSPredictor(Predictor): - def __init__(self, - model_path, - fg_json_path=None, - profiling_file=None, - all_cols='', - all_col_types=''): - super(ODPSPredictor, self).__init__(model_path, profiling_file, - fg_json_path) + def __init__( + self, + model_path, + fg_json_path=None, + profiling_file=None, + all_cols='', + all_col_types='' + ): + super(ODPSPredictor, + self).__init__(model_path, profiling_file, fg_json_path) self._all_cols = [x.strip() for x in all_cols.split(',') if x != ''] self._all_col_types = [ - x.strip() for x in all_col_types.split(',') if x != '' + x.strip() for x in all_col_types.split(',') if x != '' ] self._record_defaults = [ - self._get_defaults(col_name, col_type) - for col_name, col_type in zip(self._all_cols, self._all_col_types) + self._get_defaults(col_name, col_type) + for col_name, col_type in zip(self._all_cols, self._all_col_types) ] def _get_reserved_cols(self, reserved_cols): @@ -37,15 +37,17 @@ def _parse_line(self, *fields): field_dict = {self._all_cols[i]: fields[i] for i in range(len(fields))} return field_dict - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, - slice_id): + def _get_dataset( + self, input_path, num_parallel_calls, batch_size, slice_num, slice_id + ): input_list = input_path.split(',') dataset = tf.data.TableRecordDataset( - input_list, - record_defaults=self._record_defaults, - slice_id=slice_id, - slice_count=slice_num, - selected_cols=','.join(self._all_cols)) + input_list, + record_defaults=self._record_defaults, + slice_id=slice_id, + slice_count=slice_num, + selected_cols=','.join(self._all_cols) + ) dataset = dataset.batch(batch_size) dataset = dataset.prefetch(buffer_size=64) return dataset diff --git a/easy_rec/python/inference/parquet_predictor.py b/easy_rec/python/inference/parquet_predictor.py index 7fead6388..3d6f8dac1 100644 --- a/easy_rec/python/inference/parquet_predictor.py +++ b/easy_rec/python/inference/parquet_predictor.py @@ -1,13 +1,10 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import logging -import os - import numpy as np +import os import pandas as pd import tensorflow as tf from tensorflow.python.platform import gfile @@ -15,11 +12,11 @@ from easy_rec.python.inference.predictor import Predictor from easy_rec.python.input.parquet_input import ParquetInput from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.utils import config_util -from easy_rec.python.utils import input_utils +from easy_rec.python.utils import config_util, input_utils try: from tensorflow.python.framework.load_library import load_op_library + import easy_rec load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) @@ -29,17 +26,19 @@ class ParquetPredictor(Predictor): - def __init__(self, - model_path, - data_config, - ds_vector_recall=False, - fg_json_path=None, - profiling_file=None, - selected_cols=None, - output_sep=chr(1), - pipeline_config=None): - super(ParquetPredictor, self).__init__(model_path, profiling_file, - fg_json_path) + def __init__( + self, + model_path, + data_config, + ds_vector_recall=False, + fg_json_path=None, + profiling_file=None, + selected_cols=None, + output_sep=chr(1), + pipeline_config=None + ): + super(ParquetPredictor, + self).__init__(model_path, profiling_file, fg_json_path) self._output_sep = output_sep self._ds_vector_recall = ds_vector_recall input_type = DatasetConfig.InputType.Name(data_config.input_type).lower() @@ -73,10 +72,12 @@ def _get_reserved_cols(self, reserved_cols): # already parsed in _get_dataset return self._reserved_cols - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, - slice_id): + def _get_dataset( + self, input_path, num_parallel_calls, batch_size, slice_num, slice_id + ): feature_configs = config_util.get_compatible_feature_configs( - self.pipeline_config) + self.pipeline_config + ) kwargs = {} if self._reserved_args is not None and len(self._reserved_args) > 0: @@ -89,31 +90,37 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, self._all_fields = all_cols self._reserved_cols = all_cols kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( - all_cols, parquet_file) + all_cols, parquet_file + ) else: self._reserved_cols = [ - x.strip() for x in self._reserved_args.split(',') if x.strip() != '' + x.strip() for x in self._reserved_args.split(',') if x.strip() != '' ] kwargs['reserve_fields'] = self._reserved_cols parquet_file = gfile.Glob(input_path.split(',')[0])[0] kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( - self._reserved_cols, parquet_file) - logging.info('reserve_fields=%s reserve_types=%s' % - (','.join(self._reserved_cols), ','.join( - [str(x) for x in kwargs['reserve_types']]))) + self._reserved_cols, parquet_file + ) + logging.info( + 'reserve_fields=%s reserve_types=%s' % ( + ','.join(self._reserved_cols + ), ','.join([str(x) for x in kwargs['reserve_types']]) + ) + ) else: self._reserved_cols = [] self.pipeline_config.data_config.batch_size = batch_size kwargs['is_predictor'] = True parquet_input = ParquetInput( - self.pipeline_config.data_config, - feature_configs, - input_path, - task_index=slice_id, - task_num=slice_num, - pipeline_config=self.pipeline_config, - **kwargs) + self.pipeline_config.data_config, + feature_configs, + input_path, + task_index=slice_id, + task_num=slice_num, + pipeline_config=self.pipeline_config, + **kwargs + ) return parquet_input._build(tf.estimator.ModeKeys.PREDICT, {}) def _get_writer(self, output_path, slice_id): @@ -122,12 +129,14 @@ def _get_writer(self, output_path, slice_id): res_path = os.path.join(output_path, 'part-%d.csv' % slice_id) table_writer = gfile.GFile(res_path, 'w') table_writer.write( - self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') + self._output_sep.join(self._output_cols + self._reserved_cols) + '\n' + ) return table_writer def _write_lines(self, table_writer, outputs): outputs = '\n'.join( - [self._output_sep.join([str(i) for i in output]) for output in outputs]) + [self._output_sep.join([str(i) for i in output]) for output in outputs] + ) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): diff --git a/easy_rec/python/inference/parquet_predictor_v2.py b/easy_rec/python/inference/parquet_predictor_v2.py index 1ee08517b..2ece02cea 100644 --- a/easy_rec/python/inference/parquet_predictor_v2.py +++ b/easy_rec/python/inference/parquet_predictor_v2.py @@ -1,13 +1,10 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import logging -import os - import numpy as np +import os import pandas as pd import tensorflow as tf from tensorflow.python.platform import gfile @@ -15,11 +12,11 @@ from easy_rec.python.inference.predictor import Predictor from easy_rec.python.input.parquet_input_v2 import ParquetInputV2 from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.utils import config_util -from easy_rec.python.utils import input_utils +from easy_rec.python.utils import config_util, input_utils try: from tensorflow.python.framework.load_library import load_op_library + import easy_rec load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) @@ -29,17 +26,19 @@ class ParquetPredictorV2(Predictor): - def __init__(self, - model_path, - data_config, - ds_vector_recall=False, - fg_json_path=None, - profiling_file=None, - selected_cols=None, - output_sep=chr(1), - pipeline_config=None): - super(ParquetPredictorV2, self).__init__(model_path, profiling_file, - fg_json_path) + def __init__( + self, + model_path, + data_config, + ds_vector_recall=False, + fg_json_path=None, + profiling_file=None, + selected_cols=None, + output_sep=chr(1), + pipeline_config=None + ): + super(ParquetPredictorV2, + self).__init__(model_path, profiling_file, fg_json_path) self._output_sep = output_sep self._ds_vector_recall = ds_vector_recall input_type = DatasetConfig.InputType.Name(data_config.input_type).lower() @@ -73,10 +72,12 @@ def _get_reserved_cols(self, reserved_cols): # already parsed in _get_dataset return self._reserved_cols - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, - slice_id): + def _get_dataset( + self, input_path, num_parallel_calls, batch_size, slice_num, slice_id + ): feature_configs = config_util.get_compatible_feature_configs( - self.pipeline_config) + self.pipeline_config + ) kwargs = {} if self._reserved_args is not None and len(self._reserved_args) > 0: @@ -89,31 +90,37 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, self._all_fields = all_cols self._reserved_cols = all_cols kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( - all_cols, parquet_file) + all_cols, parquet_file + ) else: self._reserved_cols = [ - x.strip() for x in self._reserved_args.split(',') if x.strip() != '' + x.strip() for x in self._reserved_args.split(',') if x.strip() != '' ] kwargs['reserve_fields'] = self._reserved_cols parquet_file = gfile.Glob(input_path.split(',')[0])[0] kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( - self._reserved_cols, parquet_file) - logging.info('reserve_fields=%s reserve_types=%s' % - (','.join(self._reserved_cols), ','.join( - [str(x) for x in kwargs['reserve_types']]))) + self._reserved_cols, parquet_file + ) + logging.info( + 'reserve_fields=%s reserve_types=%s' % ( + ','.join(self._reserved_cols + ), ','.join([str(x) for x in kwargs['reserve_types']]) + ) + ) else: self._reserved_cols = [] self.pipeline_config.data_config.batch_size = batch_size kwargs['is_predictor'] = True parquet_input = ParquetInputV2( - self.pipeline_config.data_config, - feature_configs, - input_path, - task_index=slice_id, - task_num=slice_num, - pipeline_config=self.pipeline_config, - **kwargs) + self.pipeline_config.data_config, + feature_configs, + input_path, + task_index=slice_id, + task_num=slice_num, + pipeline_config=self.pipeline_config, + **kwargs + ) return parquet_input._build(tf.estimator.ModeKeys.PREDICT, {}) def _get_writer(self, output_path, slice_id): @@ -122,12 +129,14 @@ def _get_writer(self, output_path, slice_id): res_path = os.path.join(output_path, 'part-%d.csv' % slice_id) table_writer = gfile.GFile(res_path, 'w') table_writer.write( - self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') + self._output_sep.join(self._output_cols + self._reserved_cols) + '\n' + ) return table_writer def _write_lines(self, table_writer, outputs): outputs = '\n'.join( - [self._output_sep.join([str(i) for i in output]) for output in outputs]) + [self._output_sep.join([str(i) for i in output]) for output in outputs] + ) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): diff --git a/easy_rec/python/inference/predictor.py b/easy_rec/python/inference/predictor.py index 009338fb8..ba9b4204d 100644 --- a/easy_rec/python/inference/predictor.py +++ b/easy_rec/python/inference/predictor.py @@ -1,29 +1,23 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import abc import json import logging import math -import os -import time - import numpy as np +import os import six import tensorflow as tf +import time from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.python.platform import gfile -from tensorflow.python.saved_model import constants -from tensorflow.python.saved_model import signature_constants +from tensorflow.python.saved_model import constants, signature_constants import easy_rec from easy_rec.python.utils import numpy_utils -from easy_rec.python.utils.config_util import get_configs_from_pipeline_file -from easy_rec.python.utils.config_util import get_input_name_from_fg_json -from easy_rec.python.utils.config_util import search_fg_json +from easy_rec.python.utils.config_util import get_configs_from_pipeline_file, get_input_name_from_fg_json, search_fg_json # NOQA from easy_rec.python.utils.input_utils import get_type_defaults from easy_rec.python.utils.load_class import get_register_class_meta @@ -39,7 +33,8 @@ _PREDICTOR_CLASS_MAP = {} _register_abc_meta = get_register_class_meta( - _PREDICTOR_CLASS_MAP, have_abstract_class=True) + _PREDICTOR_CLASS_MAP, have_abstract_class=True +) class PredictorInterface(six.with_metaclass(_register_abc_meta, object)): @@ -154,14 +149,18 @@ def search_pb(self, directory): raise ValueError('savedmodel is not found in directory %s' % directory) elif len(dir_list) > 1: if self._use_latest: - logging.info('find %d models: %s' % (len(dir_list), ','.join(dir_list))) + logging.info( + 'find %d models: %s' % (len(dir_list), ','.join(dir_list)) + ) dir_list = sorted( - dir_list, - key=lambda x: int(x.split('/')[(-2 if (x[-1] == '/') else -1)])) + dir_list, + key=lambda x: int(x.split('/')[(-2 if (x[-1] == '/') else -1)]) + ) return dir_list[-1] else: - raise ValueError('multiple saved model found in directory %s' % - directory) + raise ValueError( + 'multiple saved model found in directory %s' % directory + ) return dir_list[0] @@ -169,17 +168,20 @@ def _get_input_fields_from_pipeline_config(self, model_path): pipeline_path = os.path.join(model_path, 'assets/pipeline.config') if not gfile.Exists(pipeline_path): logging.warning( - '%s not exists, default values maybe inconsistent with the values used in training.' - % pipeline_path) + '%s not exists, default values maybe inconsistent with the values used in training.' + % pipeline_path + ) return {} pipeline_config = get_configs_from_pipeline_file(pipeline_path) input_fields = pipeline_config.data_config.input_fields input_fields_info = { - input_field.input_name: - (input_field.input_type, input_field.default_val) - for input_field in input_fields + input_field.input_name: + (input_field.input_type, input_field.default_val) + for input_field in input_fields } - input_fields_list = [input_field.input_name for input_field in input_fields] + input_fields_list = [ + input_field.input_name for input_field in input_fields + ] return input_fields_info, input_fields_list @@ -189,9 +191,10 @@ def _build_model(self): self._graph = tf.Graph() gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=(self._profiling_file is not None)) + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=(self._profiling_file is not None) + ) self._session = tf.Session(config=session_config, graph=self._graph) with self._graph.as_default(): @@ -203,15 +206,17 @@ def _build_model(self): model_path = self.search_pb(model_path) logging.info('model find in %s' % model_path) self._input_fields_info, self._input_fields_list = self._get_input_fields_from_pipeline_config( - model_path) + model_path + ) assert tf.saved_model.loader.maybe_saved_model_directory(model_path), \ 'saved model does not exists in %s' % model_path self._is_saved_model = True meta_graph_def = tf.saved_model.loader.load( - self._session, [tf.saved_model.tag_constants.SERVING], model_path) + self._session, [tf.saved_model.tag_constants.SERVING], model_path + ) # parse signature signature_def = meta_graph_def.signature_def[ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] inputs = signature_def.inputs # each input_info is a tuple of input_id, name, data_type input_info = [] @@ -219,7 +224,9 @@ def _build_model(self): if self._is_multi_placeholder: for gid, item in enumerate(inputs.items()): name, tensor = item - logging.info('Load input binding: %s -> %s' % (name, tensor.name)) + logging.info( + 'Load input binding: %s -> %s' % (name, tensor.name) + ) input_name = tensor.name input_name, _ = input_name.split(':') try: @@ -231,19 +238,24 @@ def _build_model(self): # same as they are defined, thereforce, list input # could not be supported, only dict input could be supported logging.warning( - 'could not determine input_id from input_name: %s' % - input_name) + 'could not determine input_id from input_name: %s' % + input_name + ) input_id = gid input_info.append((input_id, name, tensor.dtype)) self._inputs_map[name] = self._graph.get_tensor_by_name( - tensor.name) + tensor.name + ) else: # only one input, all features concatenate together for name, tensor in inputs.items(): - logging.info('Load input binding: %s -> %s' % (name, tensor.name)) + logging.info( + 'Load input binding: %s -> %s' % (name, tensor.name) + ) input_info.append((0, name, tensor.dtype)) self._inputs_map[name] = self._graph.get_tensor_by_name( - tensor.name) + tensor.name + ) # sort inputs by input_ids so as to match the order of csv data input_info.sort(key=lambda t: t[0]) self._input_names = [t[1] for t in input_info] @@ -252,7 +264,8 @@ def _build_model(self): for name, tensor in outputs.items(): logging.info('Load output binding: %s -> %s' % (name, tensor.name)) self._outputs_map[name] = self._graph.get_tensor_by_name( - tensor.name) + tensor.name + ) # get assets self._assets = {} @@ -261,8 +274,9 @@ def _build_model(self): asset_file = meta_graph_pb2.AssetFileDef() any_proto.Unpack(asset_file) type_name = asset_file.tensor_info.name.split(':')[0] - asset_path = os.path.join(model_path, constants.ASSETS_DIRECTORY, - asset_file.filename) + asset_path = os.path.join( + model_path, constants.ASSETS_DIRECTORY, asset_file.filename + ) # assert gfile.Exists( # asset_path), '%s is missing in saved model' % asset_path if gfile.Exists(asset_path): @@ -316,10 +330,11 @@ def predict(self, input_data_dict, output_names=None): run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() results = self._session.run( - fetch_dict, - feed_dict, - options=run_options, - run_metadata=run_metadata) + fetch_dict, + feed_dict, + options=run_options, + run_metadata=run_metadata + ) # Create the Timeline object, and write it to a json from tensorflow.python.client import timeline tl = timeline.Timeline(run_metadata.step_stats) @@ -331,11 +346,9 @@ def predict(self, input_data_dict, output_names=None): class Predictor(PredictorInterface): - def __init__(self, - model_path, - profiling_file=None, - fg_json_path=None, - use_latest=True): + def __init__( + self, model_path, profiling_file=None, fg_json_path=None, use_latest=True + ): """Initialize a `Predictor`. Args: @@ -346,7 +359,9 @@ def __init__(self, fg_json_path: fg.json file use_latest: use latest saved_model.pb if multiple one exists. """ - self._predictor_impl = PredictorImpl(model_path, profiling_file, use_latest) + self._predictor_impl = PredictorImpl( + model_path, profiling_file, use_latest + ) self._inputs_map = self._predictor_impl._inputs_map self._outputs_map = self._predictor_impl._outputs_map self._profiling_file = profiling_file @@ -385,18 +400,21 @@ def _get_defaults(self, col_name, col_type='string'): else: defaults = {'string': '', 'double': 0.0, 'bigint': 0} assert col_type in defaults, 'invalid col_type: %s, col_type: %s' % ( - col_name, col_type) + col_name, col_type + ) default_val = defaults[col_type] logging.info( - 'col_name: %s, default_val: %s.[not defined in saved_model_dir/assets/pipeline.config]' - % (col_name, default_val)) + 'col_name: %s, default_val: %s.[not defined in saved_model_dir/assets/pipeline.config]' + % (col_name, default_val) + ) return default_val def _parse_line(self, line): pass - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, - slice_id): + def _get_dataset( + self, input_path, num_parallel_calls, batch_size, slice_num, slice_id + ): pass def _get_writer(self, output_path, slice_id): @@ -433,14 +451,14 @@ def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): pass def predict_impl( - self, - input_path, - output_path, - reserved_cols='', - output_cols=None, - batch_size=1024, - slice_id=0, - slice_num=1, + self, + input_path, + output_path, + reserved_cols='', + output_cols=None, + batch_size=1024, + slice_id=0, + slice_num=1, ): """Predict table input with loaded model. @@ -471,10 +489,12 @@ def predict_impl( with tf.Graph().as_default(), tf.Session() as sess: num_parallel_calls = 8 self._reserved_args = reserved_cols - dataset = self._get_dataset(input_path, num_parallel_calls, batch_size, - slice_num, slice_id) + dataset = self._get_dataset( + input_path, num_parallel_calls, batch_size, slice_num, slice_id + ) dataset = dataset.map( - self._parse_line, num_parallel_calls=num_parallel_calls) + self._parse_line, num_parallel_calls=num_parallel_calls + ) if hasattr(tf.data, 'make_one_shot_iterator'): iterator = tf.data.make_one_shot_iterator(dataset) else: @@ -498,9 +518,10 @@ def _parse_value(all_vals): else: assert self._all_input_names, 'must set fg_json_path when use fg input' assert fg_input_size == len(self._all_input_names), ( - 'The number of features defined in fg_json != the size of fg input. ' - 'The number of features defined in fg_json is: %d; The size of fg input is: %d' - % (len(self._all_input_names), fg_input_size)) + 'The number of features defined in fg_json != the size of fg input. ' + 'The number of features defined in fg_json is: %d; The size of fg input is: %d' + % (len(self._all_input_names), fg_input_size) + ) for i, k in enumerate(self._all_input_names): split_index.append(k) split_vals[k] = [] @@ -530,19 +551,19 @@ def _parse_value(all_vals): outputs[x] = [val[0] for val in outputs[x]] elif len(outputs[x].shape) > 1: outputs[x] = [ - json.dumps(val, cls=numpy_utils.NumpyEncoder) - for val in outputs[x] + json.dumps(val, cls=numpy_utils.NumpyEncoder) + for val in outputs[x] ] for k in self._reserved_cols: if k in all_vals and all_vals[k].dtype == np.object: all_vals[k] = [ - val.decode('utf-8', errors='ignore') for val in all_vals[k] + val.decode('utf-8', errors='ignore') for val in all_vals[k] ] ts2 = time.time() - reserve_vals = self._get_reserve_vals(self._reserved_cols, - self._output_cols, all_vals, - outputs) + reserve_vals = self._get_reserve_vals( + self._reserved_cols, self._output_cols, all_vals, outputs + ) outputs = [x for x in zip(*reserve_vals)] logging.info('predict size: %s' % len(outputs)) self._write_lines(table_writer, outputs) @@ -555,12 +576,18 @@ def _parse_value(all_vals): except self.out_of_range_exception: break if progress % 100 == 0: - logging.info('progress: batch_num=%d sample_num=%d' % - (progress, progress * batch_size)) - logging.info('time_stats: read: %.2f predict: %.2f write: %.2f' % - (sum_t0, sum_t1, sum_t2)) - logging.info('Final_time_stats: read: %.2f predict: %.2f write: %.2f' % - (sum_t0, sum_t1, sum_t2)) + logging.info( + 'progress: batch_num=%d sample_num=%d' % + (progress, progress * batch_size) + ) + logging.info( + 'time_stats: read: %.2f predict: %.2f write: %.2f' % + (sum_t0, sum_t1, sum_t2) + ) + logging.info( + 'Final_time_stats: read: %.2f predict: %.2f write: %.2f' % + (sum_t0, sum_t1, sum_t2) + ) table_writer.close() self.load_to_table(output_path, slice_num, slice_id) logging.info('Predict %s done.' % input_path) diff --git a/easy_rec/python/inference/processor/test.py b/easy_rec/python/inference/processor/test.py index 088c93edc..741f744bd 100644 --- a/easy_rec/python/inference/processor/test.py +++ b/easy_rec/python/inference/processor/test.py @@ -5,19 +5,17 @@ import glob import json import logging +import numpy as np import os import subprocess import time - -import numpy as np from google.protobuf import text_format -from easy_rec.python.protos import dataset_pb2 -from easy_rec.python.protos import pipeline_pb2 -from easy_rec.python.protos import tf_predict_pb2 +from easy_rec.python.protos import dataset_pb2, pipeline_pb2, tf_predict_pb2 logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) PROCESSOR_VERSION = 'LaRec-0.9.5d-b1b1604-TF-2.5.0-Linux' PROCESSOR_FILE = PROCESSOR_VERSION + '.tar.gz' @@ -51,18 +49,20 @@ def build_array_proto(array_proto, data, dtype): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--input_path', type=str, default=None, help='input data path') + '--input_path', type=str, default=None, help='input data path' + ) parser.add_argument( - '--output_path', type=str, default=None, help='output data path') + '--output_path', type=str, default=None, help='output data path' + ) parser.add_argument( - '--libc_path', - type=str, - default='/lib64/libc.so.6', - help='libc.so.6 path') + '--libc_path', type=str, default='/lib64/libc.so.6', help='libc.so.6 path' + ) parser.add_argument( - '--saved_model_dir', type=str, default=None, help='saved model directory') + '--saved_model_dir', type=str, default=None, help='saved model directory' + ) parser.add_argument( - '--test_dir', type=str, default=None, help='test directory') + '--test_dir', type=str, default=None, help='test directory' + ) args = parser.parse_args() if not os.path.exists('processor'): @@ -70,33 +70,39 @@ def build_array_proto(array_proto, data, dtype): if not os.path.exists(PROCESSOR_ENTRY_LIB): if not os.path.exists('processor/' + PROCESSOR_FILE): subprocess.check_output( - 'wget %s -O processor/%s' % (PROCESSOR_URL, PROCESSOR_FILE), - shell=True) + 'wget %s -O processor/%s' % (PROCESSOR_URL, PROCESSOR_FILE), + shell=True + ) subprocess.check_output( - 'cd processor && tar -zvxf %s' % PROCESSOR_FILE, shell=True) + 'cd processor && tar -zvxf %s' % PROCESSOR_FILE, shell=True + ) assert os.path.exists( - PROCESSOR_ENTRY_LIB), 'invalid processor path: %s' % PROCESSOR_ENTRY_LIB + PROCESSOR_ENTRY_LIB + ), 'invalid processor path: %s' % PROCESSOR_ENTRY_LIB assert os.path.exists(args.libc_path), '%s does not exist' % args.libc_path assert args.saved_model_dir is not None and os.path.isdir( - args.saved_model_dir + args.saved_model_dir ), '%s is not a valid directory' % args.saved_model_dir assert args.input_path is not None and os.path.exists( - args.input_path), '%s does not exist' % args.input_path + args.input_path + ), '%s does not exist' % args.input_path assert args.output_path is not None, 'output_path is not set' pipeline_config = pipeline_pb2.EasyRecConfig() - pipeline_config_path = os.path.join(args.saved_model_dir, - 'assets/pipeline.config') + pipeline_config_path = os.path.join( + args.saved_model_dir, 'assets/pipeline.config' + ) with open(pipeline_config_path) as fin: config_str = fin.read() text_format.Merge(config_str, pipeline_config) data_config = pipeline_config.data_config - input_fields = [[] - for x in data_config.input_fields - if x.input_name not in data_config.label_fields] + input_fields = [ + [] for x in data_config.input_fields + if x.input_name not in data_config.label_fields + ] with open(args.input_path, 'r') as fin: for line_str in fin: @@ -108,9 +114,10 @@ def build_array_proto(array_proto, data, dtype): req = tf_predict_pb2.PredictRequest() req.signature_name = 'serving_default' for i in range(len(input_fields)): - build_array_proto(req.inputs[data_config.input_fields[i + 1].input_name], - input_fields[i], - data_config.input_fields[i + 1].input_type) + build_array_proto( + req.inputs[data_config.input_fields[i + 1].input_name], input_fields[i], + data_config.input_fields[i + 1].input_type + ) tf_predictor = ctypes.cdll.LoadLibrary(PROCESSOR_ENTRY_LIB) tf_predictor.saved_model_init.restype = ctypes.c_void_p @@ -123,7 +130,8 @@ def build_array_proto(array_proto, data, dtype): # last_step could be greater than num_steps for sync_replicas: false train_dir = os.path.dirname(args.saved_model_dir.strip('/')) all_models = glob.glob( - os.path.join(args.test_dir, 'train/model.ckpt-*.index')) + os.path.join(args.test_dir, 'train/model.ckpt-*.index') + ) iters = [int(x.split('-')[-1].replace('.index', '')) for x in all_models] iters.sort() last_step = iters[-1] @@ -134,13 +142,15 @@ def build_array_proto(array_proto, data, dtype): start_ts = time.time() while sparse_step.value < last_step or dense_step.value < last_step: tf_predictor.saved_model_step( - ctypes.c_void_p(handle), ctypes.byref(sparse_step), - ctypes.byref(dense_step)) + ctypes.c_void_p(handle), ctypes.byref(sparse_step), + ctypes.byref(dense_step) + ) time.sleep(1) if time.time() - start_ts > 300: logging.warning( - 'could not reach last_step, sparse_step=%d dense_step=%d' % - (sparse_step.value, dense_step.value)) + 'could not reach last_step, sparse_step=%d dense_step=%d' % + (sparse_step.value, dense_step.value) + ) break data_bin = req.SerializeToString() @@ -152,8 +162,9 @@ def build_array_proto(array_proto, data, dtype): tf_predictor.saved_model_predict.restype = ctypes.c_void_p out_len = ctypes.c_int(0) res_p = tf_predictor.saved_model_predict( - ctypes.c_void_p(handle), data_bin, ctypes.c_int32(len(data_bin)), - ctypes.byref(out_len)) + ctypes.c_void_p(handle), data_bin, ctypes.c_int32(len(data_bin)), + ctypes.byref(out_len) + ) res_bytes = bytearray(ctypes.string_at(res_p, out_len)) res = tf_predict_pb2.PredictResponse() res.ParseFromString(res_bytes) diff --git a/easy_rec/python/inference/vector_retrieve.py b/easy_rec/python/inference/vector_retrieve.py index a8a8122d5..53f2368e4 100644 --- a/easy_rec/python/inference/vector_retrieve.py +++ b/easy_rec/python/inference/vector_retrieve.py @@ -1,21 +1,18 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import logging -from datetime import datetime +from __future__ import absolute_import, division, print_function import common_io +import logging import numpy as np import tensorflow as tf +from datetime import datetime try: import graphlearn as gl except: # noqa: E722 logging.warning( - 'GraphLearn is not installed. You can install it by "pip install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-0.7-cp27-cp27mu-linux_x86_64.whl.' # noqa: E501 + 'GraphLearn is not installed. You can install it by "pip install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-0.7-cp27-cp27mu-linux_x86_64.whl.' # noqa: E501 ) if tf.__version__ >= '2.0': @@ -24,18 +21,20 @@ class VectorRetrieve(object): - def __init__(self, - query_table, - doc_table, - out_table, - ndim, - delimiter=',', - batch_size=4, - index_type='ivfflat', - nlist=10, - nprobe=2, - distance=1, - m=8): + def __init__( + self, + query_table, + doc_table, + out_table, + ndim, + delimiter=',', + batch_size=4, + index_type='ivfflat', + nlist=10, + nprobe=2, + distance=1, + m=8 + ): """Retrieve top n neighbours by query vector. Args: @@ -71,15 +70,18 @@ def __init__(self, def __call__(self, top_n, task_index, task_count, *args, **kwargs): g = gl.Graph() g.node( - self.doc_table, - 'doc', - decoder=gl.Decoder( - attr_types=['float'] * self.ndim, attr_delimiter=self.delimiter), - option=self.knn_option) + self.doc_table, + 'doc', + decoder=gl.Decoder( + attr_types=['float'] * self.ndim, attr_delimiter=self.delimiter + ), + option=self.knn_option + ) g.init(task_index=task_index, task_count=task_count) query_reader = common_io.table.TableReader( - self.query_table, slice_id=task_index, slice_count=task_count) + self.query_table, slice_id=task_index, slice_count=task_count + ) num_records = query_reader.get_row_count() total_batch_num = num_records // self.batch_size + 1.0 batch_num = 0 @@ -87,24 +89,31 @@ def __call__(self, top_n, task_index, task_count, *args, **kwargs): print('total_batch_num: {}'.format(total_batch_num)) print('output_table: {}'.format(self.out_table)) - output_table_writer = common_io.table.TableWriter(self.out_table, - task_index) + output_table_writer = common_io.table.TableWriter( + self.out_table, task_index + ) count = 0 while True: try: batch_query_nodes, batch_query_feats = zip( - *query_reader.read(self.batch_size, allow_smaller_final_batch=True)) + *query_reader.read(self.batch_size, allow_smaller_final_batch=True) + ) batch_num += 1.0 - print('{} process: {:.2f}'.format(datetime.now().time(), - batch_num / total_batch_num)) + print( + '{} process: {:.2f}'.format( + datetime.now().time(), batch_num / total_batch_num + ) + ) feats = to_np_array(batch_query_feats, self.delimiter) rt_ids, rt_dists = g.search('doc', feats, gl.KnnOption(k=top_n)) - for query_node, nodes, dists in zip(batch_query_nodes, rt_ids, - rt_dists): + for query_node, nodes, dists in zip( + batch_query_nodes, rt_ids, rt_dists + ): query = np.array([query_node] * len(nodes), dtype='int64') output_table_writer.write( - zip(query, nodes, dists), (0, 1, 2), allow_type_cast=False) + zip(query, nodes, dists), (0, 1, 2), allow_type_cast=False + ) count += 1 if np.mod(count, 100) == 0: print('write ', count, ' query nodes totally') @@ -120,5 +129,6 @@ def __call__(self, top_n, task_index, task_count, *args, **kwargs): def to_np_array(batch_query_feats, attr_delimiter): return np.array( - [map(float, feat.split(attr_delimiter)) for feat in batch_query_feats], - dtype='float32') + [map(float, feat.split(attr_delimiter)) for feat in batch_query_feats], + dtype='float32' + ) diff --git a/easy_rec/python/input/batch_tfrecord_input.py b/easy_rec/python/input/batch_tfrecord_input.py index fb1981f60..5e0e27ba8 100644 --- a/easy_rec/python/input/batch_tfrecord_input.py +++ b/easy_rec/python/input/batch_tfrecord_input.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.input.input import Input @@ -19,42 +18,55 @@ class BatchTFRecordInput(Input): batch_size needs to be a multiple of n. """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(BatchTFRecordInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(BatchTFRecordInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) assert data_config.HasField( - 'n_data_batch_tfrecord'), 'Need to set n_data_batch_tfrecord in config.' + 'n_data_batch_tfrecord' + ), 'Need to set n_data_batch_tfrecord in config.' self._input_shapes = [x.input_shape for x in data_config.input_fields] self.feature_desc = {} - for x, t, d, s in zip(self._input_fields, self._input_field_types, - self._input_field_defaults, self._input_shapes): + for x, t, d, s in zip( + self._input_fields, self._input_field_types, self._input_field_defaults, + self._input_shapes + ): d = self.get_type_defaults(t, d) t = get_tf_type(t) self.feature_desc[x] = tf.io.FixedLenSequenceFeature( - dtype=t, shape=s, allow_missing=True) + dtype=t, shape=s, allow_missing=True + ) def _parse_tfrecord(self, example): try: _, features, _ = tf.parse_sequence_example( - example, sequence_features=self.feature_desc) + example, sequence_features=self.feature_desc + ) except AttributeError: _, features, _ = tf.io.parse_sequence_example( - example, sequence_features=self.feature_desc) + example, sequence_features=self.feature_desc + ) # Below code will reduce one dimension when the data dimension > 2. features = dict( - (key, - tf.reshape(value, [ - -1, - ] + [x for i, x in enumerate(value.shape) if i not in (0, 1)])) for ( - key, value) in features.items()) + ( + key, + tf.reshape( + value, [ + -1, + ] + [x for i, x in enumerate(value.shape) if i not in (0, 1)] + ) + ) for (key, value) in features.items() + ) return features def _build(self, mode, params): @@ -68,8 +80,9 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls data_compression_type = self._data_config.data_compression_type if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.shuffle: # shuffle input files @@ -78,22 +91,26 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TFRecordDataset( - x, compression_type=data_compression_type), - cycle_length=parallel_num, - num_parallel_calls=parallel_num) + lambda x: tf.data. + TFRecordDataset(x, compression_type=data_compression_type), + cycle_length=parallel_num, + num_parallel_calls=parallel_num + ) dataset = dataset.shard(self._task_num, self._task_index) if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.TFRecordDataset( - file_paths, compression_type=data_compression_type) + file_paths, compression_type=data_compression_type + ) dataset = dataset.repeat(1) # We read n data from tfrecord one time. @@ -101,17 +118,20 @@ def _build(self, mode, params): cur_batch = max(1, cur_batch) dataset = dataset.batch(cur_batch) dataset = dataset.map( - self._parse_tfrecord, num_parallel_calls=num_parallel_calls) + self._parse_tfrecord, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + map_func=self._preprocess, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/criteo_binary_reader.py b/easy_rec/python/input/criteo_binary_reader.py index 6672165c0..ba69f6195 100644 --- a/easy_rec/python/input/criteo_binary_reader.py +++ b/easy_rec/python/input/criteo_binary_reader.py @@ -5,25 +5,24 @@ import concurrent.futures import glob import logging +import numpy as np import os import queue import time -import numpy as np - class BinaryDataset: def __init__( - self, - label_bins, - dense_bins, - category_bins, - batch_size=1, - drop_last=False, - prefetch=1, - global_rank=0, - global_size=1, + self, + label_bins, + dense_bins, + category_bins, + batch_size=1, + drop_last=False, + prefetch=1, + global_rank=0, + global_size=1, ): total_sample_num = 0 self._sample_num_arr = [] @@ -36,30 +35,36 @@ def __init__( self._batch_size = batch_size - self._compute_global_start_pos(total_sample_num, batch_size, global_rank, - global_size, drop_last) + self._compute_global_start_pos( + total_sample_num, batch_size, global_rank, global_size, drop_last + ) self._label_file_arr = [None for _ in self._sample_num_arr] self._dense_file_arr = [None for _ in self._sample_num_arr] self._category_file_arr = [None for _ in self._sample_num_arr] for tmp_file_id in range(self._start_file_id, self._end_file_id + 1): - self._label_file_arr[tmp_file_id] = os.open(label_bins[tmp_file_id], - os.O_RDONLY) - self._dense_file_arr[tmp_file_id] = os.open(dense_bins[tmp_file_id], - os.O_RDONLY) - self._category_file_arr[tmp_file_id] = os.open(category_bins[tmp_file_id], - os.O_RDONLY) + self._label_file_arr[tmp_file_id] = os.open( + label_bins[tmp_file_id], os.O_RDONLY + ) + self._dense_file_arr[tmp_file_id] = os.open( + dense_bins[tmp_file_id], os.O_RDONLY + ) + self._category_file_arr[tmp_file_id] = os.open( + category_bins[tmp_file_id], os.O_RDONLY + ) self._prefetch = min(prefetch, self._num_entries) self._prefetch_queue = queue.Queue() self._executor = concurrent.futures.ThreadPoolExecutor( - max_workers=self._prefetch) + max_workers=self._prefetch + ) self._os_close_func = os.close - def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, - global_size, drop_last): + def _compute_global_start_pos( + self, total_sample_num, batch_size, global_rank, global_size, drop_last + ): # ensure all workers have the same number of samples avg_sample_num = (total_sample_num // global_size) res_num = (total_sample_num % global_size) @@ -79,8 +84,10 @@ def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, if not drop_last and (self._num_samples % batch_size != 0): self._num_entries += 1 self._last_batch_size = self._num_samples % batch_size - logging.info('num_batches = %d num_samples = %d' % - (self._num_entries, self._num_samples)) + logging.info( + 'num_batches = %d num_samples = %d' % + (self._num_entries, self._num_samples) + ) start_file_id = 0 curr_pos = 0 @@ -90,8 +97,10 @@ def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, self._start_file_id = start_file_id self._start_file_pos = global_start_pos - curr_pos - logging.info('start_file_id = %d start_file_pos = %d' % - (start_file_id, self._start_file_pos)) + logging.info( + 'start_file_id = %d start_file_pos = %d' % + (start_file_id, self._start_file_pos) + ) # find the start of each batch self._start_pos_arr = np.zeros([self._num_entries, 2], dtype=np.uint32) @@ -104,14 +113,14 @@ def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, if batch_id == self._num_entries: tmp_start_pos += self._last_batch_size while start_file_id < len( - self._sample_num_arr + self._sample_num_arr ) and tmp_start_pos > self._sample_num_arr[start_file_id]: tmp_start_pos -= self._sample_num_arr[start_file_id] start_file_id += 1 else: tmp_start_pos += batch_size while start_file_id < len( - self._sample_num_arr + self._sample_num_arr ) and tmp_start_pos >= self._sample_num_arr[start_file_id]: tmp_start_pos -= self._sample_num_arr[start_file_id] start_file_id += 1 @@ -119,8 +128,10 @@ def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, self._end_file_id = start_file_id self._end_file_pos = tmp_start_pos - logging.info('end_file_id = %d end_file_pos = %d' % - (self._end_file_id, self._end_file_pos)) + logging.info( + 'end_file_id = %d end_file_pos = %d' % + (self._end_file_id, self._end_file_pos) + ) def __del__(self): for f in self._label_file_arr: @@ -149,7 +160,8 @@ def __getitem__(self, idx): if idx < (self._num_entries - self._prefetch): self._prefetch_queue.put( - self._executor.submit(self._get, (idx + self._prefetch))) + self._executor.submit(self._get, (idx + self._prefetch)) + ) return self._prefetch_queue.get().result() @@ -164,27 +176,37 @@ def _get(self, idx): dense_read_arr = [] cate_read_arr = [] while total_read_num < self._batch_size and curr_file_id < len( - self._sample_num_arr): - tmp_read_num = min(end_read_pos, - self._sample_num_arr[curr_file_id]) - start_read_pos - - label_raw_data = os.pread(self._label_file_arr[curr_file_id], - 4 * tmp_read_num, start_read_pos * 4) - tmp_lbl_np = np.frombuffer( - label_raw_data, dtype=np.int32).reshape([tmp_read_num, 1]) + self._sample_num_arr + ): + tmp_read_num = min( + end_read_pos, self._sample_num_arr[curr_file_id] + ) - start_read_pos + + label_raw_data = os.pread( + self._label_file_arr[curr_file_id], 4 * tmp_read_num, + start_read_pos * 4 + ) + tmp_lbl_np = np.frombuffer(label_raw_data, + dtype=np.int32).reshape([tmp_read_num, 1]) label_read_arr.append(tmp_lbl_np) - dense_raw_data = os.pread(self._dense_file_arr[curr_file_id], - 52 * tmp_read_num, start_read_pos * 52) - part_dense_np = np.frombuffer( - dense_raw_data, dtype=np.float32).reshape([tmp_read_num, 13]) + dense_raw_data = os.pread( + self._dense_file_arr[curr_file_id], 52 * tmp_read_num, + start_read_pos * 52 + ) + part_dense_np = np.frombuffer(dense_raw_data, dtype=np.float32).reshape( + [tmp_read_num, 13] + ) # part_dense_np = np.log(part_dense_np + 3, dtype=np.float32) dense_read_arr.append(part_dense_np) - category_raw_data = os.pread(self._category_file_arr[curr_file_id], - 104 * tmp_read_num, start_read_pos * 104) - part_cate_np = np.frombuffer( - category_raw_data, dtype=np.uint32).reshape([tmp_read_num, 26]) + category_raw_data = os.pread( + self._category_file_arr[curr_file_id], 104 * tmp_read_num, + start_read_pos * 104 + ) + part_cate_np = np.frombuffer(category_raw_data, dtype=np.uint32).reshape( + [tmp_read_num, 26] + ) cate_read_arr.append(part_cate_np) curr_file_id += 1 @@ -211,13 +233,17 @@ def _get(self, idx): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--batch_size', type=int, default=1024, help='batch_size') parser.add_argument( - '--dataset_dir', type=str, default='./', help='dataset_dir') + '--batch_size', type=int, default=1024, help='batch_size' + ) + parser.add_argument( + '--dataset_dir', type=str, default='./', help='dataset_dir' + ) parser.add_argument('--task_num', type=int, default=1, help='task number') parser.add_argument('--task_index', type=int, default=0, help='task index') parser.add_argument( - '--prefetch_size', type=int, default=10, help='prefetch size') + '--prefetch_size', type=int, default=10, help='prefetch size' + ) args = parser.parse_args() batch_size = args.batch_size @@ -234,14 +260,14 @@ def _get(self, idx): category_files.sort() test_dataset = BinaryDataset( - label_files, - dense_files, - category_files, - batch_size=batch_size, - drop_last=False, - prefetch=args.prefetch_size, - global_rank=args.task_index, - global_size=args.task_num, + label_files, + dense_files, + category_files, + batch_size=batch_size, + drop_last=False, + prefetch=args.prefetch_size, + global_rank=args.task_index, + global_size=args.task_num, ) for step, (dense, category, labels) in enumerate(test_dataset): @@ -252,8 +278,11 @@ def _get(self, idx): start_time = time.time() if step == 1000: logging.info('1000 steps time = %.3f' % (time.time() - start_time)) - logging.info('total_steps = %d total_time = %.3f' % - (step + 1, time.time() - start_time)) logging.info( - 'final step[%d] dense_shape=%s category_shape=%s labels_shape=%s' % - (step, dense.shape, category.shape, labels.shape)) + 'total_steps = %d total_time = %.3f' % + (step + 1, time.time() - start_time) + ) + logging.info( + 'final step[%d] dense_shape=%s category_shape=%s labels_shape=%s' % + (step, dense.shape, category.shape, labels.shape) + ) diff --git a/easy_rec/python/input/criteo_input.py b/easy_rec/python/input/criteo_input.py index 0eb3ee595..9d438ad60 100644 --- a/easy_rec/python/input/criteo_input.py +++ b/easy_rec/python/input/criteo_input.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.input.criteo_binary_reader import BinaryDataset @@ -13,17 +12,20 @@ class CriteoInput(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(CriteoInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(CriteoInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) all_label_paths = [] all_dense_paths = [] all_category_paths = [] @@ -35,8 +37,8 @@ def __init__(self, (len(input_path.label_path), len(input_path.dense_path), len(input_path.category_path)) for label_path, dense_path, category_path in zip( - input_path.label_path, input_path.dense_path, - input_path.category_path): + input_path.label_path, input_path.dense_path, input_path.category_path + ): label_paths = tf.gfile.Glob(input_path.label_path) dense_paths = tf.gfile.Glob(input_path.dense_path) category_paths = tf.gfile.Glob(input_path.category_path) @@ -54,13 +56,14 @@ def __init__(self, logging.info('total number of input parts: %s' % len(all_label_paths)) self._binary_reader = BinaryDataset( - all_label_paths, - all_dense_paths, - all_category_paths, - self._batch_size, - prefetch=self._prefetch_size, - global_rank=self._task_index, - global_size=self._task_num) + all_label_paths, + all_dense_paths, + all_category_paths, + self._batch_size, + prefetch=self._prefetch_size, + global_rank=self._task_index, + global_size=self._task_num + ) else: self._binary_reader = None @@ -87,21 +90,28 @@ def _to_fea_dict(self, dense, category, labels): def _build(self, mode, params): dataset = tf.data.Dataset.from_generator( - self._sample_generator, - output_types=(tf.float32, tf.int32, tf.int32), - output_shapes=(tf.TensorShape([None, 13]), tf.TensorShape([None, 26]), - tf.TensorShape([None]))) + self._sample_generator, + output_types=(tf.float32, tf.int32, tf.int32), + output_shapes=( + tf.TensorShape([None, + 13]), tf.TensorShape([None, + 26]), tf.TensorShape([None]) + ) + ) num_parallel_calls = self._data_config.num_parallel_calls dataset = dataset.map( - self._to_fea_dict, num_parallel_calls=num_parallel_calls) + self._to_fea_dict, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + map_func=self._preprocess, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/csv_input.py b/easy_rec/python/input/csv_input.py index b3afd1656..4c130d67d 100644 --- a/easy_rec/python/input/csv_input.py +++ b/easy_rec/python/input/csv_input.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.input.input import Input @@ -16,24 +15,27 @@ class CSVInput(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(CSVInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(CSVInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) self._with_header = data_config.with_header self._field_names = None def _parse_csv(self, line): record_defaults = [ - self.get_type_defaults(t, v) - for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) + for t, v in zip(self._input_field_types, self._input_field_defaults) ] if self._field_names: @@ -43,28 +45,32 @@ def _parse_csv(self, line): if field_name in self._input_fields: tid = self._input_fields.index(field_name) record_defaults.append( - self.get_type_defaults(self._input_field_types[tid], - self._input_field_defaults[tid])) + self.get_type_defaults( + self._input_field_types[tid], self._input_field_defaults[tid] + ) + ) else: record_defaults.append('') check_list = [ - tf.py_func( - check_split, [ - line, self._data_config.separator, - len(record_defaults), self._check_mode - ], - Tout=tf.bool) + tf.py_func( + check_split, [ + line, self._data_config.separator, + len(record_defaults), self._check_mode + ], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): fields = tf.decode_csv( - line, - field_delim=self._data_config.separator, - record_defaults=record_defaults, - name='decode_csv') + line, + field_delim=self._data_config.separator, + record_defaults=record_defaults, + name='decode_csv' + ) if self._field_names is not None: fields = [ - fields[self._field_names.index(x)] for x in self._input_fields + fields[self._field_names.index(x)] for x in self._input_fields ] # filter only valid fields @@ -86,7 +92,8 @@ def _build(self, mode, params): assert len(file_paths) > 0, 'match no files with %s' % self._input_path assert not file_paths[0].endswith( - '.tar.gz'), 'could only support .csv or .gz(not .tar.gz) files.' + '.tar.gz' + ), 'could only support .csv or .gz(not .tar.gz) files.' compression_type = 'GZIP' if file_paths[0].endswith('.gz') else '' if compression_type: @@ -102,8 +109,9 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -117,59 +125,67 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset( - x, compression_type=compression_type).skip( - int(self._with_header)), - cycle_length=parallel_num, - num_parallel_calls=parallel_num) + lambda x: tf.data.TextLineDataset( + x, compression_type=compression_type + ).skip(int(self._with_header)), + cycle_length=parallel_num, + num_parallel_calls=parallel_num + ) if not self._data_config.file_shard: dataset = self._safe_shard(dataset) if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) elif self._task_num > 1: # For distribute evaluate dataset = tf.data.Dataset.from_tensor_slices(file_paths) parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset( - x, compression_type=compression_type).skip( - int(self._with_header)), - cycle_length=parallel_num, - num_parallel_calls=parallel_num) + lambda x: tf.data.TextLineDataset( + x, compression_type=compression_type + ).skip(int(self._with_header)), + cycle_length=parallel_num, + num_parallel_calls=parallel_num + ) dataset = self._safe_shard(dataset) dataset = dataset.repeat(1) else: - logging.info('eval files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.Dataset.from_tensor_slices(file_paths) parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset( - x, compression_type=compression_type).skip( - int(self._with_header)), - cycle_length=parallel_num, - num_parallel_calls=parallel_num) + lambda x: tf.data.TextLineDataset( + x, compression_type=compression_type + ).skip(int(self._with_header)), + cycle_length=parallel_num, + num_parallel_calls=parallel_num + ) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) dataset = dataset.map( - self._parse_csv, num_parallel_calls=num_parallel_calls) + self._parse_csv, num_parallel_calls=num_parallel_calls + ) if self._data_config.ignore_error: dataset = dataset.apply(ignore_errors) dataset = dataset.prefetch(buffer_size=self._prefetch_size) dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + map_func=self._preprocess, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/csv_input_ex.py b/easy_rec/python/input/csv_input_ex.py index d3b506fce..8dd09fffa 100644 --- a/easy_rec/python/input/csv_input_ex.py +++ b/easy_rec/python/input/csv_input_ex.py @@ -12,22 +12,25 @@ class CSVInputEx(CSVInput): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(CSVInputEx, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(CSVInputEx, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) def _parse_csv(self, line): record_defaults = [ - self.get_type_defaults(t, v) - for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) + for t, v in zip(self._input_field_types, self._input_field_defaults) ] def _check_data(line): @@ -41,31 +44,38 @@ def _check_data(line): return True fields = str_split_by_chr( - line, self._data_config.separator, skip_empty=False) + line, self._data_config.separator, skip_empty=False + ) tmp_fields = tf.reshape(fields.values, [-1, len(record_defaults)]) fields = [] for i in range(len(record_defaults)): if type(record_defaults[i]) == int: fields.append( - tf.string_to_number( - tmp_fields[:, i], tf.int64, name='field_as_int_%d' % i)) + tf.string_to_number( + tmp_fields[:, i], tf.int64, name='field_as_int_%d' % i + ) + ) elif type(record_defaults[i]) in [float, np.float32, np.float64]: fields.append( - tf.string_to_number( - tmp_fields[:, i], tf.float32, name='field_as_flt_%d' % i)) + tf.string_to_number( + tmp_fields[:, i], tf.float32, name='field_as_flt_%d' % i + ) + ) elif type(record_defaults[i]) in [str, type(u''), bytes]: fields.append(tmp_fields[:, i]) elif type(record_defaults[i]) == bool: fields.append( - tf.logical_or( - tf.equal(tmp_fields[:, i], 'True'), - tf.equal(tmp_fields[:, i], 'true'))) + tf.logical_or( + tf.equal(tmp_fields[:, i], 'True'), + tf.equal(tmp_fields[:, i], 'true') + ) + ) else: assert 'invalid types: %s' % str(type(record_defaults[i])) keep_ids = [ - self._input_fields.index(x) - for x in self._label_fields + self._effective_fields + self._input_fields.index(x) + for x in self._label_fields + self._effective_fields ] inputs = {self._input_fields[x]: fields[x] for x in keep_ids} diff --git a/easy_rec/python/input/csv_input_v2.py b/easy_rec/python/input/csv_input_v2.py index deddc2f06..bd27df3d0 100644 --- a/easy_rec/python/input/csv_input_v2.py +++ b/easy_rec/python/input/csv_input_v2.py @@ -7,23 +7,27 @@ class CSVInputV2(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(CSVInputV2, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(CSVInputV2, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) def _build(self, mode, params): if type(self._input_path) != list: self._input_path = self._input_path.split(',') assert len( - self._input_path) > 0, 'match no files with %s' % self._input_path + self._input_path + ) > 0, 'match no files with %s' % self._input_path if self._input_path[0].startswith('hdfs://'): # support hdfs input @@ -32,25 +36,27 @@ def _build(self, mode, params): num_epochs = self.num_epochs if mode == tf.estimator.ModeKeys.TRAIN else 1 is_train = (mode == tf.estimator.ModeKeys.TRAIN) record_defaults = [ - self.get_type_defaults(x, v) - for x, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(x, v) + for x, v in zip(self._input_field_types, self._input_field_defaults) ] dataset = tf.data.experimental.make_csv_dataset( - self._input_path, - self._data_config.batch_size, - column_names=self._input_fields, - field_delim=self._data_config.separator, - column_defaults=record_defaults, - header=False, - num_epochs=num_epochs, - shuffle=is_train and self._data_config.shuffle, - num_parallel_reads=8, - sloppy=is_train) + self._input_path, + self._data_config.batch_size, + column_names=self._input_fields, + field_delim=self._data_config.separator, + column_defaults=record_defaults, + header=False, + num_epochs=num_epochs, + shuffle=is_train and self._data_config.shuffle, + num_parallel_reads=8, + sloppy=is_train + ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.chief_redundant: dataset = dataset.shard( - max(self._task_num - 1, 1), max(self._task_index - 1, 0)) + max(self._task_num - 1, 1), max(self._task_index - 1, 0) + ) else: dataset = dataset.shard(self._task_num, self._task_index) else: @@ -61,8 +67,9 @@ def _build(self, mode, params): dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/datahub_input.py b/easy_rec/python/input/datahub_input.py index 37e3292d4..e9a4111a2 100644 --- a/easy_rec/python/input/datahub_input.py +++ b/easy_rec/python/input/datahub_input.py @@ -2,9 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import json import logging -import traceback - import tensorflow as tf +import traceback from tensorflow.python.framework import dtypes from easy_rec.python.input.input import Input @@ -22,37 +21,42 @@ common_io = None try: + import urllib3 from datahub import DataHub from datahub.exceptions import DatahubException - from datahub.models import RecordType - from datahub.models import CursorType - import urllib3 + from datahub.models import CursorType, RecordType urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) logging.getLogger('datahub.account').setLevel(logging.INFO) except Exception: logging.warning( - 'DataHub is not installed[%s]. You can install it by: pip install pydatahub' - % traceback.format_exc()) + 'DataHub is not installed[%s]. You can install it by: pip install pydatahub' + % traceback.format_exc() + ) DataHub = None class DataHubInput(Input): """DataHubInput is used for online train.""" - def __init__(self, - data_config, - feature_config, - datahub_config, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(DataHubInput, - self).__init__(data_config, feature_config, '', task_index, task_num, - check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + datahub_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(DataHubInput, self).__init__( + data_config, feature_config, '', task_index, task_num, check_mode, + pipeline_config + ) if DataHub is None: - logging.error('please install datahub: ', - 'pip install pydatahub ;Python 3.6 recommended') + logging.error( + 'please install datahub: ', + 'pip install pydatahub ;Python 3.6 recommended' + ) try: self._num_epoch = 0 self._datahub_config = datahub_config @@ -72,12 +76,13 @@ def __init__(self, pass self._offset_dict = {} if datahub_config: - shard_result = self._datahub.list_shard(self._datahub_config.project, - self._datahub_config.topic) + shard_result = self._datahub.list_shard( + self._datahub_config.project, self._datahub_config.topic + ) shards = shard_result.shards self._all_shards = shards self._shards = [ - shards[i] for i in range(len(shards)) if (i % task_num) == task_index + shards[i] for i in range(len(shards)) if (i % task_num) == task_index ] logging.info('all shards: %s' % str(self._shards)) @@ -86,10 +91,10 @@ def __init__(self, ts = parse_time(datahub_config.offset_time) * 1000 for x in self._all_shards: ks = str(x.shard_id) - cursor_result = self._datahub.get_cursor(self._datahub_config.project, - self._datahub_config.topic, - ks, CursorType.SYSTEM_TIME, - ts) + cursor_result = self._datahub.get_cursor( + self._datahub_config.project, self._datahub_config.topic, ks, + CursorType.SYSTEM_TIME, ts + ) logging.info('shard[%s] cursor = %s' % (ks, cursor_result)) self._offset_dict[ks] = cursor_result.cursor elif offset_type == 'offset_info': @@ -100,21 +105,23 @@ def __init__(self, self._dh_field_names = [] self._dh_field_types = [] topic_info = self._datahub.get_topic( - project_name=self._datahub_config.project, - topic_name=self._datahub_config.topic) + project_name=self._datahub_config.project, + topic_name=self._datahub_config.topic + ) for field in topic_info.record_schema.field_list: self._dh_field_names.append(field.name) self._dh_field_types.append(field.type.value) assert len( - self._feature_fields) > 0, 'data_config.feature_fields are not set.' + self._feature_fields + ) > 0, 'data_config.feature_fields are not set.' for x in self._feature_fields: assert x in self._dh_field_names, 'feature_field[%s] is not in datahub' % x # feature column ids in datahub schema self._dh_fea_ids = [ - self._dh_field_names.index(x) for x in self._feature_fields + self._dh_field_names.index(x) for x in self._feature_fields ] for x in self._label_fields: @@ -128,12 +135,12 @@ def __init__(self, if len(self._dh_fea_ids) > 1: self._filter_fea_func = lambda record: ''.join( - [record.values[x] - for x in self._dh_fea_ids]).split(chr(2))[1] == '-1024' + [record.values[x] for x in self._dh_fea_ids] + ).split(chr(2))[1] == '-1024' else: dh_fea_id = self._dh_fea_ids[0] - self._filter_fea_func = lambda record: record.values[dh_fea_id].split( - self._data_config.separator)[1] == '-1024' + self._filter_fea_func = lambda record: record.values[ + dh_fea_id].split(self._data_config.separator)[1] == '-1024' def _parse_record(self, *fields): field_dict = {} @@ -141,13 +148,13 @@ def _parse_record(self, *fields): def _dump_offsets(): all_offsets = { - x.shard_id: self._offset_dict[x.shard_id] - for x in self._shards - if x.shard_id in self._offset_dict + x.shard_id: self._offset_dict[x.shard_id] + for x in self._shards if x.shard_id in self._offset_dict } return json.dumps(all_offsets) - field_dict[Input.DATA_OFFSET] = tf.py_func(_dump_offsets, [], dtypes.string) + field_dict[Input.DATA_OFFSET + ] = tf.py_func(_dump_offsets, [], dtypes.string) for x in self._label_fields: dh_id = self._dh_field_names.index(x) @@ -156,20 +163,21 @@ def _dump_offsets(): feature_inputs = self.get_feature_input_fields() # only for features, labels and sample_weight excluded record_types = [ - t for x, t in zip(self._input_fields, self._input_field_types) - if x in feature_inputs + t for x, t in zip(self._input_fields, self._input_field_types) + if x in feature_inputs ] feature_num = len(record_types) feature_fields = [ - fields[self._dh_field_names.index(x)] for x in self._feature_fields + fields[self._dh_field_names.index(x)] for x in self._feature_fields ] feature = feature_fields[0] for fea_id in range(1, len(feature_fields)): feature = feature + self._data_config.separator + feature_fields[fea_id] feature = tf.string_split( - feature, self._data_config.separator, skip_empty=False) + feature, self._data_config.separator, skip_empty=False + ) fields = tf.reshape(feature.values, [-1, feature_num]) @@ -227,13 +235,16 @@ def _datahub_generator(self): self._num_epoch += 1 try: - self._datahub.wait_shards_ready(self._datahub_config.project, - self._datahub_config.topic) - topic_result = self._datahub.get_topic(self._datahub_config.project, - self._datahub_config.topic) + self._datahub.wait_shards_ready( + self._datahub_config.project, self._datahub_config.topic + ) + topic_result = self._datahub.get_topic( + self._datahub_config.project, self._datahub_config.topic + ) if topic_result.record_type != RecordType.TUPLE: - logging.error('datahub topic type(%s) illegal' % - str(topic_result.record_type)) + logging.error( + 'datahub topic type(%s) illegal' % str(topic_result.record_type) + ) record_schema = topic_result.record_schema tid = 0 @@ -244,32 +255,36 @@ def _datahub_generator(self): tid = 0 if shard_id not in self._offset_dict: - cursor_result = self._datahub.get_cursor(self._datahub_config.project, - self._datahub_config.topic, - shard_id, CursorType.OLDEST) + cursor_result = self._datahub.get_cursor( + self._datahub_config.project, self._datahub_config.topic, shard_id, + CursorType.OLDEST + ) cursor = cursor_result.cursor else: cursor = self._offset_dict[shard_id] get_result = self._datahub.get_tuple_records( - self._datahub_config.project, self._datahub_config.topic, shard_id, - record_schema, cursor, self._read_cnt) + self._datahub_config.project, self._datahub_config.topic, shard_id, + record_schema, cursor, self._read_cnt + ) count = get_result.record_count if count == 0: continue for row_id, record in enumerate(get_result.records): if self._is_data_empty(record): - logging.warning('skip empty data record: %s' % - self._dump_record(record)) + logging.warning( + 'skip empty data record: %s' % self._dump_record(record) + ) continue if self._filter_fea_func is not None: if self._filter_fea_func(record): - logging.warning('filter data record: %s' % - self._dump_record(record)) + logging.warning( + 'filter data record: %s' % self._dump_record(record) + ) continue yield tuple(list(record.values)) if shard_id not in self._offset_dict or get_result.next_cursor > self._offset_dict[ - shard_id]: + shard_id]: self._offset_dict[shard_id] = get_result.next_cursor except DatahubException as ex: logging.error('DatahubException: %s' % str(ex)) @@ -282,39 +297,44 @@ def _build(self, mode, params): # get input types list_types = [ - odps_util.odps_type_2_tf_type(x) for x in self._dh_field_types + odps_util.odps_type_2_tf_type(x) for x in self._dh_field_types ] list_types = tuple(list_types) list_shapes = [ - tf.TensorShape([]) for x in range(0, len(self._dh_field_types)) + tf.TensorShape([]) for x in range(0, len(self._dh_field_types)) ] list_shapes = tuple(list_shapes) # read datahub dataset = tf.data.Dataset.from_generator( - self._datahub_generator, - output_types=list_types, - output_shapes=list_shapes) + self._datahub_generator, + output_types=list_types, + output_shapes=list_shapes + ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.batch(self._data_config.batch_size) dataset = dataset.map( - self._parse_record, - num_parallel_calls=self._data_config.num_parallel_calls) + self._parse_record, + num_parallel_calls=self._data_config.num_parallel_calls + ) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/dummy_input.py b/easy_rec/python/input/dummy_input.py index f556a3686..d12524fc0 100644 --- a/easy_rec/python/input/dummy_input.py +++ b/easy_rec/python/input/dummy_input.py @@ -16,18 +16,21 @@ class DummyInput(Input): Dummy Input is used to debug the performance bottleneck of data pipeline. """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - input_vals={}): - super(DummyInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + input_vals={} + ): + super(DummyInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) self._input_vals = input_vals def _build(self, mode, params): @@ -42,9 +45,9 @@ def _build(self, mode, params): label tensor dict """ features = {} - for field, field_type, def_val in zip(self._input_fields, - self._input_field_types, - self._input_field_defaults): + for field, field_type, def_val in zip( + self._input_fields, self._input_field_types, self._input_field_defaults + ): tf_type = get_tf_type(field_type) def_val = self.get_type_defaults(field_type, default_val=def_val) diff --git a/easy_rec/python/input/hive_input.py b/easy_rec/python/input/hive_input.py index f5c8735af..5a462de05 100644 --- a/easy_rec/python/input/hive_input.py +++ b/easy_rec/python/input/hive_input.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import logging import os - import tensorflow as tf from easy_rec.python.input.input import Input @@ -11,17 +10,20 @@ class HiveInput(Input): """Common IO based interface, could run at local or on data science.""" - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(HiveInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(HiveInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) if input_path is None: return self._data_config = data_config @@ -29,11 +31,14 @@ def __init__(self, self._hive_config = input_path hive_util = HiveUtils( - data_config=self._data_config, hive_config=self._hive_config) + data_config=self._data_config, hive_config=self._hive_config + ) self._input_hdfs_path = hive_util.get_table_location( - self._hive_config.table_name) + self._hive_config.table_name + ) self._input_table_col_names, self._input_table_col_types = hive_util.get_all_cols( - self._hive_config.table_name) + self._hive_config.table_name + ) def _parse_csv(self, line): record_defaults = [] @@ -41,21 +46,25 @@ def _parse_csv(self, line): if field_name in self._input_fields: tid = self._input_fields.index(field_name) record_defaults.append( - self.get_type_defaults(self._input_field_types[tid], - self._input_field_defaults[tid])) + self.get_type_defaults( + self._input_field_types[tid], self._input_field_defaults[tid] + ) + ) else: record_defaults.append('') tmp_fields = tf.decode_csv( - line, - field_delim=self._data_config.separator, - record_defaults=record_defaults, - name='decode_csv') + line, + field_delim=self._data_config.separator, + record_defaults=record_defaults, + name='decode_csv' + ) fields = [] for x in self._input_fields: assert x in self._input_table_col_names, 'Column %s not in Table %s.' % ( - x, self._hive_config.table_name) + x, self._hive_config.table_name + ) fields.append(tmp_fields[self._input_table_col_names.index(x)]) # filter only valid fields @@ -67,12 +76,14 @@ def _parse_csv(self, line): def _build(self, mode, params): file_paths = tf.gfile.Glob(os.path.join(self._input_hdfs_path, '*')) assert len( - file_paths) > 0, 'match no files with %s' % self._hive_config.table_name + file_paths + ) > 0, 'match no files with %s' % self._hive_config.table_name num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -86,38 +97,44 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset(x), - cycle_length=parallel_num, - num_parallel_calls=parallel_num) + lambda x: tf.data.TextLineDataset(x), + cycle_length=parallel_num, + num_parallel_calls=parallel_num + ) if not self._data_config.file_shard: dataset = self._safe_shard(dataset) if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) dataset = dataset.map( - self._parse_csv, num_parallel_calls=num_parallel_calls) + self._parse_csv, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + map_func=self._preprocess, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/hive_parquet_input.py b/easy_rec/python/input/hive_parquet_input.py index 7bb42dc42..07847bbf0 100644 --- a/easy_rec/python/input/hive_parquet_input.py +++ b/easy_rec/python/input/hive_parquet_input.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import logging -import os - import numpy as np +import os import pandas as pd import tensorflow as tf @@ -14,17 +13,20 @@ class HiveParquetInput(Input): """Common IO based interface, could run at local or on data science.""" - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(HiveParquetInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(HiveParquetInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) if input_path is None: return self._data_config = data_config @@ -32,19 +34,24 @@ def __init__(self, self._hive_config = input_path hive_util = HiveUtils( - data_config=self._data_config, hive_config=self._hive_config) - input_hdfs_path = hive_util.get_table_location(self._hive_config.table_name) + data_config=self._data_config, hive_config=self._hive_config + ) + input_hdfs_path = hive_util.get_table_location( + self._hive_config.table_name + ) self._input_table_col_names, self._input_table_col_types = hive_util.get_all_cols( - self._hive_config.table_name) + self._hive_config.table_name + ) self._all_hdfs_path = tf.gfile.Glob(os.path.join(input_hdfs_path, '*')) for x in self._input_fields: assert x in self._input_table_col_names, 'Column %s not in Table %s.' % ( - x, self._hive_config.table_name) + x, self._hive_config.table_name + ) self._record_defaults = [ - self.get_type_defaults(t, v) - for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) + for t, v in zip(self._input_field_types, self._input_field_defaults) ] def _file_shard(self, file_paths, task_num, task_index): @@ -69,10 +76,12 @@ def _parquet_read(self): for k, v in zip(self._input_fields, self._record_defaults): df[k].fillna(v, inplace=True) - for start_idx in range(0, total_records_num, - self._data_config.batch_size): - end_idx = min(total_records_num, - start_idx + self._data_config.batch_size) + for start_idx in range( + 0, total_records_num, self._data_config.batch_size + ): + end_idx = min( + total_records_num, start_idx + self._data_config.batch_size + ) batch_data = df[start_idx:end_idx] inputs = [] for k in self._input_fields: @@ -96,45 +105,51 @@ def _build(self, mode, params): if len(self._all_hdfs_path) >= 2 * self._task_num: file_shard = True - self._input_hdfs_path = self._file_shard(self._all_hdfs_path, - self._task_num, self._task_index) + self._input_hdfs_path = self._file_shard( + self._all_hdfs_path, self._task_num, self._task_index + ) else: file_shard = False self._input_hdfs_path = self._all_hdfs_path logging.info('input path: %s' % self._input_hdfs_path) - assert len(self._input_hdfs_path - ) > 0, 'match no files with %s' % self._hive_config.table_name + assert len( + self._input_hdfs_path + ) > 0, 'match no files with %s' % self._hive_config.table_name dataset = tf.data.Dataset.from_generator( - self._parquet_read, output_types=list_type, output_shapes=list_shapes) + self._parquet_read, output_types=list_type, output_shapes=list_shapes + ) if not file_shard: dataset = self._safe_shard(dataset) if mode == tf.estimator.ModeKeys.TRAIN: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) dataset = dataset.map( - self._parse_csv, - num_parallel_calls=self._data_config.num_parallel_calls) + self._parse_csv, num_parallel_calls=self._data_config.num_parallel_calls + ) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/hive_rtp_input.py b/easy_rec/python/input/hive_rtp_input.py index b7bbf2148..df71444c5 100644 --- a/easy_rec/python/input/hive_rtp_input.py +++ b/easy_rec/python/input/hive_rtp_input.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import logging import os - import tensorflow as tf from easy_rec.python.input.input import Input @@ -13,25 +12,30 @@ class HiveRTPInput(Input): """Common IO based interface, could run at local or on data science.""" - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(HiveRTPInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(HiveRTPInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) if input_path is None: return self._data_config = data_config self._feature_config = feature_config self._hive_config = input_path - logging.info('input_fields: %s label_fields: %s' % - (','.join(self._input_fields), ','.join(self._label_fields))) + logging.info( + 'input_fields: %s label_fields: %s' % + (','.join(self._input_fields), ','.join(self._label_fields)) + ) self._rtp_separator = self._data_config.rtp_separator if not isinstance(self._rtp_separator, str): @@ -41,11 +45,14 @@ def __init__(self, if self._data_config.selected_cols else None logging.info('select cols: %s' % self._selected_cols) hive_util = HiveUtils( - data_config=self._data_config, hive_config=self._hive_config) + data_config=self._data_config, hive_config=self._hive_config + ) self._input_hdfs_path = hive_util.get_table_location( - self._hive_config.table_name) + self._hive_config.table_name + ) self._input_table_col_names, self._input_table_col_types = hive_util.get_all_cols( - self._hive_config.table_name) + self._hive_config.table_name + ) def _parse_csv(self, line): non_feature_cols = self._label_fields @@ -56,16 +63,19 @@ def _parse_csv(self, line): if field_name in self._selected_cols[:-1]: idx = self._input_fields.index(field_name) record_defaults.append( - self.get_type_defaults(self._input_field_types[idx], - self._input_field_defaults[idx])) + self.get_type_defaults( + self._input_field_types[idx], self._input_field_defaults[idx] + ) + ) else: record_defaults.append('') print('record_defaults: ', record_defaults) tmp_fields = tf.decode_csv( - line, - field_delim=self._rtp_separator, - record_defaults=record_defaults, - name='decode_csv') + line, + field_delim=self._rtp_separator, + record_defaults=record_defaults, + name='decode_csv' + ) print('tmp_fields: ', tmp_fields) fields = [] @@ -78,33 +88,35 @@ def _parse_csv(self, line): # only for features, labels and sample_weight excluded record_types = [ - t for x, t in zip(self._input_fields, self._input_field_types) - if x not in non_feature_cols + t for x, t in zip(self._input_fields, self._input_field_types) + if x not in non_feature_cols ] feature_num = len(record_types) check_list = [ - tf.py_func( - check_split, - [fields[-1], self._data_config.separator, - len(record_types)], - Tout=tf.bool) + tf.py_func( + check_split, + [fields[-1], self._data_config.separator, + len(record_types)], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): fields = tf.string_split( - fields[-1], self._data_config.separator, skip_empty=False) + fields[-1], self._data_config.separator, skip_empty=False + ) tmp_fields = tf.reshape(fields.values, [-1, feature_num]) rtp_record_defaults = [ - str(self.get_type_defaults(t, v)) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x not in non_feature_cols + str(self.get_type_defaults(t, v)) for x, t, v in zip( + self._input_fields, self._input_field_types, self._input_field_defaults + ) if x not in non_feature_cols ] fields = labels[len(self._label_fields):] for i in range(feature_num): - field = string_to_number(tmp_fields[:, i], record_types[i], - rtp_record_defaults[i], i) + field = string_to_number( + tmp_fields[:, i], record_types[i], rtp_record_defaults[i], i + ) fields.append(field) field_keys = [x for x in self._input_fields if x not in self._label_fields] @@ -118,12 +130,14 @@ def _parse_csv(self, line): def _build(self, mode, params): file_paths = tf.gfile.Glob(os.path.join(self._input_hdfs_path, '*')) assert len( - file_paths) > 0, 'match no files with %s' % self._hive_config.table_name + file_paths + ) > 0, 'match no files with %s' % self._hive_config.table_name num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -137,38 +151,44 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset(x), - cycle_length=parallel_num, - num_parallel_calls=parallel_num) + lambda x: tf.data.TextLineDataset(x), + cycle_length=parallel_num, + num_parallel_calls=parallel_num + ) if not self._data_config.file_shard: dataset = self._safe_shard(dataset) if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) dataset = dataset.map( - self._parse_csv, num_parallel_calls=num_parallel_calls) + self._parse_csv, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + map_func=self._preprocess, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/input.py b/easy_rec/python/input/input.py index f53c4ee45..2aeff4ed6 100644 --- a/easy_rec/python/input/input.py +++ b/easy_rec/python/input/input.py @@ -2,50 +2,47 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os -from abc import abstractmethod -from collections import OrderedDict - import six import tensorflow as tf +from abc import abstractmethod +from collections import OrderedDict from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import string_ops +from tensorflow.python.ops import array_ops, sparse_ops, string_ops from tensorflow.python.platform import gfile from easy_rec.python.core import sampler as sampler_lib from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.utils import conditional -from easy_rec.python.utils import config_util -from easy_rec.python.utils import constant -from easy_rec.python.utils.check_utils import check_split -from easy_rec.python.utils.check_utils import check_string_to_number +from easy_rec.python.utils import conditional, config_util, constant +from easy_rec.python.utils.check_utils import check_split, check_string_to_number # NOQA from easy_rec.python.utils.expr_util import get_expression from easy_rec.python.utils.input_utils import get_type_defaults -from easy_rec.python.utils.load_class import get_register_class_meta -from easy_rec.python.utils.load_class import load_by_path +from easy_rec.python.utils.load_class import get_register_class_meta, load_by_path # NOQA from easy_rec.python.utils.tf_utils import get_tf_type if tf.__version__ >= '2.0': tf = tf.compat.v1 _INPUT_CLASS_MAP = {} -_meta_type = get_register_class_meta(_INPUT_CLASS_MAP, have_abstract_class=True) +_meta_type = get_register_class_meta( + _INPUT_CLASS_MAP, have_abstract_class=True +) class Input(six.with_metaclass(_meta_type, object)): DATA_OFFSET = 'DATA_OFFSET' - def __init__(self, - data_config, - feature_configs, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs): + def __init__( + self, + data_config, + feature_configs, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + **kwargs + ): self._pipeline_config = pipeline_config self._data_config = data_config self._check_mode = check_mode @@ -54,7 +51,8 @@ def __init__(self, # calling self._build self._mode = None if pipeline_config is not None and pipeline_config.model_config.HasField( - 'ev_params'): + 'ev_params' + ): self._has_ev = True else: self._has_ev = False @@ -75,7 +73,7 @@ def __init__(self, self._input_dims = [x.input_dim for x in data_config.input_fields] self._input_field_types = [x.input_type for x in data_config.input_fields] self._input_field_defaults = [ - x.default_val for x in data_config.input_fields + x.default_val for x in data_config.input_fields ] self._label_fields = list(data_config.label_fields) self._feature_fields = list(data_config.feature_fields) @@ -111,13 +109,14 @@ def __init__(self, for fc in self._feature_configs: for input_name in fc.input_names: assert input_name in self._input_fields, 'invalid input_name in %s' % str( - fc) + fc + ) if input_name not in self._effective_fields: self._effective_fields.append(input_name) if fc.feature_type in [fc.TagFeature, fc.SequenceFeature]: - if fc.hash_bucket_size > 0 or len( - fc.vocab_list) > 0 or fc.HasField('vocab_file'): + if fc.hash_bucket_size > 0 or len(fc.vocab_list + ) > 0 or fc.HasField('vocab_file'): self._multi_value_types[fc.input_names[0]] = tf.string self._multi_value_fields.add(fc.input_names[0]) else: @@ -132,8 +131,8 @@ def __init__(self, self._multi_value_fields.add(fc.input_names[0]) if fc.HasField('normalizer_fn'): - feature_name = fc.feature_name if fc.HasField( - 'feature_name') else fc.input_names[0] + feature_name = fc.feature_name if fc.HasField('feature_name' + ) else fc.input_names[0] self._normalizer_fn[feature_name] = load_by_path(fc.normalizer_fn) # add sample weight to effective fields @@ -159,8 +158,9 @@ def __init__(self, model_name = model_config.WhichOneof('model') if model_name in {'mmoe', 'esmm', 'dbmtl', 'simple_multi_task', 'ple'}: model = getattr(model_config, model_name) - towers = [model.ctr_tower, model.cvr_tower - ] if model_name == 'esmm' else model.task_towers + towers = [ + model.ctr_tower, model.cvr_tower + ] if model_name == 'esmm' else model.task_towers for tower in towers: metrics = tower.metrics_set for metric in metrics: @@ -175,15 +175,17 @@ def __init__(self, self._effective_fields.append(sid) self._effective_fids = [ - self._input_fields.index(x) for x in self._effective_fields + self._input_fields.index(x) for x in self._effective_fields ] # sort fids from small to large self._effective_fids = list(set(self._effective_fids)) self._effective_fields = [ - self._input_fields[x] for x in self._effective_fids + self._input_fields[x] for x in self._effective_fids ] - self._label_fids = [self._input_fields.index(x) for x in self._label_fields] + self._label_fids = [ + self._input_fields.index(x) for x in self._label_fields + ] # virtual fields generated by self._preprocess # which will be inputs to feature columns @@ -200,9 +202,11 @@ def __init__(self, def _load_label_fn(self, config): udf_class = config.user_define_fn udf_path = config.user_define_fn_path if config.HasField( - 'user_define_fn_path') else None + 'user_define_fn_path' + ) else None dtype = config.user_define_fn_res_type if config.HasField( - 'user_define_fn_res_type') else None + 'user_define_fn_res_type' + ) else None if udf_path: if udf_path.startswith('oss://') or udf_path.startswith('hdfs://'): @@ -232,8 +236,8 @@ def num_epochs(self): def get_feature_input_fields(self): return [ - x for x in self._input_fields - if x not in self._label_fields and x != self._data_config.sample_weight + x for x in self._input_fields + if x not in self._label_fields and x != self._data_config.sample_weight ] def should_stop(self, curr_epoch): @@ -267,9 +271,9 @@ def create_multi_placeholders(self, export_config): effective_fids = list(self._effective_fids) else: effective_fids = [ - fid for fid in range(len(self._input_fields)) - if self._input_fields[fid] not in self._label_fields and - self._input_fields[fid] != sample_weight_field + fid for fid in range(len(self._input_fields)) + if self._input_fields[fid] not in self._label_fields + and self._input_fields[fid] != sample_weight_field ] inputs = {} @@ -284,10 +288,12 @@ def create_multi_placeholders(self, export_config): if input_name in export_fields_name: tf_type = self._multi_value_types[input_name] if input_name in self._multi_value_types \ else get_tf_type(self._input_field_types[fid]) - logging.info('multi value input_name: %s, dtype: %s' % - (input_name, tf_type)) + logging.info( + 'multi value input_name: %s, dtype: %s' % (input_name, tf_type) + ) finput = array_ops.placeholder( - tf_type, [None, None], name=placeholder_name) + tf_type, [None, None], name=placeholder_name + ) else: ftype = self._input_field_types[fid] tf_type = get_tf_type(ftype) @@ -301,10 +307,11 @@ def create_multi_placeholders(self, export_config): def create_placeholders(self, export_config): self._mode = tf.estimator.ModeKeys.PREDICT inputs_placeholder = array_ops.placeholder( - tf.string, [None], name='features') + tf.string, [None], name='features' + ) input_vals = tf.string_split( - inputs_placeholder, self._data_config.separator, - skip_empty=False).values + inputs_placeholder, self._data_config.separator, skip_empty=False + ).values sample_weight_field = '' if self._data_config.HasField('sample_weight'): @@ -312,19 +319,23 @@ def create_placeholders(self, export_config): if export_config.filter_inputs: effective_fids = list(self._effective_fids) - logging.info('number of effective inputs:%d, total number inputs: %d' % - (len(effective_fids), len(self._input_fields))) + logging.info( + 'number of effective inputs:%d, total number inputs: %d' % + (len(effective_fids), len(self._input_fields)) + ) else: effective_fids = [ - fid for fid in range(len(self._input_fields)) - if self._input_fields[fid] not in self._label_fields and - self._input_fields[fid] != sample_weight_field + fid for fid in range(len(self._input_fields)) + if self._input_fields[fid] not in self._label_fields + and self._input_fields[fid] != sample_weight_field ] logging.info( - 'will not filter any input[except labels], total number inputs:%d' % - len(effective_fids)) + 'will not filter any input[except labels], total number inputs:%d' % + len(effective_fids) + ) input_vals = tf.reshape( - input_vals, [-1, len(effective_fids)], name='input_reshape') + input_vals, [-1, len(effective_fids)], name='input_reshape' + ) features = {} for tmp_id, fid in enumerate(effective_fids): ftype = self._input_field_types[fid] @@ -332,13 +343,15 @@ def create_placeholders(self, export_config): input_name = self._input_fields[fid] if tf_type in [tf.float32, tf.double, tf.int32, tf.int64]: features[input_name] = tf.string_to_number( - input_vals[:, tmp_id], - tf_type, - name='input_str_to_%s' % tf_type.name) + input_vals[:, tmp_id], + tf_type, + name='input_str_to_%s' % tf_type.name + ) else: if ftype not in [DatasetConfig.STRING]: - logging.warning('unexpected field type: ftype=%s tf_type=%s' % - (ftype, tf_type)) + logging.warning( + 'unexpected field type: ftype=%s tf_type=%s' % (ftype, tf_type) + ) features[input_name] = input_vals[:, tmp_id] features = self._preprocess(features) return {'features': inputs_placeholder}, features['feature'] @@ -348,17 +361,21 @@ def _get_features(self, fields): def _get_labels(self, fields): labels = fields['label'] - return OrderedDict([ - (x, tf.squeeze(labels[x], axis=1) if len(labels[x].get_shape()) == 2 and - labels[x].get_shape()[1] == 1 else labels[x]) for x in labels - ]) + return OrderedDict( + [ + ( + x, tf.squeeze(labels[x], axis=1) if len(labels[x].get_shape()) == 2 + and labels[x].get_shape()[1] == 1 else labels[x] + ) for x in labels + ] + ) def _as_string(self, field, fc): if field.dtype == tf.string: return field if field.dtype in [tf.float32, tf.double]: - feature_name = fc.feature_name if fc.HasField( - 'feature_name') else fc.input_names[0] + feature_name = fc.feature_name if fc.HasField('feature_name' + ) else fc.input_names[0] assert fc.precision > 0, 'fc.precision not set for feature[%s], it is dangerous to convert ' \ 'float or double to string due to precision problem, it is suggested ' \ ' to convert them into string format before using EasyRec; ' \ @@ -377,8 +394,8 @@ def _as_string(self, field, fc): def _parse_combo_feature(self, fc, parsed_dict, field_dict): # for compatibility with existing implementations - feature_name = fc.feature_name if fc.HasField( - 'feature_name') else fc.input_names[0] + feature_name = fc.feature_name if fc.HasField('feature_name' + ) else fc.input_names[0] if len(fc.combo_input_seps) > 0: assert len(fc.combo_input_seps) == len(fc.input_names), \ @@ -400,8 +417,10 @@ def _get_input_sep(input_id): input_sep = _get_input_sep(input_id) if input_sep != '': assert field_dict[ - input_name].dtype == tf.string, 'could not apply string_split to input-name[%s] dtype=%s' % ( - input_name, field_dict[input_name].dtype) + input_name + ].dtype == tf.string, 'could not apply string_split to input-name[%s] dtype=%s' % ( + input_name, field_dict[input_name].dtype + ) parsed_dict[key] = tf.string_split(field_dict[input_name], input_sep) else: parsed_dict[key] = self._as_string(field_dict[input_name], fc) @@ -412,22 +431,28 @@ def _get_input_sep(input_id): input_sep = fc.combo_input_seps[input_id] if len(input_sep) > 0: assert field_dict[ - input_name].dtype == tf.string, 'could not apply string_split to input-name[%s] dtype=%s' % ( - input_name, field_dict[input_name].dtype) + input_name + ].dtype == tf.string, 'could not apply string_split to input-name[%s] dtype=%s' % ( + input_name, field_dict[input_name].dtype + ) split_inputs.append( - tf.string_split(field_dict[input_name], - fc.combo_input_seps[input_id])) + tf.string_split( + field_dict[input_name], fc.combo_input_seps[input_id] + ) + ) else: split_inputs.append(tf.reshape(field_dict[input_name], [-1, 1])) parsed_dict[feature_name] = sparse_ops.sparse_cross( - split_inputs, fc.combo_join_sep) + split_inputs, fc.combo_join_sep + ) else: inputs = [ - self._as_string(field_dict[input_name], fc) - for input_name in fc.input_names + self._as_string(field_dict[input_name], fc) + for input_name in fc.input_names ] parsed_dict[feature_name] = string_ops.string_join( - inputs, fc.combo_join_sep) + inputs, fc.combo_join_sep + ) def _parse_tag_feature(self, fc, parsed_dict, field_dict): input_0 = fc.input_names[0] @@ -452,30 +477,36 @@ def _parse_tag_feature(self, fc, parsed_dict, field_dict): tmp_ks, tmp_vs = tmp_kvs[:, 0], tmp_kvs[:, 1] check_list = [ - tf.py_func(check_string_to_number, [tmp_vs, input_0], Tout=tf.bool) + tf.py_func(check_string_to_number, [tmp_vs, input_0], Tout=tf.bool) ] if self._check_mode else [] with tf.control_dependencies(check_list): tmp_vs = tf.string_to_number( - tmp_vs, tf.float32, name='kv_tag_wgt_str_2_flt_%s' % input_0) + tmp_vs, tf.float32, name='kv_tag_wgt_str_2_flt_%s' % input_0 + ) parsed_dict[feature_name] = tf.sparse.SparseTensor( - indices, tmp_ks, parsed_dict[feature_name].dense_shape) + indices, tmp_ks, parsed_dict[feature_name].dense_shape + ) parsed_dict[feature_name + '_w'] = tf.sparse.SparseTensor( - indices, tmp_vs, parsed_dict[feature_name].dense_shape) + indices, tmp_vs, parsed_dict[feature_name].dense_shape + ) if not fc.HasField('hash_bucket_size') and fc.num_buckets > 0: check_list = [ - tf.py_func( - check_string_to_number, - [parsed_dict[feature_name].values, input_0], - Tout=tf.bool) + tf.py_func( + check_string_to_number, + [parsed_dict[feature_name].values, input_0], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): vals = tf.string_to_number( - parsed_dict[feature_name].values, - tf.int32, - name='tag_fea_%s' % input_0) + parsed_dict[feature_name].values, + tf.int32, + name='tag_fea_%s' % input_0 + ) parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, vals, - parsed_dict[feature_name].dense_shape) + parsed_dict[feature_name].indices, vals, + parsed_dict[feature_name].dense_shape + ) if len(fc.input_names) > 1: input_1 = fc.input_names[1] field = field_dict[input_1] @@ -483,20 +514,25 @@ def _parse_tag_feature(self, fc, parsed_dict, field_dict): field = tf.expand_dims(field, axis=0) field = tf.string_split(field, fc.separator) check_list = [ - tf.py_func( - check_string_to_number, [field.values, input_1], Tout=tf.bool) + tf.py_func( + check_string_to_number, [field.values, input_1], Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): field_vals = tf.string_to_number( - field.values, tf.float32, name='tag_wgt_str_2_flt_%s' % input_1) + field.values, tf.float32, name='tag_wgt_str_2_flt_%s' % input_1 + ) assert_op = tf.assert_equal( - tf.shape(field_vals)[0], - tf.shape(parsed_dict[feature_name].values)[0], - message='TagFeature Error: The size of %s not equal to the size of %s. Please check input: %s and %s.' - % (input_0, input_1, input_0, input_1)) + tf.shape(field_vals)[0], + tf.shape(parsed_dict[feature_name].values)[0], + message= + 'TagFeature Error: The size of %s not equal to the size of %s. Please check input: %s and %s.' + % (input_0, input_1, input_0, input_1) + ) with tf.control_dependencies([assert_op]): - field = tf.sparse.SparseTensor(field.indices, tf.identity(field_vals), - field.dense_shape) + field = tf.sparse.SparseTensor( + field.indices, tf.identity(field_vals), field.dense_shape + ) parsed_dict[feature_name + '_w'] = field else: parsed_dict[feature_name] = field_dict[input_0] @@ -511,23 +547,27 @@ def _parse_expr_feature(self, fc, parsed_dict, field_dict): new_input_name = prefix + input_name if field_dict[input_name].dtype == tf.string: check_list = [ - tf.py_func( - check_string_to_number, [field_dict[input_name], input_name], - Tout=tf.bool) + tf.py_func( + check_string_to_number, [field_dict[input_name], input_name], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): parsed_dict[new_input_name] = tf.string_to_number( - field_dict[input_name], - tf.float64, - name='%s_str_2_int_for_expr' % new_input_name) + field_dict[input_name], + tf.float64, + name='%s_str_2_int_for_expr' % new_input_name + ) elif field_dict[input_name].dtype in [ - tf.int32, tf.int64, tf.double, tf.float32 + tf.int32, tf.int64, tf.double, tf.float32 ]: - parsed_dict[new_input_name] = tf.cast(field_dict[input_name], - tf.float64) + parsed_dict[new_input_name] = tf.cast( + field_dict[input_name], tf.float64 + ) else: assert False, 'invalid input dtype[%s] for expr feature' % str( - field_dict[input_name].dtype) + field_dict[input_name].dtype + ) expression = get_expression(fc.expression, fc.input_names, prefix=prefix) logging.info('expression: %s' % expression) @@ -544,15 +584,15 @@ def _parse_id_feature(self, fc, parsed_dict, field_dict): elif fc.num_buckets > 0: if parsed_dict[feature_name].dtype == tf.string: check_list = [ - tf.py_func( - check_string_to_number, [parsed_dict[feature_name], input_0], - Tout=tf.bool) + tf.py_func( + check_string_to_number, [parsed_dict[feature_name], input_0], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.string_to_number( - parsed_dict[feature_name], - tf.int32, - name='%s_str_2_int' % input_0) + parsed_dict[feature_name], tf.int32, name='%s_str_2_int' % input_0 + ) def _parse_raw_feature(self, fc, parsed_dict, field_dict): input_0 = fc.input_names[0] @@ -567,21 +607,24 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): segment_ids = tf.range(0, tf.shape(vals)[0]) if fc.raw_input_dim > 1: check_list = [ - tf.py_func( - check_split, [vals, fc.separator, fc.raw_input_dim, input_0], - Tout=tf.bool) + tf.py_func( + check_split, [vals, fc.separator, fc.raw_input_dim, input_0], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): tmp_fea = tf.string_split(vals, fc.separator) check_list = [ - tf.py_func( - check_string_to_number, [tmp_fea.values, input_0], Tout=tf.bool) + tf.py_func( + check_string_to_number, [tmp_fea.values, input_0], Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): tmp_vals = tf.string_to_number( - tmp_fea.values, - tf.float32, - name='multi_raw_fea_to_flt_%s' % input_0) + tmp_fea.values, + tf.float32, + name='multi_raw_fea_to_flt_%s' % input_0 + ) if fc.HasField('seq_multi_sep') and fc.HasField('combiner'): emb = tf.reshape(tmp_vals, [-1, fc.raw_input_dim]) if fc.combiner == 'max': @@ -597,17 +640,19 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name] = emb else: parsed_dict[feature_name] = tf.sparse_to_dense( - tmp_fea.indices, - [tf.shape(field_dict[input_0])[0], fc.raw_input_dim], - tmp_vals, - default_value=0) + tmp_fea.indices, + [tf.shape(field_dict[input_0])[0], fc.raw_input_dim], + tmp_vals, + default_value=0 + ) elif fc.HasField('seq_multi_sep') and fc.HasField('combiner'): check_list = [ - tf.py_func(check_string_to_number, [vals, input_0], Tout=tf.bool) + tf.py_func(check_string_to_number, [vals, input_0], Tout=tf.bool) ] if self._check_mode else [] with tf.control_dependencies(check_list): emb = tf.string_to_number( - vals, tf.float32, name='raw_fea_to_flt_%s' % input_0) + vals, tf.float32, name='raw_fea_to_flt_%s' % input_0 + ) if fc.combiner == 'max': emb = tf.segment_max(emb, segment_ids) elif fc.combiner == 'sum': @@ -621,29 +666,34 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name] = emb else: check_list = [ - tf.py_func( - check_string_to_number, [field_dict[input_0], input_0], - Tout=tf.bool) + tf.py_func( + check_string_to_number, [field_dict[input_0], input_0], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.string_to_number( - field_dict[input_0], tf.float32) + field_dict[input_0], tf.float32 + ) elif field_dict[input_0].dtype in [ - tf.int32, tf.int64, tf.double, tf.float32 + tf.int32, tf.int64, tf.double, tf.float32 ]: parsed_dict[feature_name] = tf.to_float(field_dict[input_0]) else: assert False, 'invalid dtype[%s] for raw feature' % str( - field_dict[input_0].dtype) + field_dict[input_0].dtype + ) if fc.max_val > fc.min_val: - parsed_dict[feature_name] = (parsed_dict[feature_name] - fc.min_val) / ( - fc.max_val - fc.min_val) + parsed_dict[feature_name] = (parsed_dict[feature_name] - + fc.min_val) / (fc.max_val - fc.min_val) if fc.HasField('normalizer_fn'): - logging.info('apply normalizer_fn %s to `%s`' % - (fc.normalizer_fn, feature_name)) + logging.info( + 'apply normalizer_fn %s to `%s`' % (fc.normalizer_fn, feature_name) + ) parsed_dict[feature_name] = self._normalizer_fn[feature_name]( - parsed_dict[feature_name]) + parsed_dict[feature_name] + ) if not fc.boundaries and fc.num_buckets <= 1 and \ fc.embedding_dim > 0 and \ @@ -664,13 +714,15 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): tmp_parsed = parsed_dict[feature_name] parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( - indices=indices, - values=indices_1[:, 0], - dense_shape=[sample_num, fc.raw_input_dim]) + indices=indices, + values=indices_1[:, 0], + dense_shape=[sample_num, fc.raw_input_dim] + ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( - indices=indices, - values=tf.reshape(tmp_parsed, [-1]), - dense_shape=[sample_num, fc.raw_input_dim]) + indices=indices, + values=tf.reshape(tmp_parsed, [-1]), + dense_shape=[sample_num, fc.raw_input_dim] + ) # self._appended_fields.append(input_0 + '_raw_proj_id') # self._appended_fields.append(input_0 + '_raw_proj_val') @@ -692,46 +744,53 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): out_indices = tf.concat([indices, indices_1[:, 1:]], axis=1) # 3 dimensional sparse tensor out_shape = tf.concat( - [parsed_dict[feature_name].dense_shape, multi_vals.dense_shape[1:]], - axis=0) + [parsed_dict[feature_name].dense_shape, multi_vals.dense_shape[1:]], + axis=0 + ) parsed_dict[feature_name] = tf.sparse.SparseTensor( - out_indices, multi_vals.values, out_shape) + out_indices, multi_vals.values, out_shape + ) if (fc.num_buckets > 1 and fc.max_val == fc.min_val): check_list = [ - tf.py_func( - check_string_to_number, - [parsed_dict[feature_name].values, input_0], - Tout=tf.bool) + tf.py_func( + check_string_to_number, + [parsed_dict[feature_name].values, input_0], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, - tf.string_to_number( - parsed_dict[feature_name].values, - tf.int64, - name='sequence_str_2_int_%s' % input_0), - parsed_dict[feature_name].dense_shape) + parsed_dict[feature_name].indices, + tf.string_to_number( + parsed_dict[feature_name].values, + tf.int64, + name='sequence_str_2_int_%s' % input_0 + ), parsed_dict[feature_name].dense_shape + ) elif sub_feature_type == fc.RawFeature: check_list = [ - tf.py_func( - check_string_to_number, - [parsed_dict[feature_name].values, input_0], - Tout=tf.bool) + tf.py_func( + check_string_to_number, + [parsed_dict[feature_name].values, input_0], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, - tf.string_to_number( - parsed_dict[feature_name].values, - tf.float32, - name='sequence_str_2_float_%s' % input_0), - parsed_dict[feature_name].dense_shape) + parsed_dict[feature_name].indices, + tf.string_to_number( + parsed_dict[feature_name].values, + tf.float32, + name='sequence_str_2_float_%s' % input_0 + ), parsed_dict[feature_name].dense_shape + ) if fc.num_buckets > 1 and fc.max_val > fc.min_val: - normalized_values = (parsed_dict[feature_name].values - fc.min_val) / ( - fc.max_val - fc.min_val) + normalized_values = (parsed_dict[feature_name].values - + fc.min_val) / (fc.max_val - fc.min_val) parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, normalized_values, - parsed_dict[feature_name].dense_shape) + parsed_dict[feature_name].indices, normalized_values, + parsed_dict[feature_name].dense_shape + ) else: parsed_dict[feature_name] = field if not fc.boundaries and fc.num_buckets <= 1 and\ @@ -739,12 +798,14 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): sub_feature_type == fc.RawFeature and\ fc.raw_input_dim == 1: logging.info( - 'Not set boundaries or num_buckets or hash_bucket_size, %s will process as two dimension sequence raw feature' - % feature_name) + 'Not set boundaries or num_buckets or hash_bucket_size, %s will process as two dimension sequence raw feature' + % feature_name + ) parsed_dict[feature_name] = tf.sparse_to_dense( - parsed_dict[feature_name].indices, - [tf.shape(parsed_dict[feature_name])[0], fc.sequence_length], - parsed_dict[feature_name].values) + parsed_dict[feature_name].indices, + [tf.shape(parsed_dict[feature_name])[0], fc.sequence_length], + parsed_dict[feature_name].values + ) sample_num = tf.to_int64(tf.shape(parsed_dict[feature_name])[0]) indices_0 = tf.range(sample_num, dtype=tf.int64) indices_1 = tf.range(fc.sequence_length, dtype=tf.int64) @@ -757,25 +818,32 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): indices = tf.concat([indices_0, indices_1], axis=1) tmp_parsed = parsed_dict[feature_name] parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( - indices=indices, - values=indices_1[:, 0], - dense_shape=[sample_num, fc.sequence_length]) + indices=indices, + values=indices_1[:, 0], + dense_shape=[sample_num, fc.sequence_length] + ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( - indices=indices, - values=tf.reshape(tmp_parsed, [-1]), - dense_shape=[sample_num, fc.sequence_length]) - elif (not fc.boundaries and fc.num_buckets <= 1 and - self._data_config.sample_weight != input_0 and - sub_feature_type == fc.RawFeature and fc.raw_input_dim > 1): + indices=indices, + values=tf.reshape(tmp_parsed, [-1]), + dense_shape=[sample_num, fc.sequence_length] + ) + elif ( + not fc.boundaries and fc.num_buckets <= 1 + and self._data_config.sample_weight != input_0 + and sub_feature_type == fc.RawFeature and fc.raw_input_dim > 1 + ): # for 3 dimension sequence feature input. - logging.info('Not set boundaries or num_buckets or hash_bucket_size,' - ' %s will process as three dimension sequence raw feature' % - feature_name) + logging.info( + 'Not set boundaries or num_buckets or hash_bucket_size,' + ' %s will process as three dimension sequence raw feature' % + feature_name + ) parsed_dict[feature_name] = tf.sparse_to_dense( - parsed_dict[feature_name].indices, [ - tf.shape(parsed_dict[feature_name])[0], fc.sequence_length, - fc.raw_input_dim - ], parsed_dict[feature_name].values) + parsed_dict[feature_name].indices, [ + tf.shape(parsed_dict[feature_name])[0], fc.sequence_length, + fc.raw_input_dim + ], parsed_dict[feature_name].values + ) sample_num = tf.to_int64(tf.shape(parsed_dict[feature_name])[0]) indices_0 = tf.range(sample_num, dtype=tf.int64) indices_1 = tf.range(fc.sequence_length, dtype=tf.int64) @@ -793,13 +861,15 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): tmp_parsed = parsed_dict[feature_name] parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( - indices=indices, - values=indices_1[:, 0], - dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim]) + indices=indices, + values=indices_1[:, 0], + dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim] + ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( - indices=indices, - values=tf.reshape(parsed_dict[feature_name], [-1]), - dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim]) + indices=indices, + values=tf.reshape(parsed_dict[feature_name], [-1]), + dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim] + ) # self._appended_fields.append(input_0 + '_raw_proj_id') # self._appended_fields.append(input_0 + '_raw_proj_val') @@ -864,8 +934,8 @@ def _preprocess(self, field_dict): elif feature_type == fc.ComboFeature: self._parse_combo_feature(fc, parsed_dict, field_dict) else: - feature_name = fc.feature_name if fc.HasField( - 'feature_name') else fc.input_names[0] + feature_name = fc.feature_name if fc.HasField('feature_name' + ) else fc.input_names[0] for input_id, input_name in enumerate(fc.input_names): if input_id > 0: key = feature_name + '_' + str(input_id) @@ -886,38 +956,44 @@ def _preprocess(self, field_dict): assert dtype is not None, 'must set user_define_fn_res_type' logging.info('apply py_func transform: %s' % udf_class) field_dict[input_name] = tf.py_func( - udf, [field_dict[input_name]], Tout=get_tf_type(dtype)) + udf, [field_dict[input_name]], Tout=get_tf_type(dtype) + ) field_dict[input_name].set_shape(tf.TensorShape([None])) if field_dict[input_name].dtype == tf.string: if self._label_dim[input_id] > 1: logging.info('will split labels[%d]=%s' % (input_id, input_name)) check_list = [ - tf.py_func( - check_split, [ - field_dict[input_name], self._label_sep[input_id], - self._label_dim[input_id], input_name - ], - Tout=tf.bool) + tf.py_func( + check_split, [ + field_dict[input_name], self._label_sep[input_id], + self._label_dim[input_id], input_name + ], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): label_dict[input_name] = tf.string_split( - field_dict[input_name], self._label_sep[input_id]).values - label_dict[input_name] = tf.reshape(label_dict[input_name], - [-1, self._label_dim[input_id]]) + field_dict[input_name], self._label_sep[input_id] + ).values + label_dict[input_name] = tf.reshape( + label_dict[input_name], [-1, self._label_dim[input_id]] + ) else: label_dict[input_name] = field_dict[input_name] check_list = [ - tf.py_func( - check_string_to_number, [label_dict[input_name], input_name], - Tout=tf.bool) + tf.py_func( + check_string_to_number, [label_dict[input_name], input_name], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): label_dict[input_name] = tf.string_to_number( - label_dict[input_name], tf.float32, name=input_name) + label_dict[input_name], tf.float32, name=input_name + ) else: assert field_dict[input_name].dtype in [ - tf.float32, tf.double, tf.int32, tf.int64 + tf.float32, tf.double, tf.int32, tf.int64 ], 'invalid label dtype: %s' % str(field_dict[input_name].dtype) label_dict[input_name] = field_dict[input_name] @@ -925,14 +1001,15 @@ def _preprocess(self, field_dict): for func_config in self._data_config.extra_label_func: lbl_name = func_config.label_name func_name = func_config.label_func - logging.info('generating new label `%s` by transform: %s' % - (lbl_name, func_name)) + logging.info( + 'generating new label `%s` by transform: %s' % (lbl_name, func_name) + ) lbl_fn = load_by_path(func_name) label_dict[lbl_name] = lbl_fn(label_dict) if self._data_config.HasField('sample_weight'): - parsed_dict[constant.SAMPLE_WEIGHT] = field_dict[ - self._data_config.sample_weight] + parsed_dict[constant.SAMPLE_WEIGHT + ] = field_dict[self._data_config.sample_weight] if Input.DATA_OFFSET in field_dict: parsed_dict[Input.DATA_OFFSET] = field_dict[Input.DATA_OFFSET] @@ -980,14 +1057,15 @@ def _lookup(args, pad=True): indices_0 = tf.zeros([n], dtype=tf.int64) indices_1 = tf.range(0, n, dtype=tf.int64) indices = [ - tf.expand_dims(indices_0, axis=1), - tf.expand_dims(indices_1, axis=1) + tf.expand_dims(indices_0, axis=1), + tf.expand_dims(indices_1, axis=1) ] indices = tf.concat(indices, axis=1) return tf.sparse.SparseTensor(indices, vals, [1, n]) vals, masks, indices = tf.map_fn( - _lookup, [key_fields, map_fields], dtype=(tf.string, tf.bool, tf.int64)) + _lookup, [key_fields, map_fields], dtype=(tf.string, tf.bool, tf.int64) + ) batch_size = tf.to_int64(tf.shape(vals)[0]) vals = tf.boolean_mask(vals, masks) indices_1 = tf.boolean_mask(indices, masks) @@ -996,9 +1074,10 @@ def _lookup(args, pad=True): indices_0 = indices_0 + tf.zeros([1, max_sel_num], dtype=tf.int64) indices_0 = tf.boolean_mask(indices_0, masks) indices = tf.concat( - [tf.expand_dims(indices_0, axis=1), - tf.expand_dims(indices_1, axis=1)], - axis=1) + [tf.expand_dims(indices_0, axis=1), + tf.expand_dims(indices_1, axis=1)], + axis=1 + ) shapes = tf.stack([batch_size, tf.reduce_max(indices_1) + 1]) return tf.sparse.SparseTensor(indices, vals, shapes) @@ -1018,7 +1097,8 @@ def stop(self): def _safe_shard(self, dataset): if self._data_config.chief_redundant: return dataset.shard( - max(self._task_num - 1, 1), max(self._task_index - 1, 0)) + max(self._task_num - 1, 1), max(self._task_index - 1, 0) + ) else: return dataset.shard(self._task_num, self._task_index) @@ -1040,8 +1120,10 @@ def _input_fn(mode=None, params=None, config=None): tf.estimator.export.ServingInputReceiver instance """ self._pre_build(mode, params) - if mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT): + if mode in ( + tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT + ): # build dataset from self._config.input_path self._mode = mode dataset = self._build(mode, params) @@ -1056,8 +1138,9 @@ def _input_fn(mode=None, params=None, config=None): else: with conditional(place_on_cpu, ops.device('/CPU:0')): inputs, features = self.create_placeholders(export_config) - print('built feature placeholders. features: {}'.format( - features.keys())) + print( + 'built feature placeholders. features: {}'.format(features.keys()) + ) return tf.estimator.export.ServingInputReceiver(features, inputs) _input_fn.input_creator = self diff --git a/easy_rec/python/input/kafka_dataset.py b/easy_rec/python/input/kafka_dataset.py index 22ae45b90..e381553d2 100644 --- a/easy_rec/python/input/kafka_dataset.py +++ b/easy_rec/python/input/kafka_dataset.py @@ -16,31 +16,32 @@ import logging import traceback - from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import dtypes, ops, tensor_shape try: from easy_rec.python.ops import gen_kafka_ops except ImportError: - logging.warning('failed to import gen_kafka_ops: %s' % traceback.format_exc()) + logging.warning( + 'failed to import gen_kafka_ops: %s' % traceback.format_exc() + ) class KafkaDataset(dataset_ops.Dataset): """A Kafka Dataset that consumes the message.""" - def __init__(self, - topics, - servers='localhost', - group='', - eof=False, - timeout=1000, - config_global=None, - config_topic=None, - message_key=False, - message_offset=False): + def __init__( + self, + topics, + servers='localhost', + group='', + eof=False, + timeout=1000, + config_global=None, + config_topic=None, + message_key=False, + message_offset=False + ): """Create a KafkaReader. Args: @@ -67,20 +68,26 @@ def __init__(self, message_offset: If True, the kafka will output both message value and offset. """ self._topics = ops.convert_to_tensor( - topics, dtype=dtypes.string, name='topics') + topics, dtype=dtypes.string, name='topics' + ) self._servers = ops.convert_to_tensor( - servers, dtype=dtypes.string, name='servers') + servers, dtype=dtypes.string, name='servers' + ) self._group = ops.convert_to_tensor( - group, dtype=dtypes.string, name='group') + group, dtype=dtypes.string, name='group' + ) self._eof = ops.convert_to_tensor(eof, dtype=dtypes.bool, name='eof') self._timeout = ops.convert_to_tensor( - timeout, dtype=dtypes.int64, name='timeout') + timeout, dtype=dtypes.int64, name='timeout' + ) config_global = config_global if config_global else [] self._config_global = ops.convert_to_tensor( - config_global, dtype=dtypes.string, name='config_global') + config_global, dtype=dtypes.string, name='config_global' + ) config_topic = config_topic if config_topic else [] self._config_topic = ops.convert_to_tensor( - config_topic, dtype=dtypes.string, name='config_topic') + config_topic, dtype=dtypes.string, name='config_topic' + ) self._message_key = message_key self._message_offset = message_offset super(KafkaDataset, self).__init__() @@ -90,15 +97,15 @@ def _inputs(self): def _as_variant_tensor(self): return gen_kafka_ops.io_kafka_dataset_v2( - self._topics, - self._servers, - self._group, - self._eof, - self._timeout, - self._config_global, - self._config_topic, - self._message_key, - self._message_offset, + self._topics, + self._servers, + self._group, + self._eof, + self._timeout, + self._config_global, + self._config_topic, + self._message_key, + self._message_offset, ) @property @@ -114,8 +121,12 @@ def output_shapes(self): if self._message_key ^ self._message_offset: return ((tensor_shape.TensorShape([]), tensor_shape.TensorShape([]))) elif self._message_key and self._message_offset: - return ((tensor_shape.TensorShape([]), tensor_shape.TensorShape([]), - tensor_shape.TensorShape([]))) + return ( + ( + tensor_shape.TensorShape([]), tensor_shape.TensorShape([]), + tensor_shape.TensorShape([]) + ) + ) return ((tensor_shape.TensorShape([]))) @property @@ -141,4 +152,5 @@ def write_kafka_v2(message, topic, servers='localhost', name=None): A `Tensor` of type `string`. 0-D. """ return gen_kafka_ops.io_write_kafka_v2( - message=message, topic=topic, servers=servers, name=name) + message=message, topic=topic, servers=servers, name=name + ) diff --git a/easy_rec/python/input/kafka_input.py b/easy_rec/python/input/kafka_input.py index 38eab121f..aa6f5b78f 100644 --- a/easy_rec/python/input/kafka_input.py +++ b/easy_rec/python/input/kafka_input.py @@ -2,10 +2,9 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import json import logging -import traceback - import six import tensorflow as tf +import traceback from easy_rec.python.input.input import Input from easy_rec.python.input.kafka_dataset import KafkaDataset @@ -20,8 +19,9 @@ from kafka import KafkaConsumer, TopicPartition except ImportError: logging.warning( - 'kafka-python is not installed[%s]. You can install it by: pip install kafka-python' - % traceback.format_exc()) + 'kafka-python is not installed[%s]. You can install it by: pip install kafka-python' + % traceback.format_exc() + ) if tf.__version__ >= '2.0': ignore_errors = tf.data.experimental.ignore_errors() @@ -34,27 +34,33 @@ class KafkaInput(Input): DATA_OFFSET = 'DATA_OFFSET' - def __init__(self, - data_config, - feature_config, - kafka_config, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(KafkaInput, - self).__init__(data_config, feature_config, '', task_index, task_num, - check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + kafka_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(KafkaInput, self).__init__( + data_config, feature_config, '', task_index, task_num, check_mode, + pipeline_config + ) self._kafka = kafka_config self._offset_dict = {} if self._kafka is not None: consumer = KafkaConsumer( - group_id='kafka_dataset_consumer', - bootstrap_servers=[self._kafka.server], - api_version_auto_timeout_ms=60000) # in miliseconds + group_id='kafka_dataset_consumer', + bootstrap_servers=[self._kafka.server], + api_version_auto_timeout_ms=60000 + ) # in miliseconds partitions = consumer.partitions_for_topic(self._kafka.topic) self._num_partition = len(partitions) - logging.info('all partitions[%d]: %s' % (self._num_partition, partitions)) + logging.info( + 'all partitions[%d]: %s' % (self._num_partition, partitions) + ) # determine kafka offsets for each partition offset_type = self._kafka.WhichOneof('offset') @@ -62,8 +68,9 @@ def __init__(self, if offset_type == 'offset_time': ts = parse_time(self._kafka.offset_time) input_map = { - TopicPartition(partition=part_id, topic=self._kafka.topic): - ts * 1000 for part_id in partitions + TopicPartition(partition=part_id, topic=self._kafka.topic): + ts * 1000 + for part_id in partitions } part_offsets = consumer.offsets_for_times(input_map) # part_offsets is a dictionary: @@ -74,9 +81,12 @@ def __init__(self, for part in part_offsets: self._offset_dict[part.partition] = part_offsets[part].offset logging.info( - 'Find offset by time, topic[%s], partition[%d], timestamp[%ss], offset[%d], offset_timestamp[%dms]' - % (self._kafka.topic, part.partition, ts, - part_offsets[part].offset, part_offsets[part].timestamp)) + 'Find offset by time, topic[%s], partition[%d], timestamp[%ss], offset[%d], offset_timestamp[%dms]' + % ( + self._kafka.topic, part.partition, ts, + part_offsets[part].offset, part_offsets[part].timestamp + ) + ) elif offset_type == 'offset_info': offset_dict = json.loads(self._kafka.offset_info) for part in offset_dict: @@ -101,16 +111,17 @@ def _preprocess(self, field_dict): def _parse_csv(self, line, message_key, message_offset): record_defaults = [ - self.get_type_defaults(t, v) - for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) + for t, v in zip(self._input_field_types, self._input_field_defaults) ] fields = tf.decode_csv( - line, - use_quote_delim=False, - field_delim=self._data_config.separator, - record_defaults=record_defaults, - name='decode_csv') + line, + use_quote_delim=False, + field_delim=self._data_config.separator, + record_defaults=record_defaults, + name='decode_csv' + ) inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids} @@ -129,8 +140,8 @@ def _parse_offset(message_offset): self._task_offset_dict[k] = v return json.dumps(self._task_offset_dict) - inputs[Input.DATA_OFFSET] = tf.py_func(_parse_offset, [message_offset], - tf.string) + inputs[Input.DATA_OFFSET + ] = tf.py_func(_parse_offset, [message_offset], tf.string) return inputs def restore(self, checkpoint_path): @@ -166,9 +177,10 @@ def _get_topics(self): topics.append('%s:%d:%d' % (self._kafka.topic, part_id, offset)) self._task_offset_dict[part_id] = offset logging.info('assigned topic partitions: %s' % (','.join(topics))) - assert len( - topics) > 0, 'no partitions are assigned for this task(%d/%d)' % ( - self._task_index, self._task_num) + assert len(topics + ) > 0, 'no partitions are assigned for this task(%d/%d)' % ( + self._task_index, self._task_num + ) return topics def _build(self, mode, params): @@ -178,58 +190,70 @@ def _build(self, mode, params): assert self._kafka is not None, 'kafka_train_input is not set.' train_kafka = self._kafka logging.info( - 'train kafka server: %s topic: %s task_num: %d task_index: %d topics: %s' - % (train_kafka.server, train_kafka.topic, self._task_num, - self._task_index, task_topics)) + 'train kafka server: %s topic: %s task_num: %d task_index: %d topics: %s' + % ( + train_kafka.server, train_kafka.topic, self._task_num, + self._task_index, task_topics + ) + ) dataset = KafkaDataset( - task_topics, - servers=train_kafka.server, - group=train_kafka.group, - eof=False, - config_global=list(self._kafka.config_global), - config_topic=list(self._kafka.config_topic), - message_key=True, - message_offset=True) + task_topics, + servers=train_kafka.server, + group=train_kafka.group, + eof=False, + config_global=list(self._kafka.config_global), + config_topic=list(self._kafka.config_topic), + message_key=True, + message_offset=True + ) if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) else: eval_kafka = self._kafka assert self._kafka is not None, 'kafka_eval_input is not set.' logging.info( - 'eval kafka server: %s topic: %s task_num: %d task_index: %d topics: %s' - % (eval_kafka.server, eval_kafka.topic, self._task_num, - self._task_index, task_topics)) + 'eval kafka server: %s topic: %s task_num: %d task_index: %d topics: %s' + % ( + eval_kafka.server, eval_kafka.topic, self._task_num, + self._task_index, task_topics + ) + ) dataset = KafkaDataset( - task_topics, - servers=self._kafka.server, - group=eval_kafka.group, - eof=False, - config_global=list(self._kafka.config_global), - config_topic=list(self._kafka.config_topic), - message_key=True, - message_offset=True) + task_topics, + servers=self._kafka.server, + group=eval_kafka.group, + eof=False, + config_global=list(self._kafka.config_global), + config_topic=list(self._kafka.config_topic), + message_key=True, + message_offset=True + ) dataset = dataset.batch(self._data_config.batch_size) dataset = dataset.map( - self._parse_csv, num_parallel_calls=num_parallel_calls) + self._parse_csv, num_parallel_calls=num_parallel_calls + ) if self._data_config.ignore_error: dataset = dataset.apply(ignore_errors) dataset = dataset.prefetch(buffer_size=self._prefetch_size) dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + map_func=self._preprocess, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/load_parquet.py b/easy_rec/python/input/load_parquet.py index a798efb58..c8940a126 100644 --- a/easy_rec/python/input/load_parquet.py +++ b/easy_rec/python/input/load_parquet.py @@ -1,36 +1,39 @@ import logging import multiprocessing -import queue - import numpy as np import pandas as pd +import queue -def start_data_proc(task_index, - task_num, - num_proc, - file_que, - data_que, - proc_start_que, - proc_stop_que, - batch_size, - label_fields, - sparse_fea_names, - dense_fea_names, - dense_fea_cfgs, - reserve_fields, - drop_remainder, - need_pack=True): +def start_data_proc( + task_index, + task_num, + num_proc, + file_que, + data_que, + proc_start_que, + proc_stop_que, + batch_size, + label_fields, + sparse_fea_names, + dense_fea_names, + dense_fea_cfgs, + reserve_fields, + drop_remainder, + need_pack=True +): mp_ctxt = multiprocessing.get_context('spawn') proc_arr = [] for proc_id in range(num_proc): proc = mp_ctxt.Process( - target=load_data_proc, - args=(proc_id, file_que, data_que, proc_start_que, proc_stop_que, - batch_size, label_fields, sparse_fea_names, dense_fea_names, - dense_fea_cfgs, reserve_fields, drop_remainder, task_index, - task_num, need_pack), - name='task_%d_data_proc_%d' % (task_index, proc_id)) + target=load_data_proc, + args=( + proc_id, file_que, data_que, proc_start_que, proc_stop_que, batch_size, + label_fields, sparse_fea_names, dense_fea_names, dense_fea_cfgs, + reserve_fields, drop_remainder, task_index, task_num, need_pack + ), + name='task_%d_data_proc_%d' % (task_index, proc_id) + ) proc.daemon = True proc.start() proc_arr.append(proc) @@ -93,7 +96,9 @@ def _pack_sparse_feas(data_dict, sparse_fea_names): def _pack_dense_feas(data_dict, dense_fea_names, dense_fea_cfgs): fea_val_arr = [] for fea_name, fea_cfg in zip(dense_fea_names, dense_fea_cfgs): - fea_val_arr.append(data_dict[fea_name].reshape([-1, fea_cfg.raw_input_dim])) + fea_val_arr.append( + data_dict[fea_name].reshape([-1, fea_cfg.raw_input_dim]) + ) del data_dict[fea_name] fea_vals = np.concatenate(fea_val_arr, axis=1) data_dict['dense_fea'] = fea_vals @@ -102,21 +107,25 @@ def _pack_dense_feas(data_dict, dense_fea_names, dense_fea_cfgs): def _reshape_dense_feas(data_dict, dense_fea_names, dense_fea_cfgs): for fea_name, fea_cfg in zip(dense_fea_names, dense_fea_cfgs): data_dict[fea_name] = data_dict[fea_name].reshape( - [-1, fea_cfg.raw_input_dim]) + [-1, fea_cfg.raw_input_dim] + ) def _load_dense(input_data, field_names, sid, eid, dense_dict): for k in field_names: if isinstance(input_data[k][0], np.ndarray): np_dtype = type(input_data[k][sid][0]) - dense_dict[k] = np.array([x[0] for x in input_data[k][sid:eid]], - dtype=np_dtype) + dense_dict[k] = np.array( + [x[0] for x in input_data[k][sid:eid]], dtype=np_dtype + ) else: dense_dict[k] = input_data[k][sid:eid].to_numpy() -def _load_and_pad_dense(input_data, field_names, sid, dense_dict, - part_dense_dict, part_dense_dict_n, batch_size): +def _load_and_pad_dense( + input_data, field_names, sid, dense_dict, part_dense_dict, part_dense_dict_n, + batch_size +): for k in field_names: if isinstance(input_data[k][0], np.ndarray): np_dtype = type(input_data[k][sid][0]) @@ -136,12 +145,15 @@ def _load_and_pad_dense(input_data, field_names, sid, dense_dict, part_dense_dict_n[k] = tmp_lbls -def load_data_proc(proc_id, file_que, data_que, proc_start_que, proc_stop_que, - batch_size, label_fields, sparse_fea_names, dense_fea_names, - dense_fea_cfgs, reserve_fields, drop_remainder, task_index, - task_num, need_pack): - logging.info('data proc %d start, proc_start_que=%s' % - (proc_id, proc_start_que.qsize())) +def load_data_proc( + proc_id, file_que, data_que, proc_start_que, proc_stop_que, batch_size, + label_fields, sparse_fea_names, dense_fea_names, dense_fea_cfgs, + reserve_fields, drop_remainder, task_index, task_num, need_pack +): + logging.info( + 'data proc %d start, proc_start_que=%s' % + (proc_id, proc_start_que.qsize()) + ) proc_start_que.get() effective_fields = sparse_fea_names + dense_fea_names all_fields = effective_fields @@ -151,8 +163,9 @@ def load_data_proc(proc_id, file_que, data_que, proc_start_que, proc_stop_que, for tmp in reserve_fields: if tmp not in all_fields: all_fields.append(tmp) - logging.info('data proc %d start, file_que.qsize=%d' % - (proc_id, file_que.qsize())) + logging.info( + 'data proc %d start, file_que.qsize=%d' % (proc_id, file_que.qsize()) + ) num_files = 0 part_data_dict = {} @@ -195,8 +208,10 @@ def load_data_proc(proc_id, file_que, data_que, proc_start_que, proc_stop_que, all_lens = np.ones([len(val)], dtype=np.int32) all_vals = val.to_numpy() assert np.sum(all_lens) == len( - all_vals), 'len(all_vals)=%d np.sum(all_lens)=%d' % ( - len(all_vals), np.sum(all_lens)) + all_vals + ), 'len(all_vals)=%d np.sum(all_lens)=%d' % ( + len(all_vals), np.sum(all_lens) + ) data_dict[k] = (all_lens, all_vals) if len(dense_fea_names) > 0: @@ -224,19 +239,24 @@ def load_data_proc(proc_id, file_que, data_que, proc_start_que, proc_stop_que, part_data_dict_n = {} if label_fields is not None and len(label_fields) > 0: - _load_and_pad_dense(input_data, label_fields, sid, data_dict, - part_data_dict, part_data_dict_n, batch_size) + _load_and_pad_dense( + input_data, label_fields, sid, data_dict, part_data_dict, + part_data_dict_n, batch_size + ) if reserve_fields is not None and len(reserve_fields) > 0: data_dict['reserve'] = {} part_data_dict_n['reserve'] = {} - _load_and_pad_dense(input_data, label_fields, sid, data_dict['reserve'], - part_data_dict['reserve'], - part_data_dict_n['reserve'], batch_size) + _load_and_pad_dense( + input_data, label_fields, sid, data_dict['reserve'], + part_data_dict['reserve'], part_data_dict_n['reserve'], batch_size + ) if len(dense_fea_names) > 0: - _load_and_pad_dense(input_data, dense_fea_names, sid, data_dict, - part_data_dict, part_data_dict_n, batch_size) + _load_and_pad_dense( + input_data, dense_fea_names, sid, data_dict, part_data_dict, + part_data_dict_n, batch_size + ) if len(sparse_fea_names) > 0: for k in sparse_fea_names: @@ -293,18 +313,22 @@ def load_data_proc(proc_id, file_que, data_que, proc_start_que, proc_stop_que, else: if len(dense_fea_names) > 0: _reshape_dense_feas(part_data_dict, dense_fea_names, dense_fea_cfgs) - logging.info('remainder batch: %s sample_num=%d' % - (','.join(part_data_dict.keys()), batch_len)) + logging.info( + 'remainder batch: %s sample_num=%d' % + (','.join(part_data_dict.keys()), batch_len) + ) _add_to_que(part_data_dict, data_que, proc_stop_que) total_batch_cnt += 1 else: - logging.warning('drop remain %d samples as drop_remainder is set' % - batch_len) + logging.warning( + 'drop remain %d samples as drop_remainder is set' % batch_len + ) if is_good: is_good = _add_to_que(None, data_que, proc_stop_que) logging.info( - 'data_proc_id[%d]: is_good = %s, total_batch_cnt=%d, total_sample_cnt=%d' - % (proc_id, is_good, total_batch_cnt, total_sample_cnt)) + 'data_proc_id[%d]: is_good = %s, total_batch_cnt=%d, total_sample_cnt=%d' % + (proc_id, is_good, total_batch_cnt, total_sample_cnt) + ) data_que.close(wait_send_finish=is_good) while not is_good: diff --git a/easy_rec/python/input/odps_input.py b/easy_rec/python/input/odps_input.py index bbd08bc39..67f10315b 100644 --- a/easy_rec/python/input/odps_input.py +++ b/easy_rec/python/input/odps_input.py @@ -13,17 +13,20 @@ class OdpsInput(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(OdpsInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(OdpsInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) def _build(self, mode, params): # check data_config are consistent with odps tables @@ -33,52 +36,59 @@ def _build(self, mode, params): if self._data_config.chief_redundant and \ mode == tf.estimator.ModeKeys.TRAIN: reader = tf.TableRecordReader( - csv_delimiter=self._data_config.separator, - selected_cols=selected_cols, - slice_count=max(self._task_num - 1, 1), - slice_id=max(self._task_index - 1, 0)) + csv_delimiter=self._data_config.separator, + selected_cols=selected_cols, + slice_count=max(self._task_num - 1, 1), + slice_id=max(self._task_index - 1, 0) + ) else: reader = tf.TableRecordReader( - csv_delimiter=self._data_config.separator, - selected_cols=selected_cols, - slice_count=self._task_num, - slice_id=self._task_index) + csv_delimiter=self._data_config.separator, + selected_cols=selected_cols, + slice_count=self._task_num, + slice_id=self._task_index + ) if type(self._input_path) != list: self._input_path = self._input_path.split(',') assert len( - self._input_path) > 0, 'match no files with %s' % self._input_path + self._input_path + ) > 0, 'match no files with %s' % self._input_path if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.pai_worker_queue: work_queue = pai.data.WorkQueue( - self._input_path, - num_epochs=self.num_epochs, - shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num) + self._input_path, + num_epochs=self.num_epochs, + shuffle=self._data_config.shuffle, + num_slices=self._data_config.pai_worker_slice_num * self._task_num + ) work_queue.add_summary() file_queue = work_queue.input_producer() reader = tf.TableRecordReader() else: file_queue = tf.train.string_input_producer( - self._input_path, - num_epochs=self.num_epochs, - capacity=1000, - shuffle=self._data_config.shuffle) + self._input_path, + num_epochs=self.num_epochs, + capacity=1000, + shuffle=self._data_config.shuffle + ) else: file_queue = tf.train.string_input_producer( - self._input_path, num_epochs=1, capacity=1000, shuffle=False) + self._input_path, num_epochs=1, capacity=1000, shuffle=False + ) key, value = reader.read_up_to(file_queue, self._batch_size) record_defaults = [ - self.get_type_defaults(t, v) - for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) + for t, v in zip(self._input_field_types, self._input_field_defaults) ] fields = tf.decode_csv( - value, - record_defaults=record_defaults, - field_delim=self._data_config.separator, - name='decode_csv') + value, + record_defaults=record_defaults, + field_delim=self._data_config.separator, + name='decode_csv' + ) inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids} for x in self._label_fids: diff --git a/easy_rec/python/input/odps_input_v2.py b/easy_rec/python/input/odps_input_v2.py index bd58de390..916ab6f25 100644 --- a/easy_rec/python/input/odps_input_v2.py +++ b/easy_rec/python/input/odps_input_v2.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.input.input import Input @@ -15,17 +14,20 @@ class OdpsInputV2(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(OdpsInputV2, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(OdpsInputV2, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) def _parse_table(self, *fields): fields = list(fields) @@ -38,52 +40,59 @@ def _build(self, mode, params): if type(self._input_path) != list: self._input_path = self._input_path.split(',') assert len( - self._input_path) > 0, 'match no files with %s' % self._input_path + self._input_path + ) > 0, 'match no files with %s' % self._input_path # check data_config are consistent with odps tables odps_util.check_input_field_and_types(self._data_config) selected_cols = ','.join(self._input_fields) record_defaults = [ - self.get_type_defaults(x, v) - for x, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(x, v) + for x, v in zip(self._input_field_types, self._input_field_defaults) ] if self._data_config.pai_worker_queue and \ mode == tf.estimator.ModeKeys.TRAIN: - logging.info('pai_worker_slice_num = %d' % - self._data_config.pai_worker_slice_num) + logging.info( + 'pai_worker_slice_num = %d' % self._data_config.pai_worker_slice_num + ) work_queue = pai.data.WorkQueue( - self._input_path, - num_epochs=self.num_epochs, - shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num) + self._input_path, + num_epochs=self.num_epochs, + shuffle=self._data_config.shuffle, + num_slices=self._data_config.pai_worker_slice_num * self._task_num + ) que_paths = work_queue.input_dataset() dataset = tf.data.TableRecordDataset( - que_paths, - record_defaults=record_defaults, - selected_cols=selected_cols) + que_paths, + record_defaults=record_defaults, + selected_cols=selected_cols + ) elif self._data_config.chief_redundant and \ mode == tf.estimator.ModeKeys.TRAIN: dataset = tf.data.TableRecordDataset( - self._input_path, - record_defaults=record_defaults, - selected_cols=selected_cols, - slice_id=max(self._task_index - 1, 0), - slice_count=max(self._task_num - 1, 1)) + self._input_path, + record_defaults=record_defaults, + selected_cols=selected_cols, + slice_id=max(self._task_index - 1, 0), + slice_count=max(self._task_num - 1, 1) + ) else: dataset = tf.data.TableRecordDataset( - self._input_path, - record_defaults=record_defaults, - selected_cols=selected_cols, - slice_id=self._task_index, - slice_count=self._task_num) + self._input_path, + record_defaults=record_defaults, + selected_cols=selected_cols, + slice_id=self._task_index, + slice_count=self._task_num + ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) @@ -91,20 +100,23 @@ def _build(self, mode, params): dataset = dataset.batch(batch_size=self._data_config.batch_size) dataset = dataset.map( - self._parse_table, - num_parallel_calls=self._data_config.num_parallel_calls) + self._parse_table, + num_parallel_calls=self._data_config.num_parallel_calls + ) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/odps_input_v3.py b/easy_rec/python/input/odps_input_v3.py index 6ec737ca1..85d32bd62 100644 --- a/easy_rec/python/input/odps_input_v3.py +++ b/easy_rec/python/input/odps_input_v3.py @@ -3,7 +3,6 @@ import logging import sys - import tensorflow as tf from easy_rec.python.input.input import Input @@ -19,23 +18,27 @@ class OdpsInputV3(Input): """Common IO based interface, could run at local or on data science.""" - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(OdpsInputV3, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(OdpsInputV3, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) self._num_epoch = 0 if common_io is None: - logging.error(''' + logging.error( + ''' please install common_io pip install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.2%2Btunnel-py2.py3-none-any.whl''' - ) + ) sys.exit(1) def _parse_table(self, *fields): @@ -51,33 +54,36 @@ def _odps_read(self): if type(self._input_path) != list: self._input_path = self._input_path.split(',') assert len( - self._input_path) > 0, 'match no files with %s' % self._input_path + self._input_path + ) > 0, 'match no files with %s' % self._input_path # check data_config are consistent with odps tables odps_util.check_input_field_and_types(self._data_config) record_defaults = [ - self.get_type_defaults(x, v) - for x, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(x, v) + for x, v in zip(self._input_field_types, self._input_field_defaults) ] selected_cols = ','.join(self._input_fields) for table_path in self._input_path: reader = common_io.table.TableReader( - table_path, - selected_cols=selected_cols, - slice_id=self._task_index, - slice_count=self._task_num) + table_path, + selected_cols=selected_cols, + slice_id=self._task_index, + slice_count=self._task_num + ) total_records_num = reader.get_row_count() batch_num = int(total_records_num / self._data_config.batch_size) res_num = total_records_num - batch_num * self._data_config.batch_size batch_defaults = [ - [x] * self._data_config.batch_size for x in record_defaults + [x] * self._data_config.batch_size for x in record_defaults ] for batch_id in range(batch_num): batch_data_np = [x.copy() for x in batch_defaults] for row_id, one_data in enumerate( - reader.read(self._data_config.batch_size)): + reader.read(self._data_config.batch_size) + ): for col_id in range(len(record_defaults)): if one_data[col_id] not in ['', 'NULL', None]: batch_data_np[col_id][row_id] = one_data[col_id] @@ -101,32 +107,37 @@ def _build(self, mode, params): # read odps tables dataset = tf.data.Dataset.from_generator( - self._odps_read, output_types=list_type, output_shapes=list_shapes) + self._odps_read, output_types=list_type, output_shapes=list_shapes + ) if mode == tf.estimator.ModeKeys.TRAIN: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) dataset = dataset.map( - self._parse_table, - num_parallel_calls=self._data_config.num_parallel_calls) + self._parse_table, + num_parallel_calls=self._data_config.num_parallel_calls + ) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/odps_rtp_input.py b/easy_rec/python/input/odps_rtp_input.py index 6ae6096e0..4e175f4b0 100644 --- a/easy_rec/python/input/odps_rtp_input.py +++ b/easy_rec/python/input/odps_rtp_input.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.input.input import Input @@ -29,19 +28,24 @@ class OdpsRTPInput(Input): selected columns are labels """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(OdpsRTPInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) - logging.info('input_fields: %s label_fields: %s' % - (','.join(self._input_fields), ','.join(self._label_fields))) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(OdpsRTPInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) + logging.info( + 'input_fields: %s label_fields: %s' % + (','.join(self._input_fields), ','.join(self._label_fields)) + ) def _parse_table(self, *fields): fields = list(fields) @@ -55,38 +59,44 @@ def _parse_table(self, *fields): non_feature_cols = cols[:-1] # only for features, labels and sample_weight excluded record_types = [ - t for x, t in zip(self._input_fields, self._input_field_types) - if x not in non_feature_cols + t for x, t in zip(self._input_fields, self._input_field_types) + if x not in non_feature_cols ] record_defaults = [ - self.get_type_defaults(t, v) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x not in non_feature_cols + self.get_type_defaults(t, v) for x, t, v in zip( + self._input_fields, self._input_field_types, self._input_field_defaults + ) if x not in non_feature_cols ] feature_num = len(record_types) # assume that the last field is the generated feature column - print('field_delim = %s, feature_num = %d' % - (self._data_config.separator, feature_num)) - logging.info('field_delim = %s, input_field_name = %d' % - (self._data_config.separator, len(record_types))) + print( + 'field_delim = %s, feature_num = %d' % + (self._data_config.separator, feature_num) + ) + logging.info( + 'field_delim = %s, input_field_name = %d' % + (self._data_config.separator, len(record_types)) + ) check_list = [ - tf.py_func( - check_split, - [fields[-1], self._data_config.separator, - len(record_types)], - Tout=tf.bool) + tf.py_func( + check_split, + [fields[-1], self._data_config.separator, + len(record_types)], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): fields = str_split_by_chr( - fields[-1], self._data_config.separator, skip_empty=False) + fields[-1], self._data_config.separator, skip_empty=False + ) tmp_fields = tf.reshape(fields.values, [-1, feature_num]) fields = labels[len(self._label_fields):] for i in range(feature_num): - field = string_to_number(tmp_fields[:, i], record_types[i], - record_defaults[i], i) + field = string_to_number( + tmp_fields[:, i], record_types[i], record_defaults[i], i + ) fields.append(field) field_keys = [x for x in self._input_fields if x not in self._label_fields] @@ -95,72 +105,84 @@ def _parse_table(self, *fields): for x in range(len(self._label_fields)): inputs[self._label_fields[x]] = labels[x] - print('effective field num = %d, input_num = %d' % - (len(fields), len(inputs))) + print( + 'effective field num = %d, input_num = %d' % (len(fields), len(inputs)) + ) return inputs def _build(self, mode, params): if type(self._input_path) != list: self._input_path = self._input_path.split(',') assert len( - self._input_path) > 0, 'match no files with %s' % self._input_path + self._input_path + ) > 0, 'match no files with %s' % self._input_path selected_cols = self._data_config.selected_cols \ if self._data_config.selected_cols else None if selected_cols: cols = [c.strip() for c in selected_cols.split(',')] record_defaults = [ - self.get_type_defaults(t, v) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x in cols[:-1] + self.get_type_defaults(t, v) for x, t, v in zip( + self._input_fields, self._input_field_types, + self._input_field_defaults + ) if x in cols[:-1] ] - print('selected_cols: %s; defaults num: %d' % - (','.join(cols), len(record_defaults))) + print( + 'selected_cols: %s; defaults num: %d' % + (','.join(cols), len(record_defaults)) + ) else: record_defaults = [ - self.get_type_defaults(t, v) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x in self._label_fields + self.get_type_defaults(t, v) for x, t, v in zip( + self._input_fields, self._input_field_types, + self._input_field_defaults + ) if x in self._label_fields ] # the actual features are in one single column record_defaults.append( - self._data_config.separator.join([ - str(self.get_type_defaults(t, v)) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x not in self._label_fields - ])) + self._data_config.separator.join( + [ + str(self.get_type_defaults(t, v)) for x, t, v in zip( + self._input_fields, self._input_field_types, + self._input_field_defaults + ) if x not in self._label_fields + ] + ) + ) if self._data_config.pai_worker_queue and \ mode == tf.estimator.ModeKeys.TRAIN: - logging.info('pai_worker_slice_num = %d' % - self._data_config.pai_worker_slice_num) + logging.info( + 'pai_worker_slice_num = %d' % self._data_config.pai_worker_slice_num + ) work_queue = pai.data.WorkQueue( - self._input_path, - num_epochs=self.num_epochs, - shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num) + self._input_path, + num_epochs=self.num_epochs, + shuffle=self._data_config.shuffle, + num_slices=self._data_config.pai_worker_slice_num * self._task_num + ) que_paths = work_queue.input_dataset() dataset = tf.data.TableRecordDataset( - que_paths, - record_defaults=record_defaults, - selected_cols=selected_cols) + que_paths, + record_defaults=record_defaults, + selected_cols=selected_cols + ) else: dataset = tf.data.TableRecordDataset( - self._input_path, - record_defaults=record_defaults, - selected_cols=selected_cols, - slice_id=self._task_index, - slice_count=self._task_num) + self._input_path, + record_defaults=record_defaults, + selected_cols=selected_cols, + slice_id=self._task_index, + slice_count=self._task_num + ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) @@ -168,20 +190,23 @@ def _build(self, mode, params): dataset = dataset.batch(batch_size=self._data_config.batch_size) dataset = dataset.map( - self._parse_table, - num_parallel_calls=self._data_config.num_parallel_calls) + self._parse_table, + num_parallel_calls=self._data_config.num_parallel_calls + ) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/odps_rtp_input_v2.py b/easy_rec/python/input/odps_rtp_input_v2.py index 77edb46c1..7ac7b234c 100644 --- a/easy_rec/python/input/odps_rtp_input_v2.py +++ b/easy_rec/python/input/odps_rtp_input_v2.py @@ -2,7 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import json import logging - import tensorflow as tf from easy_rec.python.input.odps_rtp_input import OdpsRTPInput @@ -31,18 +30,21 @@ class OdpsRTPInputV2(OdpsRTPInput): selected columns are labels """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - fg_json_path=None, - pipeline_config=None): - super(OdpsRTPInputV2, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + fg_json_path=None, + pipeline_config=None + ): + super(OdpsRTPInputV2, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) if fg_json_path.startswith('!'): fg_json_path = fg_json_path[1:] self._fg_config_path = fg_json_path @@ -87,12 +89,14 @@ def create_placeholders(self, *args, **kwargs): def create_multi_placeholders(self, *args, **kwargs): """Create serving multi-placeholders with rtp_fg.""" raise NotImplementedError( - 'create_multi_placeholders is not supported for OdpsRTPInputV2') + 'create_multi_placeholders is not supported for OdpsRTPInputV2' + ) def check_rtp(self): if rtp_fg is None: raise NotImplementedError( - 'OdpsRTPInputV2 cannot run without rtp_fg, which is not installed') + 'OdpsRTPInputV2 cannot run without rtp_fg, which is not installed' + ) def _pre_build(self, mode, params): try: diff --git a/easy_rec/python/input/parquet_input.py b/easy_rec/python/input/parquet_input.py index dcc6e867f..b7446409f 100644 --- a/easy_rec/python/input/parquet_input.py +++ b/easy_rec/python/input/parquet_input.py @@ -3,9 +3,8 @@ import logging import multiprocessing import queue -import time - import tensorflow as tf +import time from tensorflow.python.ops import array_ops from easy_rec.python.compat import queues @@ -18,18 +17,21 @@ class ParquetInput(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs): - super(ParquetInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config, **kwargs) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + **kwargs + ): + super(ParquetInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config, **kwargs + ) self._need_pack = True if input_path is None: return @@ -37,15 +39,20 @@ def __init__(self, self._input_files = [] for sub_path in input_path.strip().split(','): self._input_files.extend(tf.gfile.Glob(sub_path)) - logging.info('parquet input_path=%s file_num=%d' % - (input_path, len(self._input_files))) + logging.info( + 'parquet input_path=%s file_num=%d' % + (input_path, len(self._input_files)) + ) mp_ctxt = multiprocessing.get_context('spawn') self._data_que = queues.Queue( - name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size) + name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size + ) file_num = len(self._input_files) - logging.info('[task_index=%d] total_file_num=%d task_num=%d' % - (task_index, file_num, task_num)) + logging.info( + '[task_index=%d] total_file_num=%d task_num=%d' % + (task_index, file_num, task_num) + ) self._my_files = [] for file_id in range(file_num): @@ -53,8 +60,9 @@ def __init__(self, self._my_files.append(self._input_files[file_id]) # self._my_files = self._input_files - logging.info('[task_index=%d] task_file_num=%d' % - (task_index, len(self._my_files))) + logging.info( + '[task_index=%d] task_file_num=%d' % (task_index, len(self._my_files)) + ) self._file_que = queues.Queue(name='file_que', ctx=mp_ctxt) self._num_proc = 8 @@ -100,7 +108,8 @@ def __init__(self, def _rebuild_que(self): mp_ctxt = multiprocessing.get_context('spawn') self._data_que = queues.Queue( - name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size) + name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size + ) self._file_que = queues.Queue(name='file_que', ctx=mp_ctxt) self._proc_start_que = queues.Queue(name='proc_start_que', ctx=mp_ctxt) self._proc_stop_que = queues.Queue(name='proc_stop_que', ctx=mp_ctxt) @@ -110,8 +119,10 @@ def _sample_generator(self): self._proc_start = True for proc in (self._proc_arr): self._proc_start_que.put(True) - logging.info('task[%s] data_proc=%s is_alive=%s' % - (self._task_index, proc, proc.is_alive())) + logging.info( + 'task[%s] data_proc=%s is_alive=%s' % + (self._task_index, proc, proc.is_alive()) + ) done_proc_cnt = 0 fetch_timeout_cnt = 0 @@ -142,27 +153,36 @@ def _sample_generator(self): yield sample if fetch_good_cnt % 200 == 0: logging.info( - 'task[%d] fetch_batch_cnt=%d, fetch_timeout_cnt=%d, qsize=%d' % - (self._task_index, fetch_good_cnt, fetch_timeout_cnt, - self._data_que.qsize())) + 'task[%d] fetch_batch_cnt=%d, fetch_timeout_cnt=%d, qsize=%d' % ( + self._task_index, fetch_good_cnt, fetch_timeout_cnt, + self._data_que.qsize() + ) + ) except queue.Empty: fetch_timeout_cnt += 1 if done_proc_cnt >= len(self._proc_arr): - logging.info('all sample finished, fetch_timeout_cnt=%d' % - fetch_timeout_cnt) + logging.info( + 'all sample finished, fetch_timeout_cnt=%d' % fetch_timeout_cnt + ) break except Exception as ex: - logging.warning('task[%d] get from data_que exception: %s' % - (self._task_index, str(ex))) + logging.warning( + 'task[%d] get from data_que exception: %s' % + (self._task_index, str(ex)) + ) break - logging.info('task[%d] sample_generator: total_batches=%d' % - (self._task_index, fetch_good_cnt)) + logging.info( + 'task[%d] sample_generator: total_batches=%d' % + (self._task_index, fetch_good_cnt) + ) def stop(self): if self._proc_arr is None or len(self._proc_arr) == 0: return - logging.info('task[%d] will stop dataset procs, proc_num=%d' % - (self._task_index, len(self._proc_arr))) + logging.info( + 'task[%d] will stop dataset procs, proc_num=%d' % + (self._task_index, len(self._proc_arr)) + ) self._file_que.close() if self._proc_start: logging.info('try close data que') @@ -204,12 +224,12 @@ def _to_fea_dict(self, input_dict): if len(self._sparse_fea_names) > 0: if self._has_ev: tmp_vals, tmp_lens = input_dict['sparse_fea'][1], input_dict[ - 'sparse_fea'][0] + 'sparse_fea'][0] fea_dict['sparse_fea'] = (tmp_vals, tmp_lens) else: tmp_vals, tmp_lens = input_dict['sparse_fea'][1], input_dict[ - 'sparse_fea'][0] + 'sparse_fea'][0] num_buckets = -1 for fc in self._feature_configs: if fc.num_buckets > 0: @@ -217,7 +237,8 @@ def _to_fea_dict(self, input_dict): num_buckets = fc.num_buckets else: assert num_buckets == fc.num_buckets, 'all features must share the same buckets, but are %d and %s' % ( - num_buckets, str(fc)) + num_buckets, str(fc) + ) fea_dict['sparse_fea'] = (tmp_vals % num_buckets, tmp_lens) if len(self._dense_fea_names) > 0: @@ -244,17 +265,20 @@ def add_fea_type_and_shape(self, out_types, out_shapes): # second field: field values if len(self._sparse_fea_names) > 0: out_types['sparse_fea'] = (tf.int32, tf.int64) - out_shapes['sparse_fea'] = (tf.TensorShape([None]), tf.TensorShape([None - ])) + out_shapes['sparse_fea'] = ( + tf.TensorShape([None]), tf.TensorShape([None]) + ) if len(self._dense_fea_names) > 0: out_types['dense_fea'] = tf.float32 out_shapes['dense_fea'] = tf.TensorShape( - [None, self._total_dense_fea_dim]) + [None, self._total_dense_fea_dim] + ) def _build(self, mode, params): if mode == tf.estimator.ModeKeys.TRAIN and self._data_config.num_epochs > 1: - logging.info('will repeat train data for %d epochs' % - self._data_config.num_epochs) + logging.info( + 'will repeat train data for %d epochs' % self._data_config.num_epochs + ) my_files = self._my_files * self._data_config.num_epochs else: my_files = self._my_files @@ -268,22 +292,23 @@ def _build(self, mode, params): lbl_fields = None drop_remainder = False self._proc_arr = load_parquet.start_data_proc( - self._task_index, - self._task_num, - self._num_proc, - self._file_que, - self._data_que, - self._proc_start_que, - self._proc_stop_que, - self._batch_size, - lbl_fields, - # self._effective_fields, - self._sparse_fea_names, - self._dense_fea_names, - self._dense_fea_cfgs, - self._reserve_fields, - drop_remainder, - need_pack=self._need_pack) + self._task_index, + self._task_num, + self._num_proc, + self._file_que, + self._data_que, + self._proc_start_que, + self._proc_stop_que, + self._batch_size, + lbl_fields, + # self._effective_fields, + self._sparse_fea_names, + self._dense_fea_names, + self._dense_fea_cfgs, + self._reserve_fields, + drop_remainder, + need_pack=self._need_pack + ) for input_file in my_files: self._file_que.put(input_file) @@ -291,8 +316,9 @@ def _build(self, mode, params): # add end signal for proc in self._proc_arr: self._file_que.put(None) - logging.info('add input_files to file_que, qsize=%d' % - self._file_que.qsize()) + logging.info( + 'add input_files to file_que, qsize=%d' % self._file_que.qsize() + ) out_types = {} out_shapes = {} @@ -312,12 +338,12 @@ def _build(self, mode, params): self.add_fea_type_and_shape(out_types, out_shapes) dataset = tf.data.Dataset.from_generator( - self._sample_generator, - output_types=out_types, - output_shapes=out_shapes) + self._sample_generator, output_types=out_types, output_shapes=out_shapes + ) num_parallel_calls = self._data_config.num_parallel_calls dataset = dataset.map( - self._to_fea_dict, num_parallel_calls=num_parallel_calls) + self._to_fea_dict, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) # Note: Input._preprocess is currently not supported as all features @@ -326,8 +352,9 @@ def _build(self, mode, params): # map_func=self._preprocess, num_parallel_calls=num_parallel_calls) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) # initial test show that prefetch to gpu has no performance gain # dataset = dataset.apply(tf.data.experimental.prefetch_to_device('/gpu:0')) else: @@ -340,10 +367,10 @@ def _build(self, mode, params): def _get_for_predictor(self, fea_dict): out_dict = { - 'feature': { - 'ragged_ids': fea_dict['feature']['sparse_fea'][0], - 'ragged_lens': fea_dict['feature']['sparse_fea'][1] - } + 'feature': { + 'ragged_ids': fea_dict['feature']['sparse_fea'][0], + 'ragged_lens': fea_dict['feature']['sparse_fea'][1] + } } if self._is_predictor and self._reserve_fields is not None: out_dict['reserve'] = fea_dict['reserve'] @@ -366,8 +393,10 @@ def _input_fn(mode=None, params=None, config=None): else, return: tf.estimator.export.ServingInputReceiver instance """ - if mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT): + if mode in ( + tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT + ): # build dataset from self._config.input_path self._mode = mode dataset = self._build(mode, params) @@ -376,20 +405,23 @@ def _input_fn(mode=None, params=None, config=None): inputs, features = {}, {} if len(self._sparse_fea_names) > 0: ragged_ids = array_ops.placeholder( - tf.int64, [None], name='ragged_ids') + tf.int64, [None], name='ragged_ids' + ) ragged_lens = array_ops.placeholder( - tf.int32, [None], name='ragged_lens') + tf.int32, [None], name='ragged_lens' + ) inputs = {'ragged_ids': ragged_ids, 'ragged_lens': ragged_lens} if self._has_ev: features = {'ragged_ids': ragged_ids, 'ragged_lens': ragged_lens} else: features = { - 'ragged_ids': ragged_ids % self._feature_configs[0].num_buckets, - 'ragged_lens': ragged_lens + 'ragged_ids': ragged_ids % self._feature_configs[0].num_buckets, + 'ragged_lens': ragged_lens } if len(self._dense_fea_names) > 0: inputs['dense_fea'] = array_ops.placeholder( - tf.float32, [None, self._total_dense_fea_dim], name='dense_fea') + tf.float32, [None, self._total_dense_fea_dim], name='dense_fea' + ) features['dense_fea'] = inputs['dense_fea'] return tf.estimator.export.ServingInputReceiver(features, inputs) diff --git a/easy_rec/python/input/parquet_input_v2.py b/easy_rec/python/input/parquet_input_v2.py index dba54498c..92968ef56 100644 --- a/easy_rec/python/input/parquet_input_v2.py +++ b/easy_rec/python/input/parquet_input_v2.py @@ -2,16 +2,13 @@ # Copyright (c) Alibaba, Inc. and its affiliates. # import logging import os - # import numpy as np # import pandas as pd import tensorflow as tf -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops +from tensorflow.python.framework import dtypes, ops # from tensorflow.python.ops import math_ops # from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import string_ops +from tensorflow.python.ops import array_ops, string_ops from easy_rec.python.input.parquet_input import ParquetInput from easy_rec.python.utils import conditional @@ -21,18 +18,21 @@ class ParquetInputV2(ParquetInput): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs): - super(ParquetInputV2, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config, **kwargs) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + **kwargs + ): + super(ParquetInputV2, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config, **kwargs + ) self._need_pack = False def _predictor_preprocess(self, input_dict): @@ -84,7 +84,7 @@ def _preprocess(self, inputs=None): placeholders = {} for fc in self._feature_configs: feature_name = fc.feature_name if fc.feature_name != '' else fc.input_names[ - 0] + 0] feature_type = fc.feature_type if feature_type in [fc.IdFeature, fc.TagFeature]: input_name0 = fc.input_names[0] @@ -95,9 +95,11 @@ def _preprocess(self, inputs=None): input_lens, input_vals = placeholders[input_name0] else: input_vals = array_ops.placeholder( - dtypes.int64, [None], name=input_name0 + '/ids') + dtypes.int64, [None], name=input_name0 + '/ids' + ) input_lens = array_ops.placeholder( - dtypes.int64, [None], name=input_name0 + '/lens') + dtypes.int64, [None], name=input_name0 + '/lens' + ) placeholders[input_name0] = (input_lens, input_vals) if not self._has_ev: if fc.num_buckets > 0: @@ -105,7 +107,8 @@ def _preprocess(self, inputs=None): else: input_vals = string_ops.as_string(input_vals) features[feature_name] = tf.RaggedTensor.from_row_lengths( - values=input_vals, row_lengths=input_lens) + values=input_vals, row_lengths=input_lens + ) elif feature_type in [fc.RawFeature]: input_name0 = fc.input_names[0] if inputs is not None: @@ -116,10 +119,12 @@ def _preprocess(self, inputs=None): else: if fc.raw_input_dim > 1: input_vals = array_ops.placeholder( - dtypes.float32, [None, fc.raw_input_dim], name=input_name0) + dtypes.float32, [None, fc.raw_input_dim], name=input_name0 + ) else: input_vals = array_ops.placeholder( - dtypes.float32, [None], name=input_name0) + dtypes.float32, [None], name=input_name0 + ) placeholders[input_name0] = input_vals features[feature_name] = input_vals else: @@ -163,8 +168,10 @@ def _input_fn(mode=None, params=None, config=None): else, return: tf.estimator.export.ServingInputReceiver instance """ - if mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT): + if mode in ( + tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT + ): # build dataset from self._config.input_path self._mode = mode dataset = self._build(mode, params) diff --git a/easy_rec/python/input/parquet_input_v3.py b/easy_rec/python/input/parquet_input_v3.py index 300a5bd1e..6a970522d 100644 --- a/easy_rec/python/input/parquet_input_v3.py +++ b/easy_rec/python/input/parquet_input_v3.py @@ -1,16 +1,13 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.input.input import Input from easy_rec.python.utils.input_utils import get_type_defaults try: - from tensorflow.python.data.experimental.ops import parquet_dataset_ops - from tensorflow.python.data.experimental.ops import parquet_pybind - from tensorflow.python.data.experimental.ops import dataframe + from tensorflow.python.data.experimental.ops import dataframe, parquet_dataset_ops, parquet_pybind # NOQA from tensorflow.python.ops import gen_ragged_conversion_ops from tensorflow.python.ops.work_queue import WorkQueue _has_deep_rec = True @@ -24,32 +21,35 @@ class ParquetInputV3(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs): + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + **kwargs + ): if not _has_deep_rec: raise RuntimeError('You should install DeepRec first.') - super(ParquetInputV3, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + super(ParquetInputV3, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) self._ignore_val_dict = {} for f in data_config.input_fields: if f.HasField('ignore_val'): - self._ignore_val_dict[f.input_name] = get_type_defaults( - f.input_type, f.ignore_val) + self._ignore_val_dict[f.input_name + ] = get_type_defaults(f.input_type, f.ignore_val) self._true_type_dict = {} for fc in self._feature_configs: if fc.feature_type in [fc.IdFeature, fc.TagFeature, fc.SequenceFeature]: - if fc.hash_bucket_size > 0 or len( - fc.vocab_list) > 0 or fc.HasField('vocab_file'): + if fc.hash_bucket_size > 0 or len(fc.vocab_list + ) > 0 or fc.HasField('vocab_file'): self._true_type_dict[fc.input_names[0]] = tf.string else: self._true_type_dict[fc.input_names[0]] = tf.int64 @@ -73,14 +73,16 @@ def _ignore_and_cast(self, name, value): if isinstance(value, tf.SparseTensor): indices = tf.where(tf.equal(value.values, ignore_value)) value = tf.SparseTensor( - tf.gather_nd(value.indices, indices), - tf.gather_nd(value.values, indices), value.dense_shape) + tf.gather_nd(value.indices, indices), + tf.gather_nd(value.values, indices), value.dense_shape + ) elif isinstance(value, tf.Tensor): indices = tf.where(tf.not_equal(value, ignore_value), name='indices') value = tf.SparseTensor( - indices=indices, - values=tf.gather_nd(value, indices), - dense_shape=tf.shape(value, out_type=tf.int64)) + indices=indices, + values=tf.gather_nd(value, indices), + dense_shape=tf.shape(value, out_type=tf.int64) + ) dtype = self._true_type_dict.get(name, None) if dtype: value = tf.cast(value, dtype) @@ -91,10 +93,12 @@ def _parse_dataframe_value(self, value): return value.values value.values.set_shape([None]) sparse_value = gen_ragged_conversion_ops.ragged_tensor_to_sparse( - value.nested_row_splits, value.values) - return tf.SparseTensor(sparse_value.sparse_indices, - sparse_value.sparse_values, - sparse_value.sparse_dense_shape) + value.nested_row_splits, value.values + ) + return tf.SparseTensor( + sparse_value.sparse_indices, sparse_value.sparse_values, + sparse_value.sparse_dense_shape + ) def _parse_dataframe(self, df): inputs = {} @@ -118,8 +122,10 @@ def _build(self, mode, params): for sub_path in self._input_path.strip().split(','): input_files.extend(tf.gfile.Glob(sub_path)) file_num = len(input_files) - logging.info('[task_index=%d] total_file_num=%d task_num=%d' % - (self._task_index, file_num, self._task_num)) + logging.info( + '[task_index=%d] total_file_num=%d task_num=%d' % + (self._task_index, file_num, self._task_num) + ) task_index = self._task_index task_num = self._task_num @@ -130,9 +136,10 @@ def _build(self, mode, params): if self._data_config.pai_worker_queue and \ mode == tf.estimator.ModeKeys.TRAIN: work_queue = WorkQueue( - input_files, - num_epochs=self.num_epochs, - shuffle=self._data_config.shuffle) + input_files, + num_epochs=self.num_epochs, + shuffle=self._data_config.shuffle + ) my_files = work_queue.input_dataset() else: my_files = [] @@ -157,42 +164,49 @@ def _build(self, mode, params): if f.name in all_fields: selected_fields.append(f) - num_parallel_reads = min(self._data_config.num_parallel_calls, - len(input_files) // task_num) + num_parallel_reads = min( + self._data_config.num_parallel_calls, + len(input_files) // task_num + ) dataset = parquet_dataset_ops.ParquetDataset( - my_files, - batch_size=self._batch_size, - fields=selected_fields, - drop_remainder=self._data_config.drop_remainder, - num_parallel_reads=num_parallel_reads) + my_files, + batch_size=self._batch_size, + fields=selected_fields, + drop_remainder=self._data_config.drop_remainder, + num_parallel_reads=num_parallel_reads + ) # partition_count=task_num, # partition_index=task_index) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) dataset = dataset.map( - self._parse_dataframe, - num_parallel_calls=self._data_config.num_parallel_calls) + self._parse_dataframe, + num_parallel_calls=self._data_config.num_parallel_calls + ) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/rtp_input.py b/easy_rec/python/input/rtp_input.py index 9f9679b9e..322719bcc 100644 --- a/easy_rec/python/input/rtp_input.py +++ b/easy_rec/python/input/rtp_input.py @@ -1,13 +1,11 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.input.input import Input from easy_rec.python.ops.gen_str_avx_op import str_split_by_chr -from easy_rec.python.utils.check_utils import check_split -from easy_rec.python.utils.check_utils import check_string_to_number +from easy_rec.python.utils.check_utils import check_split, check_string_to_number # NOQA from easy_rec.python.utils.input_utils import string_to_number from easy_rec.python.utils.tf_utils import get_tf_type @@ -30,24 +28,29 @@ class RTPInput(Input): columns are labels """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(RTPInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) - logging.info('input_fields: %s label_fields: %s' % - (','.join(self._input_fields), ','.join(self._label_fields))) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(RTPInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) + logging.info( + 'input_fields: %s label_fields: %s' % + (','.join(self._input_fields), ','.join(self._label_fields)) + ) self._rtp_separator = self._data_config.rtp_separator if not isinstance(self._rtp_separator, str): self._rtp_separator = self._rtp_separator.encode('utf-8') self._selected_cols = [ - int(x) for x in self._data_config.selected_cols.split(',') + int(x) for x in self._data_config.selected_cols.split(',') ] self._num_cols = -1 self._feature_col_id = self._selected_cols[-1] @@ -57,18 +60,21 @@ def _parse_csv(self, line): record_defaults = ['' for i in range(self._num_cols)] # the actual features are in one single column - record_defaults[self._feature_col_id] = self._data_config.separator.join([ - str(self.get_type_defaults(t, v)) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x not in self._label_fields - ]) + record_defaults[self._feature_col_id] = self._data_config.separator.join( + [ + str(self.get_type_defaults(t, v)) for x, t, v in zip( + self._input_fields, self._input_field_types, + self._input_field_defaults + ) if x not in self._label_fields + ] + ) check_list = [ - tf.py_func( - check_split, [line, self._rtp_separator, - len(record_defaults)], - Tout=tf.bool) + tf.py_func( + check_split, [line, self._rtp_separator, + len(record_defaults)], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): fields = tf.string_split(line, self._rtp_separator, skip_empty=False) @@ -83,7 +89,7 @@ def _parse_csv(self, line): tf_type = get_tf_type(ftype) if field.dtype in [tf.string]: check_list = [ - tf.py_func(check_string_to_number, [field, fname], Tout=tf.bool) + tf.py_func(check_string_to_number, [field, fname], Tout=tf.bool) ] if self._check_mode else [] with tf.control_dependencies(check_list): field = tf.string_to_number(field, tf_type) @@ -91,33 +97,35 @@ def _parse_csv(self, line): # only for features, labels excluded record_types = [ - t for x, t in zip(self._input_fields, self._input_field_types) - if x not in self._label_fields + t for x, t in zip(self._input_fields, self._input_field_types) + if x not in self._label_fields ] # assume that the last field is the generated feature column print('field_delim = %s' % self._data_config.separator) feature_str = fields[:, self._feature_col_id] check_list = [ - tf.py_func( - check_split, - [feature_str, self._data_config.separator, - len(record_types)], - Tout=tf.bool) + tf.py_func( + check_split, + [feature_str, self._data_config.separator, + len(record_types)], + Tout=tf.bool + ) ] if self._check_mode else [] with tf.control_dependencies(check_list): fields = str_split_by_chr( - feature_str, self._data_config.separator, skip_empty=False) + feature_str, self._data_config.separator, skip_empty=False + ) tmp_fields = tf.reshape(fields.values, [-1, len(record_types)]) rtp_record_defaults = [ - str(self.get_type_defaults(t, v)) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x not in self._label_fields + str(self.get_type_defaults(t, v)) for x, t, v in zip( + self._input_fields, self._input_field_types, self._input_field_defaults + ) if x not in self._label_fields ] fields = [] for i in range(len(record_types)): - field = string_to_number(tmp_fields[:, i], record_types[i], - rtp_record_defaults[i], i) + field = string_to_number( + tmp_fields[:, i], record_types[i], rtp_record_defaults[i], i + ) fields.append(field) field_keys = [x for x in self._input_fields if x not in self._label_fields] @@ -152,25 +160,28 @@ def _build(self, mode, params): logging.info('num selected cols = %d' % self._num_cols) record_defaults = [ - self.get_type_defaults(t, v) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x in self._label_fields + self.get_type_defaults(t, v) for x, t, v in zip( + self._input_fields, self._input_field_types, self._input_field_defaults + ) if x in self._label_fields ] # the features are in one single column record_defaults.append( - self._data_config.separator.join([ - str(self.get_type_defaults(t, v)) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x not in self._label_fields - ])) + self._data_config.separator.join( + [ + str(self.get_type_defaults(t, v)) for x, t, v in zip( + self._input_fields, self._input_field_types, + self._input_field_defaults + ) if x not in self._label_fields + ] + ) + ) num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -184,42 +195,47 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - tf.data.TextLineDataset, - cycle_length=parallel_num, - num_parallel_calls=parallel_num) + tf.data.TextLineDataset, + cycle_length=parallel_num, + num_parallel_calls=parallel_num + ) if not self._data_config.file_shard: dataset = self._safe_shard(dataset) if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(batch_size=self._data_config.batch_size) dataset = dataset.map( - self._parse_csv, - num_parallel_calls=self._data_config.num_parallel_calls) + self._parse_csv, num_parallel_calls=self._data_config.num_parallel_calls + ) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/rtp_input_v2.py b/easy_rec/python/input/rtp_input_v2.py index 32e841f8d..60b792013 100644 --- a/easy_rec/python/input/rtp_input_v2.py +++ b/easy_rec/python/input/rtp_input_v2.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.input.input import Input @@ -17,17 +16,20 @@ class RTPInputV2(Input): the original rtp format, it is not efficient for training, the performance have to be tuned. """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(RTPInputV2, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(RTPInputV2, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) def _parse_rtp(self, lines): tf_types = [tf.string for x in self._input_field_types] @@ -40,44 +42,51 @@ def _parse_one_line_tf(line): keys = field_vals[:, 0] vals = field_vals[:, 1] temp_vals = [ - str( - self.get_type_defaults(self._input_field_types[i], - self._input_field_defaults[i])) - for i in range(len(self._input_fields)) + str( + self.get_type_defaults( + self._input_field_types[i], self._input_field_defaults[i] + ) + ) for i in range(len(self._input_fields)) ] for i, key in enumerate(self._input_fields): msk = tf.equal(key, keys) val = tf.boolean_mask(vals, msk) - def_val = self.get_type_defaults(self._input_field_types[i], - self._input_field_defaults[i]) + def_val = self.get_type_defaults( + self._input_field_types[i], self._input_field_defaults[i] + ) temp_vals[i] = tf.cond( - tf.reduce_any(msk), lambda: tf.reduce_join(val, separator=','), - lambda: tf.constant(str(def_val))) + tf.reduce_any(msk), lambda: tf.reduce_join(val, separator=','), + lambda: tf.constant(str(def_val)) + ) return temp_vals fields = tf.map_fn( - _parse_one_line_tf, - lines, - tf_types, - parallel_iterations=64, - name='parse_one_line_tf_map_fn') + _parse_one_line_tf, + lines, + tf_types, + parallel_iterations=64, + name='parse_one_line_tf_map_fn' + ) def _convert(x, target_type, name): if target_type in [DatasetConfig.FLOAT, DatasetConfig.DOUBLE]: return tf.string_to_number( - x, tf.float32, name='convert_input_flt32/%s' % name) + x, tf.float32, name='convert_input_flt32/%s' % name + ) elif target_type == DatasetConfig.INT32: return tf.string_to_number( - x, tf.int32, name='convert_input_int32/%s' % name) + x, tf.int32, name='convert_input_int32/%s' % name + ) elif target_type == DatasetConfig.INT64: return tf.string_to_number( - x, tf.int64, name='convert_input_int64/%s' % name) + x, tf.int64, name='convert_input_int64/%s' % name + ) return x inputs = { - self._input_fields[x]: _convert(fields[x], self._input_field_types[x], - self._input_fields[x]) - for x in self._effective_fids + self._input_fields[x]: + _convert(fields[x], self._input_field_types[x], self._input_fields[x]) + for x in self._effective_fids } for x in self._label_fids: @@ -94,8 +103,9 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -109,37 +119,43 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - tf.data.TextLineDataset, - cycle_length=parallel_num, - num_parallel_calls=parallel_num) + tf.data.TextLineDataset, + cycle_length=parallel_num, + num_parallel_calls=parallel_num + ) if not self._data_config.file_shard: dataset = self._safe_shard(dataset) if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) dataset = dataset.map( - self._parse_rtp, num_parallel_calls=num_parallel_calls) + self._parse_rtp, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + map_func=self._preprocess, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/tfrecord_input.py b/easy_rec/python/input/tfrecord_input.py index b9e25eef0..a3c130653 100644 --- a/easy_rec/python/input/tfrecord_input.py +++ b/easy_rec/python/input/tfrecord_input.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.input.input import Input @@ -13,25 +12,30 @@ class TFRecordInput(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(TFRecordInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None + ): + super(TFRecordInput, self).__init__( + data_config, feature_config, input_path, task_index, task_num, + check_mode, pipeline_config + ) self.feature_desc = {} - for x, t, d in zip(self._input_fields, self._input_field_types, - self._input_field_defaults): + for x, t, d in zip( + self._input_fields, self._input_field_types, self._input_field_defaults + ): d = self.get_type_defaults(t, d) t = get_tf_type(t) self.feature_desc[x] = tf.FixedLenFeature( - dtype=t, shape=1, default_value=d) + dtype=t, shape=1, default_value=d + ) def _parse_tfrecord(self, example): try: @@ -51,8 +55,9 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls data_compression_type = self._data_config.data_compression_type if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.shuffle: # shuffle input files @@ -61,36 +66,43 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TFRecordDataset( - x, compression_type=data_compression_type), - cycle_length=parallel_num, - num_parallel_calls=parallel_num) + lambda x: tf.data. + TFRecordDataset(x, compression_type=data_compression_type), + cycle_length=parallel_num, + num_parallel_calls=parallel_num + ) dataset = dataset.shard(self._task_num, self._task_index) if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True + ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % - (len(file_paths), ','.join(file_paths))) + logging.info( + 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) + ) dataset = tf.data.TFRecordDataset( - file_paths, compression_type=data_compression_type) + file_paths, compression_type=data_compression_type + ) dataset = dataset.repeat(1) dataset = dataset.map( - self._parse_tfrecord, num_parallel_calls=num_parallel_calls) + self._parse_tfrecord, num_parallel_calls=num_parallel_calls + ) dataset = dataset.batch(self._data_config.batch_size) dataset = dataset.prefetch(buffer_size=self._prefetch_size) dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + map_func=self._preprocess, num_parallel_calls=num_parallel_calls + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: - (self._get_features(x), self._get_labels(x))) + dataset = dataset.map( + lambda x: (self._get_features(x), self._get_labels(x)) + ) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/layers/backbone.py b/easy_rec/python/layers/backbone.py index e77ea1da5..927aa19f8 100644 --- a/easy_rec/python/layers/backbone.py +++ b/easy_rec/python/layers/backbone.py @@ -1,14 +1,12 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import six import tensorflow as tf from google.protobuf import struct_pb2 from easy_rec.python.layers.common_layers import EnhancedInputLayer -from easy_rec.python.layers.keras import MLP -from easy_rec.python.layers.keras import EmbeddingLayer +from easy_rec.python.layers.keras import MLP, EmbeddingLayer from easy_rec.python.layers.utils import Parameter from easy_rec.python.protos import backbone_pb2 from easy_rec.python.utils.dag import DAG @@ -60,13 +58,15 @@ def __init__(self, config, features, input_layer, l2_reg=None): layer = block.WhichOneof('layer') if layer in {'input_layer', 'raw_input', 'embedding_layer'}: if len(block.inputs) != 1: - raise ValueError('input layer `%s` takes only one input' % block.name) + raise ValueError( + 'input layer `%s` takes only one input' % block.name + ) one_input = block.inputs[0] name = one_input.WhichOneof('name') if name != 'feature_group_name': raise KeyError( - '`feature_group_name` should be set for input layer: ' + - block.name) + '`feature_group_name` should be set for input layer: ' + block.name + ) group = one_input.feature_group_name if not input_layer.has_group(group): raise KeyError('invalid feature group name: ' + group) @@ -84,21 +84,26 @@ def __init__(self, config, features, input_layer, l2_reg=None): self._name_to_layer[block.name] = input_fn else: if layer == 'input_layer': - input_fn = EnhancedInputLayer(self._input_layer, self._features, - group, reuse) + input_fn = EnhancedInputLayer( + self._input_layer, self._features, group, reuse + ) input_feature_groups[group] = input_fn elif layer == 'raw_input': - input_fn = self._input_layer.get_raw_features(self._features, group) + input_fn = self._input_layer.get_raw_features( + self._features, group + ) input_feature_groups[group] = input_fn else: # embedding_layer inputs, vocab, weights = self._input_layer.get_bucketized_features( - self._features, group) + self._features, group + ) block.embedding_layer.vocab_size = vocab params = Parameter.make_from_pb(block.embedding_layer) input_fn = EmbeddingLayer(params, block.name) input_feature_groups[group] = (inputs, vocab, weights) - logging.info('add an embedding layer %s with vocab size %d', - block.name, vocab) + logging.info( + 'add an embedding layer %s with vocab size %d', block.name, vocab + ) self._name_to_layer[block.name] = input_fn else: self.define_layers(layer, block, block.name, reuse) @@ -167,21 +172,24 @@ def __init__(self, config, features, input_layer, l2_reg=None): num_pkg_input += 1 else: raise KeyError( - 'invalid input name `%s`, must be the name of either a feature group or an another block' - % iname) + 'invalid input name `%s`, must be the name of either a feature group or an another block' + % iname + ) num_groups = len(input_feature_groups) assert num_pkg_input > 0 or num_groups > 0, 'there must be at least one input layer/feature group' if len(config.concat_blocks) == 0 and len(config.output_blocks) == 0: leaf = self._dag.all_leaves() logging.warning( - '%s has no `concat_blocks` or `output_blocks`, try to concat all leaf blocks: %s' - % (config.name, ','.join(leaf))) + '%s has no `concat_blocks` or `output_blocks`, try to concat all leaf blocks: %s' + % (config.name, ','.join(leaf)) + ) self._config.concat_blocks.extend(leaf) Package.__packages[self._config.name] = self - logging.info('%s layers: %s' % - (config.name, ','.join(self._name_to_layer.keys()))) + logging.info( + '%s layers: %s' % (config.name, ','.join(self._name_to_layer.keys())) + ) def define_layers(self, layer, layer_cnf, name, reuse): if layer == 'keras_layer': @@ -232,8 +240,9 @@ def block_input(self, config, block_outputs, training=None, **kwargs): pkg_input = block_outputs[pkg_input_name] else: if pkg_input_name not in Package.__packages: - raise KeyError('package name `%s` does not exists' % - pkg_input_name) + raise KeyError( + 'package name `%s` does not exists' % pkg_input_name + ) inner_package = Package.__packages[pkg_input_name] pkg_input = inner_package(training) if input_node.HasField('package_input_fn'): @@ -294,7 +303,9 @@ def call(self, is_training, **kwargs): output = self.block_input(config, block_outputs, is_training, **kwargs) for i, layer in enumerate(config.layers): name_i = '%s_l%d' % (block, i) - output = self.call_layer(output, layer, name_i, is_training, **kwargs) + output = self.call_layer( + output, layer, name_i, is_training, **kwargs + ) block_outputs[block] = output continue # just one of layer @@ -318,8 +329,9 @@ def call(self, is_training, **kwargs): block_outputs[block] = input_fn([inputs, weights], is_training) else: with tf.name_scope(block + '_input'): - inputs = self.block_input(config, block_outputs, is_training, - **kwargs) + inputs = self.block_input( + config, block_outputs, is_training, **kwargs + ) output = self.call_layer(inputs, config, block, is_training, **kwargs) block_outputs[block] = output @@ -350,8 +362,9 @@ def call(self, is_training, **kwargs): def load_keras_layer(self, layer_conf, name, reuse=None): layer_cls, customize = load_keras_layer(layer_conf.class_name) if layer_cls is None: - raise ValueError('Invalid keras layer class name: ' + - layer_conf.class_name) + raise ValueError( + 'Invalid keras layer class name: ' + layer_conf.class_name + ) param_type = layer_conf.WhichOneof('params') if customize: @@ -386,14 +399,17 @@ def load_keras_layer(self, layer_conf, name, reuse=None): assert param_type == 'st_params', 'internal keras layer only support st_params' try: kwargs = convert_to_dict(layer_conf.st_params) - logging.info('call %s layer with params %r' % - (layer_conf.class_name, kwargs)) + logging.info( + 'call %s layer with params %r' % (layer_conf.class_name, kwargs) + ) layer = layer_cls(name=name, **kwargs) except TypeError as e: logging.warning(e) args = map(format_value, layer_conf.st_params.values()) - logging.info('try to call %s layer with params %r' % - (layer_conf.class_name, args)) + logging.info( + 'try to call %s layer with params %r' % + (layer_conf.class_name, args) + ) layer = layer_cls(*args, name=name) return layer, customize @@ -542,8 +558,9 @@ def merge_inputs(inputs, axis=-1, msg=''): if any(map(lambda x: type(x) == list, inputs)): logging.warning('%s: try to merge inputs into list' % msg) - return reduce(lambda x, y: x + y, - [e if type(e) == list else [e] for e in inputs]) + return reduce( + lambda x, y: x + y, [e if type(e) == list else [e] for e in inputs] + ) if axis != -1: logging.info('concat inputs %s axis=%d' % (msg, axis)) diff --git a/easy_rec/python/layers/capsule_layer.py b/easy_rec/python/layers/capsule_layer.py index 22c8363cd..1fac8e797 100644 --- a/easy_rec/python/layers/capsule_layer.py +++ b/easy_rec/python/layers/capsule_layer.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import numpy as np import tensorflow as tf @@ -41,8 +40,10 @@ def squash(self, inputs): return scale_factor * inputs def _build_capsule_simi(self, high_capsules, capsule_num): - high_capsule_mask = tf.sequence_mask(capsule_num, - tf.shape(high_capsules)[1]) + high_capsule_mask = tf.sequence_mask( + capsule_num, + tf.shape(high_capsules)[1] + ) high_capsules = high_capsules * tf.to_float(high_capsule_mask[:, :, None]) high_capsules = tf.nn.l2_normalize(high_capsules, axis=-1) sum_sqr = tf.square(tf.reduce_sum(high_capsules, axis=1)) @@ -69,27 +70,32 @@ def __call__(self, seq_feas, seq_lens): """ # pad or clip to max_seq_len seq_feas = tf.cond( - tf.greater(tf.shape(seq_feas)[1], self._max_seq_len), - lambda: seq_feas[:, :self._max_seq_len, :], lambda: tf.cond( - tf.less(tf.shape(seq_feas)[1], self._max_seq_len), lambda: tf.pad( - seq_feas, [[0, 0], [ - 0, self._max_seq_len - tf.shape(seq_feas)[1] - ], [0, 0]]), lambda: seq_feas)) + tf.greater(tf.shape(seq_feas)[1], self._max_seq_len), + lambda: seq_feas[:, :self._max_seq_len, :], lambda: tf.cond( + tf.less(tf.shape(seq_feas)[1], self._max_seq_len), lambda: tf.pad( + seq_feas, + [[0, 0], [0, self._max_seq_len - tf.shape(seq_feas)[1]], [0, 0]] + ), lambda: seq_feas + ) + ) seq_lens = tf.minimum(seq_lens, self._max_seq_len) batch_size = tf.shape(seq_lens)[0] # max_seq_len x max_num_high_capsule(sh) if self._is_training: routing_logits = tf.truncated_normal( - [batch_size, self._max_seq_len, self._max_k], - stddev=self._routing_logits_stddev) + [batch_size, self._max_seq_len, self._max_k], + stddev=self._routing_logits_stddev + ) else: np.random.seed(28) routing_logits = tf.constant( - np.random.uniform( - high=self._routing_logits_stddev, - size=[self._max_seq_len, self._max_k]), - dtype=tf.float32) + np.random.uniform( + high=self._routing_logits_stddev, + size=[self._max_seq_len, self._max_k] + ), + dtype=tf.float32 + ) routing_logits = tf.tile(routing_logits[None, :, :], [batch_size, 1, 1]) routing_logits = tf.stop_gradient(routing_logits) # batch_size x max_seq_len x max_k(bsh) @@ -97,7 +103,8 @@ def __call__(self, seq_feas, seq_lens): # map low capsule features to high capsule features: # low_fea_dim x high_dim(de) bilinear_matrix = tf.get_variable( - dtype=tf.float32, shape=[low_fea_dim, self._high_dim], name='capsule/S') + dtype=tf.float32, shape=[low_fea_dim, self._high_dim], name='capsule/S' + ) # map sequence feature to high dimensional space seq_feas_high = tf.tensordot(seq_feas, bilinear_matrix, axes=1) seq_feas_high_stop = tf.stop_gradient(seq_feas_high) @@ -108,11 +115,12 @@ def __call__(self, seq_feas, seq_lens): num_high_capsules = tf.zeros_like(seq_lens, dtype=tf.int32) + self._max_k else: logging.info( - 'will use log(seq_len) number of capsules, max_capsules: %d' % - self._max_k) + 'will use log(seq_len) number of capsules, max_capsules: %d' % + self._max_k + ) num_high_capsules = tf.maximum( - 1, tf.minimum(self._max_k, - tf.to_int32(tf.log(tf.to_float(seq_lens))))) + 1, tf.minimum(self._max_k, tf.to_int32(tf.log(tf.to_float(seq_lens)))) + ) # batch_size x max_seq_len(bs) mask = tf.sequence_mask(seq_lens, self._max_seq_len) @@ -139,19 +147,27 @@ def __call__(self, seq_feas, seq_lens): # batch_size x max_k x high_dim(bse,bsh->bhe) high_capsules = tf.einsum( - 'bse, bsh->bhe', seq_feas_high_stop - if iter_id + 1 < self._num_iters else seq_feas_high, routing_logits) + 'bse, bsh->bhe', + seq_feas_high_stop if iter_id + 1 < self._num_iters else seq_feas_high, + routing_logits + ) if iter_id + 1 == self._num_iters: - capsule_simi = self._build_capsule_simi(high_capsules, - num_high_capsules) + capsule_simi = self._build_capsule_simi( + high_capsules, num_high_capsules + ) tf.summary.scalar('caspule/simi_%d' % iter_id, capsule_simi) - tf.summary.scalar('capsule/before_squash', - tf.reduce_mean(tf.norm(high_capsules, axis=-1))) + tf.summary.scalar( + 'capsule/before_squash', + tf.reduce_mean(tf.norm(high_capsules, axis=-1)) + ) high_capsules = self.squash(high_capsules) - tf.summary.scalar('capsule/after_squash', - tf.reduce_mean(tf.norm(high_capsules, axis=-1))) - capsule_simi_final = self._build_capsule_simi(high_capsules, - num_high_capsules) + tf.summary.scalar( + 'capsule/after_squash', + tf.reduce_mean(tf.norm(high_capsules, axis=-1)) + ) + capsule_simi_final = self._build_capsule_simi( + high_capsules, num_high_capsules + ) tf.summary.scalar('caspule/simi_final', capsule_simi_final) break @@ -162,13 +178,16 @@ def __call__(self, seq_feas, seq_lens): # batch_size x max_seq_len x max_k(bse, bhe->bsh) if self._routing_logits_scale > 0: if iter_id == 0: - logging.info('routing_logits_scale = %.2f' % - self._routing_logits_scale) - routing_logits = tf.einsum('bse, bhe->bsh', seq_feas_high_norm, - high_capsules) * self._routing_logits_scale + logging.info( + 'routing_logits_scale = %.2f' % self._routing_logits_scale + ) + routing_logits = tf.einsum( + 'bse, bhe->bsh', seq_feas_high_norm, high_capsules + ) * self._routing_logits_scale else: - routing_logits = tf.einsum('bse, bhe->bsh', seq_feas_high_stop, - high_capsules) + routing_logits = tf.einsum( + 'bse, bhe->bsh', seq_feas_high_stop, high_capsules + ) # zero paddings high_capsule_mask = tf.sequence_mask(num_high_capsules, self._max_k) diff --git a/easy_rec/python/layers/cmbf.py b/easy_rec/python/layers/cmbf.py index 37b27a288..96a7fcc7e 100644 --- a/easy_rec/python/layers/cmbf.py +++ b/easy_rec/python/layers/cmbf.py @@ -2,8 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import dnn -from easy_rec.python.layers import multihead_cross_attention +from easy_rec.python.layers import dnn, multihead_cross_attention from easy_rec.python.utils.shape_utils import get_shape_list if tf.__version__ >= '2.0': @@ -18,8 +17,9 @@ class CMBF(object): https://www.mdpi.com/1424-8220/21/16/5275 """ - def __init__(self, model_config, feature_configs, features, cmbf_config, - input_layer): + def __init__( + self, model_config, feature_configs, features, cmbf_config, input_layer + ): self._model_config = cmbf_config has_feature = False @@ -34,7 +34,8 @@ def __init__(self, model_config, feature_configs, features, cmbf_config, self._txt_seq_features = None if input_layer.has_group('text'): self._txt_seq_features, _, _ = input_layer( - features, 'text', is_combine=False) + features, 'text', is_combine=False + ) has_feature = True self._other_features = None if input_layer.has_group('other'): # e.g. statistical feature @@ -51,17 +52,20 @@ def __init__(self, model_config, feature_configs, features, cmbf_config, self._general_feature_num = len(fea_group.feature_names) general_feature_names = set(fea_group.feature_names) assert self._general_feature_num == len(general_feature_names), ( - 'there are duplicate features in `general` feature group') + 'there are duplicate features in `general` feature group' + ) elif fea_group.group_name == 'image': self._img_feature_num = len(fea_group.feature_names) img_feature_names = set(fea_group.feature_names) assert self._img_feature_num == len(img_feature_names), ( - 'there are duplicate features in `image` feature group') + 'there are duplicate features in `image` feature group' + ) elif fea_group.group_name == 'text': txt_seq_feature_names = set(fea_group.feature_names) self._txt_feature_num = len(fea_group.feature_names) assert self._txt_feature_num == len(txt_seq_feature_names), ( - 'there are duplicate features in `text` feature group') + 'there are duplicate features in `text` feature group' + ) max_seq_len = 0 txt_fea_emb_dim_list = [] @@ -79,40 +83,45 @@ def __init__(self, model_config, feature_configs, features, cmbf_config, txt_fea_emb_dim_list.append(feature_config.embedding_dim) if feature_config.HasField('max_seq_len'): assert feature_config.max_seq_len > 0, ( - 'feature config `max_seq_len` must be greater than 0 for feature: ' - + fea_name) + 'feature config `max_seq_len` must be greater than 0 for feature: ' + + fea_name + ) if feature_config.max_seq_len > max_seq_len: max_seq_len = feature_config.max_seq_len unique_dim_num = len(set(txt_fea_emb_dim_list)) assert unique_dim_num <= 1 and len( - txt_fea_emb_dim_list + txt_fea_emb_dim_list ) == self._txt_feature_num, ( - 'CMBF requires that all `text` feature dimensions must be consistent.') + 'CMBF requires that all `text` feature dimensions must be consistent.' + ) unique_dim_num = len(set(general_emb_dim_list)) assert unique_dim_num <= 1 and len( - general_emb_dim_list + general_emb_dim_list ) == self._general_feature_num, ( - 'CMBF requires that all `general` feature dimensions must be consistent.' + 'CMBF requires that all `general` feature dimensions must be consistent.' ) unique_dim_num = len(set(img_fea_emb_dim_list)) assert unique_dim_num <= 1 and len( - img_fea_emb_dim_list + img_fea_emb_dim_list ) == self._img_feature_num, ( - 'CMBF requires that all `image` feature dimensions must be consistent.') + 'CMBF requires that all `image` feature dimensions must be consistent.' + ) if cmbf_config.use_position_embeddings: assert cmbf_config.max_position_embeddings > 0, ( - 'model config `max_position_embeddings` must be greater than 0. ' - 'It must be set when `use_position_embeddings` is true (default)') + 'model config `max_position_embeddings` must be greater than 0. ' + 'It must be set when `use_position_embeddings` is true (default)' + ) assert cmbf_config.max_position_embeddings >= max_seq_len, ( - 'model config `max_position_embeddings` must be greater than or equal to the maximum of all feature config ' - '`max_seq_len`, which is %d' % max_seq_len) + 'model config `max_position_embeddings` must be greater than or equal to the maximum of all feature config ' + '`max_seq_len`, which is %d' % max_seq_len + ) self._img_emb_size = img_fea_emb_dim_list[0] if img_fea_emb_dim_list else 0 self._txt_emb_size = txt_fea_emb_dim_list[0] if txt_fea_emb_dim_list else 0 self._general_emb_size = general_emb_dim_list[ - 0] if general_emb_dim_list else 0 + 0] if general_emb_dim_list else 0 self._head_num = cmbf_config.multi_head_num self._img_head_num = cmbf_config.image_multi_head_num self._txt_head_num = cmbf_config.text_multi_head_num @@ -122,11 +131,18 @@ def __init__(self, model_config, feature_configs, features, cmbf_config, self._img_self_attention_layer_num = cmbf_config.image_self_attention_layer_num self._txt_self_attention_layer_num = cmbf_config.text_self_attention_layer_num self._cross_modal_layer_num = cmbf_config.cross_modal_layer_num - print('txt_feature_num: {0}, img_feature_num: {1}, txt_seq_feature_num: {2}' - .format(self._general_feature_num, self._img_feature_num, - len(self._txt_seq_features) if self._txt_seq_features else 0)) - print('txt_embedding_size: {0}, img_embedding_size: {1}'.format( - self._txt_emb_size, self._img_emb_size)) + print( + 'txt_feature_num: {0}, img_feature_num: {1}, txt_seq_feature_num: {2}'. + format( + self._general_feature_num, self._img_feature_num, + len(self._txt_seq_features) if self._txt_seq_features else 0 + ) + ) + print( + 'txt_embedding_size: {0}, img_embedding_size: {1}'.format( + self._txt_emb_size, self._img_emb_size + ) + ) if self._img_features is not None: assert self._img_emb_size > 0, '`image` feature dimensions must be greater than 0, set by `raw_input_dim`' @@ -149,11 +165,14 @@ def image_self_attention_tower(self): if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` image_features = tf.reshape( - self._img_features, shape=[-1, self._img_emb_size]) + self._img_features, shape=[-1, self._img_emb_size] + ) image_features = tf.layers.dense( - image_features, hidden_size, name='img_projection') + image_features, hidden_size, name='img_projection' + ) image_features = tf.reshape( - image_features, shape=[-1, img_fea_num, hidden_size]) + image_features, shape=[-1, img_fea_num, hidden_size] + ) return image_features hidden_size = self._img_head_size * self._img_head_num @@ -161,47 +180,57 @@ def image_self_attention_tower(self): if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` image_features = tf.reshape( - self._img_features, shape=[-1, self._img_emb_size]) + self._img_features, shape=[-1, self._img_emb_size] + ) image_features = tf.layers.dense( - image_features, hidden_size, name='img_projection') + image_features, hidden_size, name='img_projection' + ) image_features = tf.reshape( - image_features, shape=[-1, img_fea_num, hidden_size]) + image_features, shape=[-1, img_fea_num, hidden_size] + ) elif img_fea_num == 1: if self._img_patch_num > 1: # image feature dimension: patch_num * emb_size img_fea_num = self._img_patch_num img_emb_size = self._img_emb_size // self._img_patch_num assert img_emb_size * self._img_patch_num == self._img_emb_size, ( - 'image feature dimension must equal to `image_feature_slice_num * embedding_size_per_region`' + 'image feature dimension must equal to `image_feature_slice_num * embedding_size_per_region`' ) self._img_emb_size = img_emb_size if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` image_features = tf.reshape( - self._img_features, shape=[-1, self._img_emb_size]) + self._img_features, shape=[-1, self._img_emb_size] + ) image_features = tf.layers.dense( - image_features, hidden_size, name='img_projection') + image_features, hidden_size, name='img_projection' + ) image_features = tf.reshape( - image_features, shape=[-1, img_fea_num, hidden_size]) + image_features, shape=[-1, img_fea_num, hidden_size] + ) else: img_fea_num = self._model_config.image_feature_dim if img_fea_num != self._img_emb_size: image_features = tf.layers.dense( - image_features, img_fea_num, name='img_projection') + image_features, img_fea_num, name='img_projection' + ) # convert each element of image feature to a feature vector img_mapping_matrix = tf.get_variable( - 'img_map_matrix', [1, img_fea_num, hidden_size], dtype=tf.float32) - image_features = tf.expand_dims(image_features, -1) * img_mapping_matrix + 'img_map_matrix', [1, img_fea_num, hidden_size], dtype=tf.float32 + ) + image_features = tf.expand_dims( + image_features, -1 + ) * img_mapping_matrix img_attention_fea = multihead_cross_attention.transformer_encoder( - image_features, - hidden_size=hidden_size, # head_num * size_per_head - num_hidden_layers=self._img_self_attention_layer_num, - num_attention_heads=self._head_num, - intermediate_size=hidden_size * 4, - hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config - .attention_probs_dropout_prob, - name='image_self_attention' + image_features, + hidden_size=hidden_size, # head_num * size_per_head + num_hidden_layers=self._img_self_attention_layer_num, + num_attention_heads=self._head_num, + intermediate_size=hidden_size * 4, + hidden_dropout_prob=self._model_config.hidden_dropout_prob, + attention_probs_dropout_prob=self._model_config. + attention_probs_dropout_prob, + name='image_self_attention' ) # shape: [batch_size, image_seq_num/image_feature_dim, hidden_size] # print('img_attention_fea:', img_attention_fea.shape) return img_attention_fea @@ -217,17 +246,21 @@ def text_self_attention_tower(self): if self._general_emb_size != hidden_size: # Run a linear projection of `hidden_size` general_features = tf.reshape( - general_features, shape=[-1, self._general_emb_size]) + general_features, shape=[-1, self._general_emb_size] + ) general_features = tf.layers.dense( - general_features, hidden_size, name='txt_projection') + general_features, hidden_size, name='txt_projection' + ) txt_features = tf.reshape( - general_features, shape=[-1, self._general_feature_num, hidden_size]) + general_features, shape=[-1, self._general_feature_num, hidden_size] + ) all_txt_features.append(txt_features) batch_size = tf.shape(txt_features)[0] mask = tf.ones( - shape=tf.stack([batch_size, self._general_feature_num]), - dtype=tf.int32) + shape=tf.stack([batch_size, self._general_feature_num]), + dtype=tf.int32 + ) input_masks.append(mask) input_mask = None @@ -245,46 +278,50 @@ def dynamic_mask(x, max_len): if emb_size != hidden_size: seq_fea = tf.reshape(seq_fea, shape=[-1, emb_size]) seq_fea = tf.layers.dense( - seq_fea, hidden_size, name='txt_seq_projection_%d' % i) + seq_fea, hidden_size, name='txt_seq_projection_%d' % i + ) seq_fea = tf.reshape(seq_fea, shape=[-1, max_seq_len, hidden_size]) seq_fea = multihead_cross_attention.embedding_postprocessor( - seq_fea, - use_token_type=self._model_config.use_token_type, - token_type_ids=tf.ones( - shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * i, - token_type_vocab_size=token_type_vocab_size, - reuse_token_type=tf.AUTO_REUSE, - use_position_embeddings=self._model_config.use_position_embeddings, - max_position_embeddings=self._model_config.max_position_embeddings, - position_embedding_name='position_embeddings_%d' % i, - dropout_prob=self._model_config.text_seq_emb_dropout_prob) + seq_fea, + use_token_type=self._model_config.use_token_type, + token_type_ids=tf. + ones(shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * i, + token_type_vocab_size=token_type_vocab_size, + reuse_token_type=tf.AUTO_REUSE, + use_position_embeddings=self._model_config.use_position_embeddings, + max_position_embeddings=self._model_config.max_position_embeddings, + position_embedding_name='position_embeddings_%d' % i, + dropout_prob=self._model_config.text_seq_emb_dropout_prob + ) all_txt_features.append(seq_fea) input_mask = tf.map_fn( - fn=lambda t: dynamic_mask(t, max_seq_len), - elems=tf.to_int32(seq_len)) + fn=lambda t: dynamic_mask(t, max_seq_len), + elems=tf.to_int32(seq_len) + ) input_masks.append(input_mask) txt_features = tf.concat(all_txt_features, axis=1) input_mask = tf.concat(input_masks, axis=1) attention_mask = multihead_cross_attention.create_attention_mask_from_input_mask( - from_tensor=txt_features, to_mask=input_mask) + from_tensor=txt_features, to_mask=input_mask + ) if txt_features is None: return None, None, None txt_attention_fea = multihead_cross_attention.transformer_encoder( - txt_features, - hidden_size=hidden_size, - num_hidden_layers=self._txt_self_attention_layer_num, - num_attention_heads=self._head_num, - attention_mask=attention_mask, - intermediate_size=hidden_size * 4, - hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config - .attention_probs_dropout_prob, - name='text_self_attention' + txt_features, + hidden_size=hidden_size, + num_hidden_layers=self._txt_self_attention_layer_num, + num_attention_heads=self._head_num, + attention_mask=attention_mask, + intermediate_size=hidden_size * 4, + hidden_dropout_prob=self._model_config.hidden_dropout_prob, + attention_probs_dropout_prob=self._model_config. + attention_probs_dropout_prob, + name='text_self_attention' ) # shape: [batch_size, txt_seq_length, hidden_size] print('txt_attention_fea:', txt_attention_fea.shape) return txt_attention_fea, input_mask, input_masks @@ -296,23 +333,27 @@ def merge_text_embedding(self, txt_embeddings, input_masks): text_seq_emb = [] if self._general_feature_num > 0: - text_emb = tf.slice(txt_embeddings, [0, 0, 0], - [shape[0], self._general_feature_num, shape[2]]) + text_emb = tf.slice( + txt_embeddings, [0, 0, 0], + [shape[0], self._general_feature_num, shape[2]] + ) text_seq_emb.append(text_emb) begin = self._general_feature_num for i in range(len(text_seq_emb), len(input_masks)): size = tf.shape(input_masks[i])[1] - temp_emb = tf.slice(txt_embeddings, [0, begin, 0], - [shape[0], size, shape[2]]) + temp_emb = tf.slice( + txt_embeddings, [0, begin, 0], [shape[0], size, shape[2]] + ) mask = tf.expand_dims(tf.to_float(input_masks[i]), -1) temp_emb = temp_emb * mask # avg pooling emb_sum = tf.reduce_sum( - temp_emb, axis=1, - keepdims=True) # shape: [batch_size, 1, hidden_size] + temp_emb, axis=1, keepdims=True + ) # shape: [batch_size, 1, hidden_size] count = tf.reduce_sum( - mask, axis=1, keepdims=True) # shape: [batch_size, 1, 1] + mask, axis=1, keepdims=True + ) # shape: [batch_size, 1, 1] seq_emb = emb_sum / count # shape: [batch_size, 1, hidden_size] text_seq_emb.append(seq_emb) @@ -341,20 +382,21 @@ def __call__(self, is_training, *args, **kwargs): all_fea = [] if None not in [img_attention_fea, txt_attention_fea]: img_embeddings, txt_embeddings = multihead_cross_attention.cross_attention_tower( - img_attention_fea, - txt_attention_fea, - num_hidden_layers=self._cross_modal_layer_num, - num_attention_heads=self._head_num, - right_input_mask=input_mask, - left_size_per_head=self._model_config.image_cross_head_size, - left_intermediate_size=4 * self._model_config.image_cross_head_size * - self._head_num, - right_size_per_head=self._model_config.text_cross_head_size, - right_intermediate_size=4 * self._model_config.text_cross_head_size * - self._head_num, - hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config - .attention_probs_dropout_prob) + img_attention_fea, + txt_attention_fea, + num_hidden_layers=self._cross_modal_layer_num, + num_attention_heads=self._head_num, + right_input_mask=input_mask, + left_size_per_head=self._model_config.image_cross_head_size, + left_intermediate_size=4 * self._model_config.image_cross_head_size * + self._head_num, + right_size_per_head=self._model_config.text_cross_head_size, + right_intermediate_size=4 * self._model_config.text_cross_head_size * + self._head_num, + hidden_dropout_prob=self._model_config.hidden_dropout_prob, + attention_probs_dropout_prob=self._model_config. + attention_probs_dropout_prob + ) # img_embeddings shape: [batch_size, image_(region_)num/image_feature_dim, multi_head_num * image_cross_head_size] print('img_embeddings:', img_embeddings.shape) # txt_embeddings shape: [batch_size, general_feature_num + max_txt_seq_len, multi_head_num * text_cross_head_size] @@ -374,14 +416,18 @@ def __call__(self, is_training, *args, **kwargs): elif txt_attention_fea is not None: # only has text tower # shape: [batch_size, (general_feature_num + txt_seq_num) * multi_head_num * text_head_size] - txt_embeddings = self.merge_text_embedding(txt_attention_fea, input_masks) + txt_embeddings = self.merge_text_embedding( + txt_attention_fea, input_masks + ) all_fea = [txt_embeddings] if self._other_features is not None: if self._model_config.HasField('other_feature_dnn'): l2_reg = kwargs['l2_reg'] if 'l2_reg' in kwargs else 0 - other_dnn_layer = dnn.DNN(self._model_config.other_feature_dnn, l2_reg, - 'other_dnn', is_training) + other_dnn_layer = dnn.DNN( + self._model_config.other_feature_dnn, l2_reg, 'other_dnn', + is_training + ) other_fea = other_dnn_layer(self._other_features) all_fea.append(other_fea) # e.g. statistical features else: diff --git a/easy_rec/python/layers/common_layers.py b/easy_rec/python/layers/common_layers.py index 68ecf37f5..7a25916bf 100644 --- a/easy_rec/python/layers/common_layers.py +++ b/easy_rec/python/layers/common_layers.py @@ -11,14 +11,16 @@ tf = tf.compat.v1 -def highway(x, - size=None, - activation=None, - num_layers=1, - scope='highway', - dropout=0.0, - init_gate_bias=-1.0, - reuse=None): +def highway( + x, + size=None, + activation=None, + num_layers=1, + scope='highway', + dropout=0.0, + init_gate_bias=-1.0, + reuse=None +): if isinstance(activation, six.string_types): activation = get_activation(activation) with tf.variable_scope(scope, reuse): @@ -30,25 +32,29 @@ def highway(x, initializer = tf.constant_initializer(init_gate_bias) for i in range(num_layers): T = tf.layers.dense( - x, - size, - activation=tf.sigmoid, - bias_initializer=initializer, - name='gate_%d' % i, - reuse=reuse) + x, + size, + activation=tf.sigmoid, + bias_initializer=initializer, + name='gate_%d' % i, + reuse=reuse + ) H = tf.layers.dense( - x, size, activation=activation, name='activation_%d' % i, reuse=reuse) + x, size, activation=activation, name='activation_%d' % i, reuse=reuse + ) if dropout > 0.0: H = tf.nn.dropout(H, 1.0 - dropout) x = H * T + x * (1.0 - T) return x -def text_cnn(x, - filter_sizes=(3, 4, 5), - num_filters=(128, 64, 64), - scope_name='textcnn', - reuse=False): +def text_cnn( + x, + filter_sizes=(3, 4, 5), + num_filters=(128, 64, 64), + scope_name='textcnn', + reuse=False +): # x: None * step_dim * embed_dim assert len(filter_sizes) == len(num_filters) initializer = tf.variance_scaling_initializer() @@ -60,30 +66,34 @@ def text_cnn(x, with tf.variable_scope(scope_name_i, reuse=reuse): # conv shape: (batch_size, seq_len - filter_size + 1, num_filters) conv = tf.layers.conv1d( - x, - filters=int(num_filter), - kernel_size=int(filter_size), - activation=tf.nn.relu, - name='conv_layer', - reuse=reuse, - kernel_initializer=initializer, - padding='same') + x, + filters=int(num_filter), + kernel_size=int(filter_size), + activation=tf.nn.relu, + name='conv_layer', + reuse=reuse, + kernel_initializer=initializer, + padding='same' + ) pool = tf.reduce_max( - conv, axis=1) # max pooling, shape: (batch_size, num_filters) + conv, axis=1 + ) # max pooling, shape: (batch_size, num_filters) pooled_outputs.append(pool) pool_flat = tf.concat( - pooled_outputs, 1) # shape: (batch_size, num_filters * len(filter_sizes)) + pooled_outputs, 1 + ) # shape: (batch_size, num_filters * len(filter_sizes)) return pool_flat def layer_norm(input_tensor, name=None, reuse=None): """Run layer normalization on the last dimension of the tensor.""" return tf_layer_norm( - inputs=input_tensor, - begin_norm_axis=-1, - begin_params_axis=-1, - reuse=reuse, - scope=name) + inputs=input_tensor, + begin_norm_axis=-1, + begin_params_axis=-1, + reuse=reuse, + scope=name + ) class EnhancedInputLayer(object): @@ -106,7 +116,7 @@ def __call__(self, config, is_training, **kwargs): if config.do_batch_norm and config.do_layer_norm: raise ValueError( - 'can not do batch norm and layer norm for input layer at the same time' + 'can not do batch norm and layer norm for input layer at the same time' ) with tf.name_scope(self.name): return self.call(config, is_training) @@ -115,7 +125,8 @@ def build(self, config, training): self.built = True combine = not config.output_seq_and_normal_feature self.inputs = self._input_layer( - self._feature_dict, self._group_name, is_combine=combine) + self._feature_dict, self._group_name, is_combine=combine + ) if config.output_seq_and_normal_feature: seq_feature_and_len, _, target_features = self.inputs seq_len = seq_feature_and_len[0][1] @@ -126,7 +137,8 @@ def build(self, config, training): else: target_features = None assert len( - seq_features) > 0, '[%s] sequence feature is empty' % self.name + seq_features + ) > 0, '[%s] sequence feature is empty' % self.name seq_features = tf.concat(seq_features, axis=-1) self.inputs = seq_features, seq_len, target_features self.reset(config, training) @@ -151,10 +163,12 @@ def call(self, config, training): mask = self.bern.sample(num_features) elif do_bn: features = tf.layers.batch_normalization( - features, training=training, reuse=self._reuse) + features, training=training, reuse=self._reuse + ) elif do_ln: features = layer_norm( - features, name=self._group_name + '_features', reuse=self._reuse) + features, name=self._group_name + '_features', reuse=self._reuse + ) output_feature_list = config.output_2d_tensor_and_feature_list output_feature_list = output_feature_list or config.only_output_feature_list @@ -166,7 +180,8 @@ def call(self, config, training): fea = feature_list[i] if do_bn: fea = tf.layers.batch_normalization( - fea, training=training, reuse=self._reuse) + fea, training=training, reuse=self._reuse + ) elif do_ln: ln_name = self._group_name + 'f_%d' % i fea = layer_norm(fea, name=ln_name, reuse=self._reuse) diff --git a/easy_rec/python/layers/dnn.py b/easy_rec/python/layers/dnn.py index 7a57f5661..78290ba11 100644 --- a/easy_rec/python/layers/dnn.py +++ b/easy_rec/python/layers/dnn.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.utils.activation import get_activation @@ -12,13 +11,15 @@ class DNN: - def __init__(self, - dnn_config, - l2_reg, - name='dnn', - is_training=False, - last_layer_no_activation=False, - last_layer_no_batch_norm=False): + def __init__( + self, + dnn_config, + l2_reg, + name='dnn', + is_training=False, + last_layer_no_activation=False, + last_layer_no_batch_norm=False + ): """Initializes a `DNN` Layer. Args: @@ -35,7 +36,8 @@ def __init__(self, self._is_training = is_training logging.info('dnn activation function = %s' % self._config.activation) self.activation = get_activation( - self._config.activation, training=is_training) + self._config.activation, training=is_training + ) self._last_layer_no_activation = last_layer_no_activation self._last_layer_no_batch_norm = last_layer_no_batch_norm @@ -55,28 +57,33 @@ def __call__(self, deep_fea, hidden_layer_feature_output=False): hidden_feature_dict = {} for i, unit in enumerate(self.hidden_units): deep_fea = tf.layers.dense( - inputs=deep_fea, - units=unit, - kernel_regularizer=self._l2_reg, - activation=None, - name='%s/dnn_%d' % (self._name, i)) - if self._config.use_bn and ((i + 1 < hidden_units_len) or - not self._last_layer_no_batch_norm): + inputs=deep_fea, + units=unit, + kernel_regularizer=self._l2_reg, + activation=None, + name='%s/dnn_%d' % (self._name, i) + ) + if self._config.use_bn and ( + (i + 1 < hidden_units_len) or not self._last_layer_no_batch_norm + ): deep_fea = tf.layers.batch_normalization( - deep_fea, - training=self._is_training, - trainable=True, - name='%s/dnn_%d/bn' % (self._name, i)) + deep_fea, + training=self._is_training, + trainable=True, + name='%s/dnn_%d/bn' % (self._name, i) + ) if (i + 1 < hidden_units_len) or not self._last_layer_no_activation: deep_fea = self.activation( - deep_fea, name='%s/dnn_%d/act' % (self._name, i)) + deep_fea, name='%s/dnn_%d/act' % (self._name, i) + ) if len(self.dropout_ratio) > 0 and self._is_training: assert self.dropout_ratio[ - i] < 1, 'invalid dropout_ratio: %.3f' % self.dropout_ratio[i] + i] < 1, 'invalid dropout_ratio: %.3f' % self.dropout_ratio[i] deep_fea = tf.nn.dropout( - deep_fea, - keep_prob=1 - self.dropout_ratio[i], - name='%s/%d/dropout' % (self._name, i)) + deep_fea, + keep_prob=1 - self.dropout_ratio[i], + name='%s/%d/dropout' % (self._name, i) + ) if hidden_layer_feature_output: hidden_feature_dict['hidden_layer' + str(i)] = deep_fea diff --git a/easy_rec/python/layers/embed_input_layer.py b/easy_rec/python/layers/embed_input_layer.py index 284593fa0..77b7c97b8 100644 --- a/easy_rec/python/layers/embed_input_layer.py +++ b/easy_rec/python/layers/embed_input_layer.py @@ -10,7 +10,8 @@ class EmbedInputLayer(object): def __init__(self, feature_groups_config, dump_dir=None): self._feature_groups = { - x.group_name: FeatureGroup(x) for x in feature_groups_config + x.group_name: FeatureGroup(x) + for x in feature_groups_config } self._dump_dir = dump_dir diff --git a/easy_rec/python/layers/input_layer.py b/easy_rec/python/layers/input_layer.py index 27bc9bdf4..452009ff1 100644 --- a/easy_rec/python/layers/input_layer.py +++ b/easy_rec/python/layers/input_layer.py @@ -2,26 +2,21 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os -from collections import OrderedDict - import tensorflow as tf +from collections import OrderedDict from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import array_ops, variable_scope from easy_rec.python.compat import regularizers from easy_rec.python.compat.feature_column import feature_column +from easy_rec.python.compat.feature_column.feature_column_v2 import is_embedding_column # NOQA from easy_rec.python.feature_column.feature_column import FeatureColumnParser from easy_rec.python.feature_column.feature_group import FeatureGroup -from easy_rec.python.layers import sequence_feature_layer -from easy_rec.python.layers import variational_dropout_layer +from easy_rec.python.layers import sequence_feature_layer, variational_dropout_layer # NOQA from easy_rec.python.layers.keras import TextCNN from easy_rec.python.layers.utils import Parameter from easy_rec.python.protos.feature_config_pb2 import WideOrDeep -from easy_rec.python.utils import conditional -from easy_rec.python.utils import shape_utils - -from easy_rec.python.compat.feature_column.feature_column_v2 import is_embedding_column # NOQA +from easy_rec.python.utils import conditional, shape_utils class InputLayer(object): @@ -30,37 +25,41 @@ class InputLayer(object): This class apply feature_columns to input tensors to generate wide features and deep features. """ - def __init__(self, - feature_configs, - feature_groups_config, - variational_dropout_config=None, - wide_output_dim=-1, - ev_params=None, - embedding_regularizer=None, - kernel_regularizer=None, - is_training=False, - is_predicting=False): + def __init__( + self, + feature_configs, + feature_groups_config, + variational_dropout_config=None, + wide_output_dim=-1, + ev_params=None, + embedding_regularizer=None, + kernel_regularizer=None, + is_training=False, + is_predicting=False + ): self._feature_groups = { - x.group_name: FeatureGroup(x) for x in feature_groups_config + x.group_name: FeatureGroup(x) + for x in feature_groups_config } self.sequence_feature_layer = sequence_feature_layer.SequenceFeatureLayer( - feature_configs, feature_groups_config, ev_params, - embedding_regularizer, kernel_regularizer, is_training, is_predicting) + feature_configs, feature_groups_config, ev_params, embedding_regularizer, + kernel_regularizer, is_training, is_predicting + ) self._seq_feature_groups_config = [] for x in feature_groups_config: for y in x.sequence_features: self._seq_feature_groups_config.append(y) self._group_name_to_seq_features = { - x.group_name: x.sequence_features - for x in feature_groups_config - if len(x.sequence_features) > 0 + x.group_name: x.sequence_features + for x in feature_groups_config if len(x.sequence_features) > 0 } wide_and_deep_dict = self.get_wide_deep_dict() self._fc_parser = FeatureColumnParser( - feature_configs, - wide_and_deep_dict, - wide_output_dim, - ev_params=ev_params) + feature_configs, + wide_and_deep_dict, + wide_output_dim, + ev_params=ev_params + ) self._embedding_regularizer = embedding_regularizer self._kernel_regularizer = kernel_regularizer @@ -85,30 +84,35 @@ def get_combined_feature(self, features, group_name, is_dict=False): feature_name_to_output_tensors: dict, feature_name to feature_value, only present when is_dict is True """ feature_name_to_output_tensors = {} - negative_sampler = self._feature_groups[group_name]._config.negative_sampler + negative_sampler = self._feature_groups[group_name + ]._config.negative_sampler place_on_cpu = os.getenv('place_embedding_on_cpu') place_on_cpu = eval(place_on_cpu) if place_on_cpu else False - with conditional(self._is_predicting and place_on_cpu, - ops.device('/CPU:0')): + with conditional( + self._is_predicting and place_on_cpu, ops.device('/CPU:0') + ): concat_features, group_features = self.single_call_input_layer( - features, group_name, feature_name_to_output_tensors) + features, group_name, feature_name_to_output_tensors + ) if group_name in self._group_name_to_seq_features: # for target attention group_seq_arr = self._group_name_to_seq_features[group_name] concat_features, all_seq_fea = self.sequence_feature_layer( - features, - concat_features, - group_seq_arr, - feature_name_to_output_tensors, - negative_sampler=negative_sampler, - scope_name=group_name) + features, + concat_features, + group_seq_arr, + feature_name_to_output_tensors, + negative_sampler=negative_sampler, + scope_name=group_name + ) group_features.extend(all_seq_fea) for col, fea in zip(group_seq_arr, all_seq_fea): feature_name_to_output_tensors['seq_fea/' + col.group_name] = fea all_seq_fea = array_ops.concat(all_seq_fea, axis=-1) - concat_features = array_ops.concat([concat_features, all_seq_fea], - axis=-1) + concat_features = array_ops.concat( + [concat_features, all_seq_fea], axis=-1 + ) if is_dict: return concat_features, group_features, feature_name_to_output_tensors else: @@ -126,7 +130,8 @@ def get_plain_feature(self, features, group_name): group_features: list of features """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, ','.join([x for x in self._feature_groups]) + ) feature_group = self._feature_groups[group_name] group_columns, _ = feature_group.select_columns(self._fc_parser) @@ -135,10 +140,11 @@ def get_plain_feature(self, features, group_name): cols_to_output_tensors = OrderedDict() output_features = feature_column.input_layer( - features, - group_columns, - cols_to_output_tensors=cols_to_output_tensors, - is_training=self._is_training) + features, + group_columns, + cols_to_output_tensors=cols_to_output_tensors, + is_training=self._is_training + ) group_features = [cols_to_output_tensors[x] for x in group_columns] embedding_reg_lst = [] @@ -148,7 +154,8 @@ def get_plain_feature(self, features, group_name): if self._embedding_regularizer is not None and len(embedding_reg_lst) > 0: regularizers.apply_regularization( - self._embedding_regularizer, weights_list=embedding_reg_lst) + self._embedding_regularizer, weights_list=embedding_reg_lst + ) return output_features, group_features def get_sequence_feature(self, features, group_name): @@ -164,11 +171,13 @@ def get_sequence_feature(self, features, group_name): 1d sequence length tensor. """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, ','.join([x for x in self._feature_groups]) + ) if self._variational_dropout_config is not None: raise ValueError( - 'variational dropout is not supported in not combined mode now.') + 'variational dropout is not supported in not combined mode now.' + ) feature_group = self._feature_groups[group_name] _, group_seq_columns = feature_group.select_columns(self._fc_parser) @@ -177,18 +186,21 @@ def get_sequence_feature(self, features, group_name): builder = feature_column._LazyBuilder(features) seq_features = [] for fc in group_seq_columns: - with variable_scope.variable_scope('input_layer/' + - fc.categorical_column.name): + with variable_scope.variable_scope( + 'input_layer/' + fc.categorical_column.name + ): tmp_embedding, tmp_seq_len = fc._get_sequence_dense_tensor(builder) if fc.max_seq_length > 0: tmp_embedding, tmp_seq_len = shape_utils.truncate_sequence( - tmp_embedding, tmp_seq_len, fc.max_seq_length) + tmp_embedding, tmp_seq_len, fc.max_seq_length + ) seq_features.append((tmp_embedding, tmp_seq_len)) embedding_reg_lst.append(tmp_embedding) if self._embedding_regularizer is not None and len(embedding_reg_lst) > 0: regularizers.apply_regularization( - self._embedding_regularizer, weights_list=embedding_reg_lst) + self._embedding_regularizer, weights_list=embedding_reg_lst + ) return seq_features def get_raw_features(self, features, group_name): @@ -202,7 +214,8 @@ def get_raw_features(self, features, group_name): features: all raw features in list """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, ','.join([x for x in self._feature_groups]) + ) feature_group = self._feature_groups[group_name] return [features[x] for x in feature_group.feature_names] @@ -217,7 +230,8 @@ def get_bucketized_features(self, features, group_name): features: all raw features in list, added feature offset """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, ','.join([x for x in self._feature_groups]) + ) feature_group = self._feature_groups[group_name] offset = 0 values = [] @@ -263,24 +277,26 @@ def __call__(self, features, group_name, is_combine=True, is_dict=False): 1 dimension sequence length tensor. """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, ','.join([x for x in self._feature_groups]) + ) if is_combine: return self.get_combined_feature(features, group_name, is_dict) # return sequence feature in raw format instead of combine them place_on_cpu = os.getenv('place_embedding_on_cpu') place_on_cpu = eval(place_on_cpu) if place_on_cpu else False - with conditional(self._is_predicting and place_on_cpu, - ops.device('/CPU:0')): + with conditional( + self._is_predicting and place_on_cpu, ops.device('/CPU:0') + ): seq_features = self.get_sequence_feature(features, group_name) plain_features, feature_list = self.get_plain_feature( - features, group_name) + features, group_name + ) return seq_features, plain_features, feature_list - def single_call_input_layer(self, - features, - group_name, - feature_name_to_output_tensors=None): + def single_call_input_layer( + self, features, group_name, feature_name_to_output_tensors=None + ): """Get features by group_name. Args: @@ -294,47 +310,54 @@ def single_call_input_layer(self, group_features: list of features """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, ','.join([x for x in self._feature_groups]) + ) feature_group = self._feature_groups[group_name] group_columns, group_seq_columns = feature_group.select_columns( - self._fc_parser) + self._fc_parser + ) cols_to_output_tensors = OrderedDict() output_features = feature_column.input_layer( - features, - group_columns, - cols_to_output_tensors=cols_to_output_tensors, - feature_name_to_output_tensors=feature_name_to_output_tensors, - is_training=self._is_training) + features, + group_columns, + cols_to_output_tensors=cols_to_output_tensors, + feature_name_to_output_tensors=feature_name_to_output_tensors, + is_training=self._is_training + ) embedding_reg_lst = [] builder = feature_column._LazyBuilder(features) seq_features = [] for column in sorted(group_seq_columns, key=lambda x: x.name): with variable_scope.variable_scope( - None, default_name=column._var_scope_name): + None, default_name=column._var_scope_name + ): seq_feature, seq_len = column._get_sequence_dense_tensor(builder) embedding_reg_lst.append(seq_feature) sequence_combiner = column.sequence_combiner if sequence_combiner is None: raise ValueError( - 'sequence_combiner is none, please set sequence_combiner or use TagFeature' + 'sequence_combiner is none, please set sequence_combiner or use TagFeature' ) if sequence_combiner.WhichOneof('combiner') == 'attention': attn_logits = tf.layers.dense( - inputs=seq_feature, - units=1, - kernel_regularizer=self._kernel_regularizer, - use_bias=False, - activation=None, - name='attention') + inputs=seq_feature, + units=1, + kernel_regularizer=self._kernel_regularizer, + use_bias=False, + activation=None, + name='attention' + ) attn_logits = tf.squeeze(attn_logits, axis=-1) attn_logits_padding = tf.ones_like(attn_logits) * (-2**32 + 1) seq_mask = tf.sequence_mask(seq_len) attn_score = tf.nn.softmax( - tf.where(seq_mask, attn_logits, attn_logits_padding)) + tf.where(seq_mask, attn_logits, attn_logits_padding) + ) seq_feature = tf.reduce_sum( - attn_score[:, :, tf.newaxis] * seq_feature, axis=1) + attn_score[:, :, tf.newaxis] * seq_feature, axis=1 + ) seq_features.append(seq_feature) cols_to_output_tensors[column] = seq_feature elif sequence_combiner.WhichOneof('combiner') == 'text_cnn': @@ -346,23 +369,29 @@ def single_call_input_layer(self, else: raise NotImplementedError if self._variational_dropout_config is not None: - features_dimension = OrderedDict([ + features_dimension = OrderedDict( + [ (k.raw_name, int(v.shape[-1])) for k, v in cols_to_output_tensors.items() - ]) + ] + ) concat_features = array_ops.concat( - [output_features] + seq_features, axis=-1) + [output_features] + seq_features, axis=-1 + ) variational_dropout = variational_dropout_layer.VariationalDropoutLayer( - self._variational_dropout_config, - features_dimension, - self._is_training, - name=group_name) + self._variational_dropout_config, + features_dimension, + self._is_training, + name=group_name + ) concat_features = variational_dropout(concat_features) group_features = tf.split( - concat_features, list(features_dimension.values()), axis=-1) + concat_features, list(features_dimension.values()), axis=-1 + ) else: concat_features = array_ops.concat( - [output_features] + seq_features, axis=-1) + [output_features] + seq_features, axis=-1 + ) group_features = [cols_to_output_tensors[x] for x in group_columns] + \ [cols_to_output_tensors[x] for x in group_seq_columns] @@ -372,7 +401,8 @@ def single_call_input_layer(self, embedding_reg_lst.append(val) if embedding_reg_lst: regularizers.apply_regularization( - self._embedding_regularizer, weights_list=embedding_reg_lst) + self._embedding_regularizer, weights_list=embedding_reg_lst + ) return concat_features, group_features def get_wide_deep_dict(self): diff --git a/easy_rec/python/layers/keras/__init__.py b/easy_rec/python/layers/keras/__init__.py index 17e7cdb1c..ede06379e 100644 --- a/easy_rec/python/layers/keras/__init__.py +++ b/easy_rec/python/layers/keras/__init__.py @@ -1,34 +1,16 @@ from .attention import Attention from .auxiliary_loss import AuxiliaryLoss -from .blocks import MLP -from .blocks import Gate -from .blocks import Highway -from .blocks import TextCNN +from .blocks import MLP, Gate, Highway, TextCNN from .bst import BST -from .custom_ops import EditDistance -from .custom_ops import MappedDotProduct -from .custom_ops import OverlapFeature -from .custom_ops import SeqAugmentOps -from .custom_ops import TextNormalize +from .custom_ops import EditDistance, MappedDotProduct, OverlapFeature, SeqAugmentOps, TextNormalize # NOQA from .data_augment import SeqAugment from .din import DIN from .embedding import EmbeddingLayer -from .fibinet import BiLinear -from .fibinet import FiBiNet -from .fibinet import SENet -from .interaction import CIN -from .interaction import FM -from .interaction import Cross -from .interaction import DotInteraction -from .mask_net import MaskBlock -from .mask_net import MaskNet +from .fibinet import BiLinear, FiBiNet, SENet +from .interaction import CIN, FM, Cross, DotInteraction +from .mask_net import MaskBlock, MaskNet from .multi_head_attention import MultiHeadAttention -from .multi_task import AITMTower -from .multi_task import MMoE -from .numerical_embedding import AutoDisEmbedding -from .numerical_embedding import NaryDisEmbedding -from .numerical_embedding import PeriodicEmbedding +from .multi_task import AITMTower, MMoE +from .numerical_embedding import AutoDisEmbedding, NaryDisEmbedding, PeriodicEmbedding # NOQA from .ppnet import PPNet -from .transformer import TextEncoder -from .transformer import TransformerBlock -from .transformer import TransformerEncoder +from .transformer import TextEncoder, TransformerBlock, TransformerEncoder diff --git a/easy_rec/python/layers/keras/activation.py b/easy_rec/python/layers/keras/activation.py index fa6218e64..27d5ec1a8 100644 --- a/easy_rec/python/layers/keras/activation.py +++ b/easy_rec/python/layers/keras/activation.py @@ -1,8 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from tensorflow.python.keras.layers import Activation -from tensorflow.python.keras.layers import Layer +from tensorflow.python.keras.layers import Activation, Layer import easy_rec.python.utils.activation @@ -51,12 +50,14 @@ def __init__(self, axis=-1, epsilon=1e-9, **kwargs): def build(self, input_shape): self.bn = BatchNormalization( - axis=self.axis, epsilon=self.epsilon, center=False, scale=False) + axis=self.axis, epsilon=self.epsilon, center=False, scale=False + ) self.alphas = self.add_weight( - shape=(input_shape[-1],), - initializer=Zeros(), - dtype=tf.float32, - name='dice_alpha') # name='alpha_'+self.name + shape=(input_shape[-1], ), + initializer=Zeros(), + dtype=tf.float32, + name='dice_alpha' + ) # name='alpha_'+self.name super(Dice, self).build(input_shape) # Be sure to call this somewhere! self.uses_learning_phase = True @@ -74,7 +75,7 @@ def compute_output_shape(self, input_shape): def updates(self): return self.bn.updates - def get_config(self,): + def get_config(self, ): config = {'axis': self.axis, 'epsilon': self.epsilon} base_config = super(Dice, self).get_config() return dict(list(base_config.items()) + list(config.items())) @@ -109,6 +110,7 @@ def activation_layer(activation, name=None): act_layer = activation(name=name) else: raise ValueError( - 'Invalid activation,found %s.You should use a str or a Activation Layer Class.' - % (activation)) + 'Invalid activation,found %s.You should use a str or a Activation Layer Class.' + % (activation) + ) return act_layer diff --git a/easy_rec/python/layers/keras/attention.py b/easy_rec/python/layers/keras/attention.py index 4831ccae8..fb5629beb 100644 --- a/easy_rec/python/layers/keras/attention.py +++ b/easy_rec/python/layers/keras/attention.py @@ -72,15 +72,18 @@ def __init__(self, params, name='attention', reuse=None, **kwargs): self.scale_by_dim = params.get_or_default('scale_by_dim', False) self.score_mode = params.get_or_default('score_mode', 'dot') if self.score_mode not in ['dot', 'concat']: - raise ValueError('Invalid value for argument score_mode. ' - "Expected one of {'dot', 'concat'}. " - 'Received: score_mode=%s' % self.score_mode) + raise ValueError( + 'Invalid value for argument score_mode. ' + "Expected one of {'dot', 'concat'}. " + 'Received: score_mode=%s' % self.score_mode + ) self.dropout = params.get_or_default('dropout', 0.0) self.seed = params.get_or_default('seed', None) self.scale = None self.concat_score_weight = None self._return_attention_scores = params.get_or_default( - 'return_attention_scores', False) + 'return_attention_scores', False + ) self.use_causal_mask = params.get_or_default('use_causal_mask', False) @property @@ -91,21 +94,22 @@ def build(self, input_shape): self._validate_inputs(input_shape) if self.use_scale: self.scale = self.add_weight( - name='scale', - shape=(), - initializer='ones', - dtype=self.dtype, - trainable=True, + name='scale', + shape=(), + initializer='ones', + dtype=self.dtype, + trainable=True, ) if self.score_mode == 'concat': self.concat_score_weight = self.add_weight( - name='concat_score_weight', - shape=(), - initializer='ones', - dtype=self.dtype, - trainable=True, + name='concat_score_weight', + shape=(), + initializer='ones', + dtype=self.dtype, + trainable=True, ) - super(Attention, self).build(input_shape) # Be sure to call this somewhere! + super(Attention, + self).build(input_shape) # Be sure to call this somewhere! def _calculate_scores(self, query, key): """Calculates attention scores as a query-key dot product. @@ -132,10 +136,12 @@ def _calculate_scores(self, query, key): k_reshaped = tf.expand_dims(key, axis=-3) if self.scale is not None: scores = self.concat_score_weight * tf.reduce_sum( - tf.tanh(self.scale * (q_reshaped + k_reshaped)), axis=-1) + tf.tanh(self.scale * (q_reshaped + k_reshaped)), axis=-1 + ) else: scores = self.concat_score_weight * tf.reduce_sum( - tf.tanh(q_reshaped + k_reshaped), axis=-1) + tf.tanh(q_reshaped + k_reshaped), axis=-1 + ) return scores def _apply_scores(self, scores, value, scores_mask=None, training=False): @@ -211,10 +217,12 @@ def call(self, inputs, mask=None, training=False, **kwargs): q_mask = mask[0] if mask else None v_mask = mask[1] if mask else None scores = self._calculate_scores(query=q, key=k) - scores_mask = self._calculate_score_mask(scores, v_mask, - self.use_causal_mask) + scores_mask = self._calculate_score_mask( + scores, v_mask, self.use_causal_mask + ) result, attention_scores = self._apply_scores( - scores=scores, value=v, scores_mask=scores_mask, training=training) + scores=scores, value=v, scores_mask=scores_mask, training=training + ) if q_mask is not None: # Mask of shape [batch_size, Tq, 1]. q_mask = tf.expand_dims(q_mask, axis=-1) @@ -237,31 +245,40 @@ def _validate_inputs(self, inputs, mask=None): """Validates arguments of the call method.""" class_name = self.__class__.__name__ if not isinstance(inputs, list): - raise ValueError('{class_name} layer must be called on a list of inputs, ' - 'namely [query, value] or [query, value, key]. ' - 'Received: inputs={inputs}.'.format( - class_name=class_name, inputs=inputs)) + raise ValueError( + '{class_name} layer must be called on a list of inputs, ' + 'namely [query, value] or [query, value, key]. ' + 'Received: inputs={inputs}.'.format( + class_name=class_name, inputs=inputs + ) + ) if len(inputs) < 2 or len(inputs) > 3: - raise ValueError('%s layer accepts inputs list of length 2 or 3, ' - 'namely [query, value] or [query, value, key]. ' - 'Received length: %d.' % (class_name, len(inputs))) + raise ValueError( + '%s layer accepts inputs list of length 2 or 3, ' + 'namely [query, value] or [query, value, key]. ' + 'Received length: %d.' % (class_name, len(inputs)) + ) if mask is not None: if not isinstance(mask, list): raise ValueError( - '{class_name} layer mask must be a list, ' - 'namely [query_mask, value_mask]. Received: mask={mask}.'.format( - class_name=class_name, mask=mask)) + '{class_name} layer mask must be a list, ' + 'namely [query_mask, value_mask]. Received: mask={mask}.'.format( + class_name=class_name, mask=mask + ) + ) if len(mask) < 2 or len(mask) > 3: raise ValueError( - '{class_name} layer accepts mask list of length 2 or 3. ' - 'Received: inputs={inputs}, mask={mask}.'.format( - class_name=class_name, inputs=inputs, mask=mask)) + '{class_name} layer accepts mask list of length 2 or 3. ' + 'Received: inputs={inputs}, mask={mask}.'.format( + class_name=class_name, inputs=inputs, mask=mask + ) + ) def get_config(self): base_config = super(Attention, self).get_config() config = { - 'use_scale': self.use_scale, - 'score_mode': self.score_mode, - 'dropout': self.dropout, + 'use_scale': self.use_scale, + 'score_mode': self.score_mode, + 'dropout': self.dropout, } return dict(list(base_config.items()) + list(config.items())) diff --git a/easy_rec/python/layers/keras/auxiliary_loss.py b/easy_rec/python/layers/keras/auxiliary_loss.py index 6be248872..abe116c09 100644 --- a/easy_rec/python/layers/keras/auxiliary_loss.py +++ b/easy_rec/python/layers/keras/auxiliary_loss.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.loss import contrastive_loss @@ -15,8 +14,10 @@ def __init__(self, params, name='auxiliary_loss', reuse=None, **kwargs): params.check_required('loss_type') self.loss_type = params.get_or_default('loss_type', None) self.loss_weight = params.get_or_default('loss_weight', 1.0) - logging.info('init layer `%s` with loss type: %s and weight: %f' % - (self.name, self.loss_type, self.loss_weight)) + logging.info( + 'init layer `%s` with loss type: %s and weight: %f' % + (self.name, self.loss_type, self.loss_weight) + ) self.temperature = params.get_or_default('temperature', 0.1) def call(self, inputs, training=None, **kwargs): @@ -35,7 +36,8 @@ def call(self, inputs, training=None, **kwargs): elif self.loss_type == 'info_nce': query, positive = inputs loss = contrastive_loss.info_nce_loss( - query, positive, temperature=self.temperature) + query, positive, temperature=self.temperature + ) loss_value = loss if self.loss_weight == 1.0 else loss * self.loss_weight loss_dict['%s_info_nce_loss' % self.name] = loss_value elif self.loss_type == 'nce_loss': diff --git a/easy_rec/python/layers/keras/blocks.py b/easy_rec/python/layers/keras/blocks.py index c9e722a67..0f8d1fd20 100644 --- a/easy_rec/python/layers/keras/blocks.py +++ b/easy_rec/python/layers/keras/blocks.py @@ -2,13 +2,9 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Convenience blocks for building models.""" import logging - import tensorflow as tf from tensorflow.python.keras.initializers import Constant -from tensorflow.python.keras.layers import Dense -from tensorflow.python.keras.layers import Dropout -from tensorflow.python.keras.layers import Lambda -from tensorflow.python.keras.layers import Layer +from tensorflow.python.keras.layers import Dense, Dropout, Lambda, Layer from easy_rec.python.layers.keras.activation import activation_layer from easy_rec.python.layers.utils import Parameter @@ -45,10 +41,12 @@ def __init__(self, params, name='mlp', reuse=None, **kwargs): use_bn_after_act = params.get_or_default('use_bn_after_activation', False) units = list(params.hidden_units) logging.info( - 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' - ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' % - (name, units, dropout_rate, activation, use_bn, use_final_bn, - final_activation, use_bias, initializer, use_bn_after_act)) + 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' + ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' % ( + name, units, dropout_rate, activation, use_bn, use_final_bn, + final_activation, use_bias, initializer, use_bn_after_act + ) + ) assert len(units) > 0, 'MLP(%s) takes at least one hidden units' % name self.reuse = reuse self.add_to_outputs = params.get_or_default('add_to_outputs', False) @@ -58,47 +56,54 @@ def __init__(self, params, name='mlp', reuse=None, **kwargs): for i, num_units in enumerate(units[:-1]): name = 'layer_%d' % i drop_rate = dropout_rate[i] if i < num_dropout else 0.0 - self.add_rich_layer(num_units, use_bn, drop_rate, activation, initializer, - use_bias, use_bn_after_act, name, - params.l2_regularizer) + self.add_rich_layer( + num_units, use_bn, drop_rate, activation, initializer, use_bias, + use_bn_after_act, name, params.l2_regularizer + ) n = len(units) - 1 drop_rate = dropout_rate[n] if num_dropout > n else 0.0 name = 'layer_%d' % n - self.add_rich_layer(units[-1], use_final_bn, drop_rate, final_activation, - initializer, use_final_bias, use_bn_after_act, name, - params.l2_regularizer) + self.add_rich_layer( + units[-1], use_final_bn, drop_rate, final_activation, initializer, + use_final_bias, use_bn_after_act, name, params.l2_regularizer + ) - def add_rich_layer(self, - num_units, - use_bn, - dropout_rate, - activation, - initializer, - use_bias, - use_bn_after_activation, - name, - l2_reg=None): + def add_rich_layer( + self, + num_units, + use_bn, + dropout_rate, + activation, + initializer, + use_bias, + use_bn_after_activation, + name, + l2_reg=None + ): act_layer = activation_layer(activation, name='%s/act' % name) if use_bn and not use_bn_after_activation: dense = Dense( - units=num_units, - use_bias=use_bias, - kernel_initializer=initializer, - kernel_regularizer=l2_reg, - name='%s/dense' % name) + units=num_units, + use_bias=use_bias, + kernel_initializer=initializer, + kernel_regularizer=l2_reg, + name='%s/dense' % name + ) self._sub_layers.append(dense) bn = tf.keras.layers.BatchNormalization( - name='%s/bn' % name, trainable=True) + name='%s/bn' % name, trainable=True + ) self._sub_layers.append(bn) self._sub_layers.append(act_layer) else: dense = Dense( - num_units, - use_bias=use_bias, - kernel_initializer=initializer, - kernel_regularizer=l2_reg, - name='%s/dense' % name) + num_units, + use_bias=use_bias, + kernel_initializer=initializer, + kernel_regularizer=l2_reg, + name='%s/dense' % name + ) self._sub_layers.append(dense) self._sub_layers.append(act_layer) if use_bn and use_bn_after_activation: @@ -139,7 +144,8 @@ def __init__(self, params, name='highway', reuse=None, **kwargs): self.init_gate_bias = params.get_or_default('init_gate_bias', -3.0) self.act_layer = activation_layer(self.activation) self.dropout_layer = Dropout( - self.dropout_rate) if self.dropout_rate > 0.0 else None + self.dropout_rate + ) if self.dropout_rate > 0.0 else None self.project_layer = None self.gate_bias_initializer = Constant(self.init_gate_bias) self.gates = [] # T @@ -152,13 +158,14 @@ def build(self, input_shape): if self.emb_size is not None and dim != self.emb_size: self.project_layer = Dense(self.emb_size, name='input_projection') dim = self.emb_size - self.carry_gate = Lambda(lambda x: 1.0 - x, output_shape=(dim,)) + self.carry_gate = Lambda(lambda x: 1.0 - x, output_shape=(dim, )) for i in range(self.num_layers): gate = Dense( - units=dim, - bias_initializer=self.gate_bias_initializer, - activation='sigmoid', - name='gate_%d' % i) + units=dim, + bias_initializer=self.gate_bias_initializer, + activation='sigmoid', + name='gate_%d' % i + ) self.gates.append(gate) self.transforms.append(Dense(units=dim)) @@ -192,7 +199,7 @@ def __init__(self, params, name='gate', reuse=None, **kwargs): def call(self, inputs, training=None, **kwargs): assert len( - inputs + inputs ) > 1, 'input of Gate layer must be a list containing at least 2 elements' weights = inputs[self.weight_index] j = 0 @@ -222,16 +229,19 @@ def __init__(self, params, name='text_cnn', reuse=None, **kwargs): self.pad_seq_length = self.config.pad_sequence_length if self.pad_seq_length <= 0: logging.warning( - 'run text cnn with pad_sequence_length <= 0, the predict of model may be unstable' + 'run text cnn with pad_sequence_length <= 0, the predict of model may be unstable' ) self.conv_layers = [] self.pool_layer = tf.keras.layers.GlobalMaxPool1D() self.concat_layer = tf.keras.layers.Concatenate(axis=-1) - for size, filters in zip(self.config.filter_sizes, self.config.num_filters): + for size, filters in zip( + self.config.filter_sizes, self.config.num_filters + ): conv = tf.keras.layers.Conv1D( - filters=int(filters), - kernel_size=int(size), - activation=self.config.activation) + filters=int(filters), + kernel_size=int(size), + activation=self.config.activation + ) self.conv_layers.append(conv) if self.config.HasField('mlp'): p = Parameter.make_from_pb(self.config.mlp) @@ -247,8 +257,9 @@ def call(self, inputs, training=None, **kwargs): seq_emb, seq_len = inputs[:2] if self.pad_seq_length > 0: - seq_emb, seq_len = pad_or_truncate_sequence(seq_emb, seq_len, - self.pad_seq_length) + seq_emb, seq_len = pad_or_truncate_sequence( + seq_emb, seq_len, self.pad_seq_length + ) pooled_outputs = [] for layer in self.conv_layers: conv = layer(seq_emb) diff --git a/easy_rec/python/layers/keras/bst.py b/easy_rec/python/layers/keras/bst.py index dbd4882ed..4c6e92086 100644 --- a/easy_rec/python/layers/keras/bst.py +++ b/easy_rec/python/layers/keras/bst.py @@ -21,35 +21,39 @@ def __init__(self, params, name='bst', reuse=None, **kwargs): def encode(self, seq_input, max_position): seq_fea = multihead_cross_attention.embedding_postprocessor( - seq_input, - position_embedding_name=self.name, - max_position_embeddings=max_position, - reuse_position_embedding=self.reuse) + seq_input, + position_embedding_name=self.name, + max_position_embeddings=max_position, + reuse_position_embedding=self.reuse + ) n = tf.count_nonzero(seq_input, axis=-1) seq_mask = tf.cast(n > 0, tf.int32) attention_mask = multihead_cross_attention.create_attention_mask_from_input_mask( - from_tensor=seq_fea, to_mask=seq_mask) + from_tensor=seq_fea, to_mask=seq_mask + ) hidden_act = get_activation(self.config.hidden_act) attention_fea = multihead_cross_attention.transformer_encoder( - seq_fea, - hidden_size=self.config.hidden_size, - num_hidden_layers=self.config.num_hidden_layers, - num_attention_heads=self.config.num_attention_heads, - attention_mask=attention_mask, - intermediate_size=self.config.intermediate_size, - intermediate_act_fn=hidden_act, - hidden_dropout_prob=self.config.hidden_dropout_prob, - attention_probs_dropout_prob=self.config.attention_probs_dropout_prob, - initializer_range=self.config.initializer_range, - name=self.name + '/transformer', - reuse=self.reuse) + seq_fea, + hidden_size=self.config.hidden_size, + num_hidden_layers=self.config.num_hidden_layers, + num_attention_heads=self.config.num_attention_heads, + attention_mask=attention_mask, + intermediate_size=self.config.intermediate_size, + intermediate_act_fn=hidden_act, + hidden_dropout_prob=self.config.hidden_dropout_prob, + attention_probs_dropout_prob=self.config.attention_probs_dropout_prob, + initializer_range=self.config.initializer_range, + name=self.name + '/transformer', + reuse=self.reuse + ) # attention_fea shape: [batch_size, seq_length, hidden_size] if self.config.output_all_token_embeddings: - out_fea = tf.reshape(attention_fea, - [-1, max_position * self.config.hidden_size]) + out_fea = tf.reshape( + attention_fea, [-1, max_position * self.config.hidden_size] + ) else: out_fea = attention_fea[:, 0, :] # target feature print('bst output shape:', out_fea.shape) @@ -67,29 +71,34 @@ def call(self, inputs, training=None, **kwargs): max_position = self.config.max_position_embeddings # max_seq_len: the max sequence length in current mini-batch, all sequences are padded to this length batch_size, cur_batch_max_seq_len, seq_embed_size = get_shape_list( - seq_input, 3) + seq_input, 3 + ) valid_len = tf.assert_less_equal( - cur_batch_max_seq_len, - max_position, - message='sequence length is greater than `max_position_embeddings`:' + - str(max_position) + ' in feature group:' + self.name + - ', you should set `max_seq_len` in sequence feature configs') + cur_batch_max_seq_len, + max_position, + message='sequence length is greater than `max_position_embeddings`:' + + str(max_position) + ' in feature group:' + self.name + + ', you should set `max_seq_len` in sequence feature configs' + ) if self.config.output_all_token_embeddings: seq_input = tf.cond( - tf.constant(max_position) > cur_batch_max_seq_len, lambda: tf.pad( - seq_input, [[0, 0], [0, max_position - cur_batch_max_seq_len], - [0, 0]], 'CONSTANT'), - lambda: tf.slice(seq_input, [0, 0, 0], [-1, max_position, -1])) + tf.constant(max_position) > cur_batch_max_seq_len, lambda: tf.pad( + seq_input, [ + [0, 0], [0, max_position - cur_batch_max_seq_len], [0, 0] + ], 'CONSTANT' + ), lambda: tf.slice(seq_input, [0, 0, 0], [-1, max_position, -1]) + ) if seq_embed_size != self.config.hidden_size: seq_input = tf.layers.dense( - seq_input, - self.config.hidden_size, - activation=tf.nn.relu, - kernel_regularizer=self.l2_reg, - name=self.name + '/seq_project', - reuse=self.reuse) + seq_input, + self.config.hidden_size, + activation=tf.nn.relu, + kernel_regularizer=self.l2_reg, + name=self.name + '/seq_project', + reuse=self.reuse + ) keep_target = self.config.target_item_position in ('head', 'tail') if target is not None and keep_target: @@ -98,12 +107,13 @@ def call(self, inputs, training=None, **kwargs): ' in feature group:' + self.name if target_size != self.config.hidden_size: target = tf.layers.dense( - target, - self.config.hidden_size, - activation=tf.nn.relu, - kernel_regularizer=self.l2_reg, - name=self.name + '/target_project', - reuse=self.reuse) + target, + self.config.hidden_size, + activation=tf.nn.relu, + kernel_regularizer=self.l2_reg, + name=self.name + '/target_project', + reuse=self.reuse + ) # target_feature: [batch_size, 1, embed_size] target = tf.expand_dims(target, 1) # seq_input: [batch_size, seq_len+1, embed_size] diff --git a/easy_rec/python/layers/keras/custom_ops.py b/easy_rec/python/layers/keras/custom_ops.py index c215ee332..decf6deed 100644 --- a/easy_rec/python/layers/keras/custom_ops.py +++ b/easy_rec/python/layers/keras/custom_ops.py @@ -3,7 +3,6 @@ """Convenience blocks for using custom ops.""" import logging import os - import tensorflow as tf from tensorflow.python.framework import ops from tensorflow.python.keras.layers import Layer @@ -30,8 +29,9 @@ custom_ops = tf.load_op_library(custom_op_path) logging.info('load custom op from %s succeed' % custom_op_path) except Exception as ex: - logging.warning('load custom op from %s failed: %s' % - (custom_op_path, str(ex))) + logging.warning( + 'load custom op from %s failed: %s' % (custom_op_path, str(ex)) + ) custom_ops = None # if tf.__version__ >= '2.0': @@ -49,20 +49,21 @@ def __init__(self, params, name='sequence_aug', reuse=None, **kwargs): def call(self, inputs, training=None, **kwargs): assert isinstance( - inputs, - (list, tuple)), 'the inputs of SeqAugmentOps must be type of list/tuple' + inputs, (list, tuple) + ), 'the inputs of SeqAugmentOps must be type of list/tuple' assert len(inputs) >= 2, 'SeqAugmentOps must have at least 2 inputs' seq_input, seq_len = inputs[:2] embedding_dim = int(seq_input.shape[-1]) with tf.variable_scope(self.name, reuse=self.reuse): mask_emb = tf.get_variable( - 'mask', (embedding_dim,), dtype=tf.float32, trainable=True) + 'mask', (embedding_dim, ), dtype=tf.float32, trainable=True + ) seq_len = tf.to_int32(seq_len) with ops.device('/CPU:0'): - aug_seq, aug_len = self.seq_augment(seq_input, seq_len, mask_emb, - self.seq_aug_params.crop_rate, - self.seq_aug_params.reorder_rate, - self.seq_aug_params.mask_rate) + aug_seq, aug_len = self.seq_augment( + seq_input, seq_len, mask_emb, self.seq_aug_params.crop_rate, + self.seq_aug_params.reorder_rate, self.seq_aug_params.mask_rate + ) return aug_seq, aug_len @@ -78,10 +79,9 @@ def call(self, inputs, training=None, **kwargs): inputs = inputs if type(inputs) in (tuple, list) else [inputs] with ops.device('/CPU:0'): result = [ - self.txt_normalizer( - txt, - parameter=self.norm_parameter, - remove_space=self.remove_space) for txt in inputs + self.txt_normalizer( + txt, parameter=self.norm_parameter, remove_space=self.remove_space + ) for txt in inputs ] if len(result) == 1: return result[0] @@ -105,38 +105,42 @@ def __init__(self, params, name='mapped_dot_product', reuse=None, **kwargs): vocab_size = len(self.boundaries) + 1 with tf.variable_scope(self.name, reuse=reuse): self.embedding_table = tf.get_variable( - name='dot_product_emb_table', - shape=[vocab_size, self.emb_dim], - dtype=tf.float32) + name='dot_product_emb_table', + shape=[vocab_size, self.emb_dim], + dtype=tf.float32 + ) def call(self, inputs, training=None, **kwargs): query, doc = inputs[:2] with ops.device('/CPU:0'): feature = self.mapped_dot_product( - query=query, - document=doc, - feature_name=self.name, - separator=self.separator, - default_value=self.default_value) + query=query, + document=doc, + feature_name=self.name, + separator=self.separator, + default_value=self.default_value + ) tf.summary.scalar(self.name, tf.reduce_mean(feature)) if self.print_first_n: encode_q = tf.regex_replace(query, self.separator, ' ') encode_t = tf.regex_replace(query, self.separator, ' ') feature = tf.Print( - feature, [encode_q, encode_t, feature], - message=self.name, - first_n=self.print_first_n, - summarize=self.summarize) + feature, [encode_q, encode_t, feature], + message=self.name, + first_n=self.print_first_n, + summarize=self.summarize + ) if self.norm_fn is not None: fn = eval(self.norm_fn) feature = fn(feature) tf.summary.scalar('normalized_%s' % self.name, tf.reduce_mean(feature)) if self.print_first_n: feature = tf.Print( - feature, [feature], - message='normalized %s' % self.name, - first_n=self.print_first_n, - summarize=self.summarize) + feature, [feature], + message='normalized %s' % self.name, + first_n=self.print_first_n, + summarize=self.summarize + ) if self.boundaries: feature = self.bucketize(feature, boundaries=self.boundaries) tf.summary.histogram('bucketized_%s' % self.name, feature) @@ -167,22 +171,24 @@ def __init__(self, params, name='overlap_feature', reuse=None, **kwargs): vocab_size *= len(self.methods) with tf.variable_scope(self.name, reuse=reuse): self.embedding_table = tf.get_variable( - name='overlap_emb_table', - shape=[vocab_size, self.emb_dim], - dtype=tf.float32) + name='overlap_emb_table', + shape=[vocab_size, self.emb_dim], + dtype=tf.float32 + ) def call(self, inputs, training=None, **kwargs): query, title = inputs[:2] with ops.device('/CPU:0'): feature = self.overlap_feature( - query=query, - title=title, - feature_name=self.name, - separator=self.separator, - default_value=self.default_value, - boundaries=self.boundaries, - methods=self.methods, - dtype=tf.int32 if self.boundaries else tf.float32) + query=query, + title=title, + feature_name=self.name, + separator=self.separator, + default_value=self.default_value, + boundaries=self.boundaries, + methods=self.methods, + dtype=tf.int32 if self.boundaries else tf.float32 + ) for i, method in enumerate(self.methods): # warning: feature[:, i] may be not the result of method @@ -194,10 +200,11 @@ def call(self, inputs, training=None, **kwargs): encode_q = tf.regex_replace(query, self.separator, ' ') encode_t = tf.regex_replace(query, self.separator, ' ') feature = tf.Print( - feature, [encode_q, encode_t, feature], - message=self.name, - first_n=self.print_first_n, - summarize=self.summarize) + feature, [encode_q, encode_t, feature], + message=self.name, + first_n=self.print_first_n, + summarize=self.summarize + ) if self.norm_fn is not None: fn = eval(self.norm_fn) feature = fn(feature) @@ -211,14 +218,18 @@ def call(self, inputs, training=None, **kwargs): # Compute offsets, add to every column indices offsets = tf.range(num_indices) * vocab_size # Shape: [3] offsets = tf.reshape(offsets, [1, num_indices]) # Shape: [1, 3] - offsets = tf.tile(offsets, - [batch_size, 1]) # Shape: [batch_size, num_indices] + offsets = tf.tile( + offsets, [batch_size, 1] + ) # Shape: [batch_size, num_indices] shifted_indices = feature + offsets # Shape: [batch_size, num_indices] flat_feature_ids = tf.reshape(shifted_indices, [-1]) - one_hot_ids = tf.one_hot(flat_feature_ids, depth=vocab_size * num_indices) + one_hot_ids = tf.one_hot( + flat_feature_ids, depth=vocab_size * num_indices + ) feature_embeddings = tf.matmul(one_hot_ids, self.embedding_table) - feature_embeddings = tf.reshape(feature_embeddings, - [batch_size, num_indices * self.emb_dim]) + feature_embeddings = tf.reshape( + feature_embeddings, [batch_size, num_indices * self.emb_dim] + ) return feature_embeddings return feature @@ -232,19 +243,20 @@ def __init__(self, params, name='edit_distance', reuse=None, **kwargs): self.emb_size = params.get_or_default('embedding_size', 512) emb_dim = params.get_or_default('embedding_dim', 4) with tf.variable_scope(self.name, reuse=reuse): - self.embedding_table = tf.get_variable('embedding_table', - [self.emb_size, emb_dim], - tf.float32) + self.embedding_table = tf.get_variable( + 'embedding_table', [self.emb_size, emb_dim], tf.float32 + ) def call(self, inputs, training=None, **kwargs): input1, input2 = inputs[:2] with ops.device('/CPU:0'): dist = self.edit_distance( - input1, - input2, - normalize=False, - dtype=tf.int32, - encoding=self.txt_encoding) + input1, + input2, + normalize=False, + dtype=tf.int32, + encoding=self.txt_encoding + ) ids = tf.clip_by_value(dist, 0, self.emb_size - 1) embed = tf.nn.embedding_lookup(self.embedding_table, ids) return embed diff --git a/easy_rec/python/layers/keras/data_augment.py b/easy_rec/python/layers/keras/data_augment.py index a11f08120..a3e0114d4 100644 --- a/easy_rec/python/layers/keras/data_augment.py +++ b/easy_rec/python/layers/keras/data_augment.py @@ -30,27 +30,26 @@ def item_crop(aug_data, length, crop_rate): max_length = tf.cast(max_len, dtype=tf.int32) num_left = tf.cast(tf.math.floor(length1 * crop_rate), dtype=tf.int32) - crop_begin = tf.random.uniform([], - minval=0, - maxval=length - num_left, - dtype=tf.int32) + crop_begin = tf.random.uniform( + [], minval=0, maxval=length - num_left, dtype=tf.int32 + ) zeros = tf.zeros_like(aug_data) x = aug_data[crop_begin:crop_begin + num_left] y = zeros[:max_length - num_left] cropped = tf.concat([x, y], axis=0) cropped_item_seq = tf.where( - crop_begin + num_left < max_length, cropped, - tf.concat([aug_data[crop_begin:], zeros[:crop_begin]], axis=0)) + crop_begin + num_left < max_length, cropped, + tf.concat([aug_data[crop_begin:], zeros[:crop_begin]], axis=0) + ) return cropped_item_seq, num_left def item_reorder(aug_data, length, reorder_rate): length1 = tf.cast(length, dtype=tf.float32) num_reorder = tf.cast(tf.math.floor(length1 * reorder_rate), dtype=tf.int32) - reorder_begin = tf.random.uniform([], - minval=0, - maxval=length - num_reorder, - dtype=tf.int32) + reorder_begin = tf.random.uniform( + [], minval=0, maxval=length - num_reorder, dtype=tf.int32 + ) shuffle_index = tf.range(reorder_begin, reorder_begin + num_reorder) shuffle_index = tf.random.shuffle(shuffle_index) x = tf.range(get_shape_list(aug_data)[0]) @@ -58,8 +57,8 @@ def item_reorder(aug_data, length, reorder_rate): right = tf.slice(x, [reorder_begin + num_reorder], [-1]) reordered_item_index = tf.concat([left, shuffle_index, right], axis=0) reordered_item_seq = tf.scatter_nd( - tf.expand_dims(reordered_item_index, axis=1), aug_data, - tf.shape(aug_data)) + tf.expand_dims(reordered_item_index, axis=1), aug_data, tf.shape(aug_data) + ) return reordered_item_seq, length @@ -95,17 +94,19 @@ def reorder_fn(): return tf.cond(tf.equal(method, 0), trans_fn[0], trans_fn[1]) aug_seq, aug_len = tf.cond( - tf.equal(method, 0), crop_fn, - lambda: tf.cond(tf.equal(method, 1), mask_fn, reorder_fn)) + tf.equal(method, 0), crop_fn, + lambda: tf.cond(tf.equal(method, 1), mask_fn, reorder_fn) + ) return aug_seq, aug_len def sequence_augment(seq_input, seq_len, mask, aug_param): lengths = tf.cast(seq_len, dtype=tf.int32) aug_seq, aug_len = tf.map_fn( - lambda elems: augment_fn(elems, aug_param, mask), - elems=(seq_input, lengths), - dtype=(tf.float32, tf.int32)) + lambda elems: augment_fn(elems, aug_param, mask), + elems=(seq_input, lengths), + dtype=(tf.float32, tf.int32) + ) aug_seq = tf.reshape(aug_seq, tf.shape(seq_input)) return aug_seq, aug_len @@ -126,8 +127,10 @@ def call(self, inputs, training=None, **kwargs): embedding_size = int(seq_input.shape[-1]) with tf.variable_scope(self.name, reuse=self.reuse): mask_emb = tf.get_variable( - 'mask', [1, embedding_size], dtype=tf.float32, trainable=True) + 'mask', [1, embedding_size], dtype=tf.float32, trainable=True + ) - aug_seq, aug_len = sequence_augment(seq_input, seq_len, mask_emb, - self.seq_aug_params) + aug_seq, aug_len = sequence_augment( + seq_input, seq_len, mask_emb, self.seq_aug_params + ) return aug_seq, aug_len diff --git a/easy_rec/python/layers/keras/din.py b/easy_rec/python/layers/keras/din.py index 082677e0b..f45be564a 100644 --- a/easy_rec/python/layers/keras/din.py +++ b/easy_rec/python/layers/keras/din.py @@ -1,7 +1,6 @@ # -*- encoding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from tensorflow.python.keras.layers import Layer @@ -31,8 +30,9 @@ def call(self, inputs, training=None, **kwargs): seq_emb_size = keys.shape.as_list()[-1] if query_emb_size != seq_emb_size: logging.info( - ' the embedding size of sequence [%d] and target item [%d] is not equal' - ' in feature group: %s', seq_emb_size, query_emb_size, self.name) + ' the embedding size of sequence [%d] and target item [%d] is not equal' + ' in feature group: %s', seq_emb_size, query_emb_size, self.name + ) if query_emb_size < seq_emb_size: query = tf.pad(query, [[0, 0], [0, seq_emb_size - query_emb_size]]) else: @@ -40,8 +40,9 @@ def call(self, inputs, training=None, **kwargs): batch_size, max_seq_len, _ = get_shape_list(keys, 3) queries = tf.tile(tf.expand_dims(query, 1), [1, max_seq_len, 1]) - din_all = tf.concat([queries, keys, queries - keys, queries * keys], - axis=-1) + din_all = tf.concat( + [queries, keys, queries - keys, queries * keys], axis=-1 + ) output = self.din_layer(din_all, training) # [B, L, 1] scores = tf.transpose(output, [0, 2, 1]) # [B, 1, L] @@ -55,8 +56,9 @@ def call(self, inputs, training=None, **kwargs): scores = scores / (seq_emb_size**0.5) scores = tf.nn.sigmoid(scores) else: - raise ValueError('unsupported attention normalizer: ' + - self.config.attention_normalizer) + raise ValueError( + 'unsupported attention normalizer: ' + self.config.attention_normalizer + ) if query_emb_size < seq_emb_size: keys = keys[:, :, :query_emb_size] # [B, L, E] diff --git a/easy_rec/python/layers/keras/einsum_dense.py b/easy_rec/python/layers/keras/einsum_dense.py index 7531644dc..86fe185c6 100644 --- a/easy_rec/python/layers/keras/einsum_dense.py +++ b/easy_rec/python/layers/keras/einsum_dense.py @@ -2,12 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import re import string - import tensorflow as tf -from tensorflow.python.keras import activations -from tensorflow.python.keras import constraints -from tensorflow.python.keras import initializers -from tensorflow.python.keras import regularizers +from tensorflow.python.keras import activations, constraints, initializers, regularizers # NOQA from tensorflow.python.keras.layers import Layer @@ -105,23 +101,25 @@ class EinsumDense(Layer): (None, 32, 64) """ - def __init__(self, - equation, - output_shape, - activation=None, - bias_axes=None, - kernel_initializer='glorot_uniform', - bias_initializer='zeros', - kernel_regularizer=None, - bias_regularizer=None, - kernel_constraint=None, - bias_constraint=None, - lora_rank=None, - **kwargs): + def __init__( + self, + equation, + output_shape, + activation=None, + bias_axes=None, + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + lora_rank=None, + **kwargs + ): super(EinsumDense, self).__init__(**kwargs) self.equation = equation if isinstance(output_shape, int): - self.partial_output_shape = (output_shape,) + self.partial_output_shape = (output_shape, ) else: self.partial_output_shape = tuple(output_shape) self.bias_axes = bias_axes @@ -137,10 +135,10 @@ def __init__(self, def build(self, input_shape): shape_data = _analyze_einsum_string( - self.equation, - self.bias_axes, - input_shape, - self.partial_output_shape, + self.equation, + self.bias_axes, + input_shape, + self.partial_output_shape, ) kernel_shape, bias_shape, full_output_shape = shape_data for i in range(len(kernel_shape)): @@ -157,23 +155,23 @@ def build(self, input_shape): full_output_shape[i] = dim.value self.full_output_shape = tuple(full_output_shape) self._kernel = self.add_weight( - name='kernel', - shape=tuple(kernel_shape), - initializer=self.kernel_initializer, - regularizer=self.kernel_regularizer, - constraint=self.kernel_constraint, - dtype=self.dtype, - trainable=True, + name='kernel', + shape=tuple(kernel_shape), + initializer=self.kernel_initializer, + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + dtype=self.dtype, + trainable=True, ) if bias_shape is not None: self.bias = self.add_weight( - name='bias', - shape=tuple(bias_shape), - initializer=self.bias_initializer, - regularizer=self.bias_regularizer, - constraint=self.bias_constraint, - dtype=self.dtype, - trainable=True, + name='bias', + shape=tuple(bias_shape), + initializer=self.bias_initializer, + regularizer=self.bias_regularizer, + constraint=self.bias_constraint, + dtype=self.dtype, + trainable=True, ) else: self.bias = None @@ -185,7 +183,8 @@ def build(self, input_shape): def kernel(self): if not self.built: raise AttributeError( - 'You must build the layer before accessing `kernel`.') + 'You must build the layer before accessing `kernel`.' + ) if self.lora_enabled: return self._kernel + tf.matmul(self.lora_kernel_a, self.lora_kernel_b) return self._kernel @@ -201,31 +200,34 @@ def call(self, inputs, training=None): x = self.activation(x) return x - def enable_lora(self, - rank, - a_initializer='he_uniform', - b_initializer='zeros'): + def enable_lora( + self, rank, a_initializer='he_uniform', b_initializer='zeros' + ): if self.kernel_constraint: - raise ValueError('Lora is incompatible with kernel constraints. ' - 'In order to enable lora on this layer, remove the ' - '`kernel_constraint` argument.') + raise ValueError( + 'Lora is incompatible with kernel constraints. ' + 'In order to enable lora on this layer, remove the ' + '`kernel_constraint` argument.' + ) if not self.built: raise ValueError("Cannot enable lora on a layer that isn't yet built.") if self.lora_enabled: - raise ValueError('lora is already enabled. ' - 'This can only be done once per layer.') + raise ValueError( + 'lora is already enabled. ' + 'This can only be done once per layer.' + ) self._tracker.unlock() self.lora_kernel_a = self.add_weight( - name='lora_kernel_a', - shape=(self.kernel.shape[:-1] + (rank,)), - initializer=initializers.get(a_initializer), - regularizer=self.kernel_regularizer, + name='lora_kernel_a', + shape=(self.kernel.shape[:-1] + (rank, )), + initializer=initializers.get(a_initializer), + regularizer=self.kernel_regularizer, ) self.lora_kernel_b = self.add_weight( - name='lora_kernel_b', - shape=(rank, self.kernel.shape[-1]), - initializer=initializers.get(b_initializer), - regularizer=self.kernel_regularizer, + name='lora_kernel_b', + shape=(rank, self.kernel.shape[-1]), + initializer=initializers.get(b_initializer), + regularizer=self.kernel_regularizer, ) self._kernel.trainable = False self._tracker.lock() @@ -265,28 +267,18 @@ def load_own_variables(self, store): def get_config(self): base_config = super(EinsumDense, self).get_config() config = { - 'output_shape': - self.partial_output_shape, - 'equation': - self.equation, - 'activation': - activations.serialize(self.activation), - 'bias_axes': - self.bias_axes, - 'kernel_initializer': - initializers.serialize(self.kernel_initializer), - 'bias_initializer': - initializers.serialize(self.bias_initializer), - 'kernel_regularizer': - regularizers.serialize(self.kernel_regularizer), - 'bias_regularizer': - regularizers.serialize(self.bias_regularizer), - 'activity_regularizer': - regularizers.serialize(self.activity_regularizer), - 'kernel_constraint': - constraints.serialize(self.kernel_constraint), - 'bias_constraint': - constraints.serialize(self.bias_constraint), + 'output_shape': self.partial_output_shape, + 'equation': self.equation, + 'activation': activations.serialize(self.activation), + 'bias_axes': self.bias_axes, + 'kernel_initializer': initializers.serialize(self.kernel_initializer), + 'bias_initializer': initializers.serialize(self.bias_initializer), + 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), + 'bias_regularizer': regularizers.serialize(self.bias_regularizer), + 'activity_regularizer': + regularizers.serialize(self.activity_regularizer), + 'kernel_constraint': constraints.serialize(self.kernel_constraint), + 'bias_constraint': constraints.serialize(self.bias_constraint), } if self.lora_rank: config['lora_rank'] = self.lora_rank @@ -298,36 +290,40 @@ def _check_load_own_variables(self, store): if len(store.keys()) != len(all_vars): if len(all_vars) == 0 and not self.built: raise ValueError( - "Layer '{name}' was never built " - "and thus it doesn't have any variables. " - 'However the weights file lists {num_keys} ' - 'variables for this layer.\n' - 'In most cases, this error indicates that either:\n\n' - '1. The layer is owned by a parent layer that ' - 'implements a `build()` method, but calling the ' - "parent's `build()` method did NOT create the state of " - "the child layer '{name}'. A `build()` method " - 'must create ALL state for the layer, including ' - 'the state of any children layers.\n\n' - '2. You need to implement ' - 'the `def build_from_config(self, config)` method ' - "on layer '{name}', to specify how to rebuild " - 'it during loading. ' - 'In this case, you might also want to implement the ' - 'method that generates the build config at saving time, ' - '`def get_build_config(self)`. ' - 'The method `build_from_config()` is meant ' - 'to create the state ' - 'of the layer (i.e. its variables) upon deserialization.'.format( - name=self.name, num_keys=len(store.keys()))) + "Layer '{name}' was never built " + "and thus it doesn't have any variables. " + 'However the weights file lists {num_keys} ' + 'variables for this layer.\n' + 'In most cases, this error indicates that either:\n\n' + '1. The layer is owned by a parent layer that ' + 'implements a `build()` method, but calling the ' + "parent's `build()` method did NOT create the state of " + "the child layer '{name}'. A `build()` method " + 'must create ALL state for the layer, including ' + 'the state of any children layers.\n\n' + '2. You need to implement ' + 'the `def build_from_config(self, config)` method ' + "on layer '{name}', to specify how to rebuild " + 'it during loading. ' + 'In this case, you might also want to implement the ' + 'method that generates the build config at saving time, ' + '`def get_build_config(self)`. ' + 'The method `build_from_config()` is meant ' + 'to create the state ' + 'of the layer (i.e. its variables) upon deserialization.'.format( + name=self.name, num_keys=len(store.keys()) + ) + ) raise ValueError( - "Layer '{name}' expected {num_var} variables, but received " - '{num_key} variables during loading. ' - 'Expected: {names}'.format( - name=self.name, - num_var=len(store.keys()), - num_key=len(store.keys()), - names=[v.name for v in all_vars])) + "Layer '{name}' expected {num_var} variables, but received " + '{num_key} variables during loading. ' + 'Expected: {names}'.format( + name=self.name, + num_var=len(store.keys()), + num_key=len(store.keys()), + names=[v.name for v in all_vars] + ) + ) def _get_kernel_with_merged_lora(self): kernel_value = self.kernel @@ -340,37 +336,43 @@ def _analyze_einsum_string(equation, bias_axes, input_shape, output_shape): dot_replaced_string = re.sub(r'\.\.\.', '0', equation) # This is the case where no ellipses are present in the string. - split_string = re.match('([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', - dot_replaced_string) + split_string = re.match( + '([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', dot_replaced_string + ) if split_string: - return _analyze_split_string(split_string, bias_axes, input_shape, - output_shape) + return _analyze_split_string( + split_string, bias_axes, input_shape, output_shape + ) # This is the case where ellipses are present on the left. - split_string = re.match('0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', - dot_replaced_string) + split_string = re.match( + '0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', dot_replaced_string + ) if split_string: return _analyze_split_string( - split_string, bias_axes, input_shape, output_shape, left_elided=True) + split_string, bias_axes, input_shape, output_shape, left_elided=True + ) # This is the case where ellipses are present on the right. - split_string = re.match('([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', - dot_replaced_string) + split_string = re.match( + '([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', dot_replaced_string + ) if split_string: - return _analyze_split_string(split_string, bias_axes, input_shape, - output_shape) + return _analyze_split_string( + split_string, bias_axes, input_shape, output_shape + ) raise ValueError( - "Invalid einsum equation '{equation}'. Equations must be in the form " - '[X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format( - equation=equation)) + "Invalid einsum equation '{equation}'. Equations must be in the form " + '[X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format( + equation=equation + ) + ) -def _analyze_split_string(split_string, - bias_axes, - input_shape, - output_shape, - left_elided=False): +def _analyze_split_string( + split_string, bias_axes, input_shape, output_shape, left_elided=False +): """Analyze an pre-split einsum string to find the weight shape.""" input_spec = split_string.group(1) weight_spec = split_string.group(2) @@ -396,7 +398,8 @@ def _analyze_split_string(split_string, # If we have beginning dimensions elided, we need to use negative # indexing to determine where in the input dimension our values are. input_dim_map = { - dim: (i + elided) - len(input_shape) for i, dim in enumerate(input_spec) + dim: (i + elided) - len(input_shape) + for i, dim in enumerate(input_spec) } # Because we've constructed the full output shape already, we don't need # to do negative indexing. @@ -409,23 +412,29 @@ def _analyze_split_string(split_string, input_shape_at_dim = input_shape[input_dim_map[dim]] if dim in output_dim_map: output_shape_at_dim = output_shape[output_dim_map[dim]] - if (output_shape_at_dim is not None and - output_shape_at_dim != input_shape_at_dim): + if ( + output_shape_at_dim is not None + and output_shape_at_dim != input_shape_at_dim + ): raise ValueError( - 'Input shape and output shape do not match at shared ' - "dimension '{dim}'. Input shape is {input_shape_at_dim}, " - 'and output shape is {output_shape}.'.format( - dim=dim, - input_shape_at_dim=input_shape_at_dim, - output_shape=output_shape[output_dim_map[dim]])) + 'Input shape and output shape do not match at shared ' + "dimension '{dim}'. Input shape is {input_shape_at_dim}, " + 'and output shape is {output_shape}.'.format( + dim=dim, + input_shape_at_dim=input_shape_at_dim, + output_shape=output_shape[output_dim_map[dim]] + ) + ) for dim in output_spec: if dim not in input_spec and dim not in weight_spec: raise ValueError( - "Dimension '{dim}' was specified in the output " - "'{output_spec}' but has no corresponding dim in the input " - "spec '{input_spec}' or weight spec '{output_spec}'".format( - dim=dim, output_spec=output_spec, input_spec=input_spec)) + "Dimension '{dim}' was specified in the output " + "'{output_spec}' but has no corresponding dim in the input " + "spec '{input_spec}' or weight spec '{output_spec}'".format( + dim=dim, output_spec=output_spec, input_spec=input_spec + ) + ) weight_shape = [] for dim in weight_spec: @@ -435,31 +444,35 @@ def _analyze_split_string(split_string, weight_shape.append(output_shape[output_dim_map[dim]]) else: raise ValueError( - "Weight dimension '{dim}' did not have a match in either " - "the input spec '{input_spec}' or the output " - "spec '{output_spec}'. For this layer, the weight must " - 'be fully specified.'.format( - dim=dim, input_spec=input_spec, output_spec=output_spec)) + "Weight dimension '{dim}' did not have a match in either " + "the input spec '{input_spec}' or the output " + "spec '{output_spec}'. For this layer, the weight must " + 'be fully specified.'.format( + dim=dim, input_spec=input_spec, output_spec=output_spec + ) + ) if bias_axes is not None: num_left_elided = elided if left_elided else 0 idx_map = { - char: output_shape[i + num_left_elided] - for i, char in enumerate(output_spec) + char: output_shape[i + num_left_elided] + for i, char in enumerate(output_spec) } for char in bias_axes: if char not in output_spec: raise ValueError( - "Bias dimension '{char}' was requested, but is not part " - "of the output spec '{output_spec}'".format( - char=char, output_spec=output_spec)) + "Bias dimension '{char}' was requested, but is not part " + "of the output spec '{output_spec}'".format( + char=char, output_spec=output_spec + ) + ) first_bias_location = min([output_spec.find(char) for char in bias_axes]) bias_output_spec = output_spec[first_bias_location:] bias_shape = [ - idx_map[char] if char in bias_axes else 1 for char in bias_output_spec + idx_map[char] if char in bias_axes else 1 for char in bias_output_spec ] if not left_elided: @@ -478,8 +491,9 @@ def get_specs(equation, input_shape): dot_replaced_string = re.sub(r'\.\.\.', '0', equation) # This is the case where no ellipses are present in the string. - split_string = re.match('([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', - dot_replaced_string) + split_string = re.match( + '([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', dot_replaced_string + ) if split_string is not None: input_spec = split_string.group(1) weight_spec = split_string.group(2) @@ -487,16 +501,18 @@ def get_specs(equation, input_shape): return input_spec, weight_spec, output_spec # This is the case where ellipses are present on the left. - split_string = re.match('0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', - dot_replaced_string) + split_string = re.match( + '0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', dot_replaced_string + ) if split_string is not None: input_spec = split_string.group(1) weight_spec = split_string.group(2) output_spec = split_string.group(3) elided = len(input_shape) - len(input_spec) possible_labels = sorted( - set(possible_labels) - set(input_spec) - set(weight_spec) - - set(output_spec)) + set(possible_labels) - set(input_spec) - set(weight_spec) - + set(output_spec) + ) # Pad labels on the left to `input_spec` and `output_spec` for i in range(elided): input_spec = possible_labels[i] + input_spec @@ -504,16 +520,18 @@ def get_specs(equation, input_shape): return input_spec, weight_spec, output_spec # This is the case where ellipses are present on the right. - split_string = re.match('([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', - dot_replaced_string) + split_string = re.match( + '([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', dot_replaced_string + ) if split_string is not None: input_spec = split_string.group(1) weight_spec = split_string.group(2) output_spec = split_string.group(3) elided = len(input_shape) - len(input_spec) possible_labels = sorted( - set(possible_labels) - set(input_spec) - set(weight_spec) - - set(output_spec)) + set(possible_labels) - set(input_spec) - set(weight_spec) - + set(output_spec) + ) # Pad labels on the right to `input_spec` and `output_spec` for i in range(elided): input_spec = input_spec + possible_labels[i] @@ -521,9 +539,11 @@ def get_specs(equation, input_shape): return input_spec, weight_spec, output_spec raise ValueError( - "Invalid einsum equation '{equation}'. Equations must be in the " - 'form [X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format( - equation=equation)) + "Invalid einsum equation '{equation}'. Equations must be in the " + 'form [X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format( + equation=equation + ) + ) input_spec, weight_spec, output_spec = get_specs(equation, input_shape) @@ -579,20 +599,21 @@ def get_specs(equation, input_shape): weight_transpose_axes.insert(index, ori_index) # Prepare equation for `einsum_with_inputs_gradient` custom_gradient_equation = '{output_spec},{weight_spec}->{input_spec}'.format( - output_spec=output_spec, input_spec=input_spec, weight_spec=weight_spec) + output_spec=output_spec, input_spec=input_spec, weight_spec=weight_spec + ) weight_reverse_transpose_axes = [ - i for (_, i) in sorted((v, i) - for (i, v) in enumerate(weight_transpose_axes)) + i for (_, + i) in sorted((v, i) for (i, v) in enumerate(weight_transpose_axes)) ] return ( - input_reduced_axes, - weight_reduced_axes, - input_transpose_axes, - weight_transpose_axes, - input_expand_axes, - weight_expand_axes, - input_squeeze_axes, - weight_squeeze_axes, - custom_gradient_equation, - weight_reverse_transpose_axes, + input_reduced_axes, + weight_reduced_axes, + input_transpose_axes, + weight_transpose_axes, + input_expand_axes, + weight_expand_axes, + input_squeeze_axes, + weight_squeeze_axes, + custom_gradient_equation, + weight_reverse_transpose_axes, ) diff --git a/easy_rec/python/layers/keras/embedding.py b/easy_rec/python/layers/keras/embedding.py index 77b513951..578e7fcf5 100644 --- a/easy_rec/python/layers/keras/embedding.py +++ b/easy_rec/python/layers/keras/embedding.py @@ -2,8 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Fused embedding layer.""" import tensorflow as tf -from tensorflow.python.keras.layers import Embedding -from tensorflow.python.keras.layers import Layer +from tensorflow.python.keras.layers import Embedding, Layer def _combine(embeddings, weights, comb_fn): @@ -71,10 +70,12 @@ def call(self, inputs, training=None, **kwargs): if is_multi[i]: batch_size = tf.shape(inputs[i])[0] embeddings[i] = tf.cond( - tf.equal(tf.size(embeddings[i]), 0), - lambda: tf.zeros([batch_size, self.embed_dim]), lambda: _combine( - tf.reshape(embeddings[i], [batch_size, -1, self.embed_dim]), - weights[i], self.combine_fn)) + tf.equal(tf.size(embeddings[i]), 0), + lambda: tf.zeros([batch_size, self.embed_dim]), lambda: _combine( + tf.reshape(embeddings[i], [batch_size, -1, self.embed_dim]), + weights[i], self.combine_fn + ) + ) if self.do_concat: embeddings = tf.concat(embeddings, axis=-1) print('Embedding layer:', self.name, embeddings) diff --git a/easy_rec/python/layers/keras/fibinet.py b/easy_rec/python/layers/keras/fibinet.py index 220c57cb5..2b667c790 100644 --- a/easy_rec/python/layers/keras/fibinet.py +++ b/easy_rec/python/layers/keras/fibinet.py @@ -2,10 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import itertools import logging - import tensorflow as tf -from tensorflow.python.keras.layers import Dense -from tensorflow.python.keras.layers import Layer +from tensorflow.python.keras.layers import Dense, Layer from easy_rec.python.layers.keras.blocks import MLP from easy_rec.python.layers.keras.layer_norm import LayerNormalization @@ -45,19 +43,22 @@ def build(self, input_shape): assert shape.ndims == 2, 'field embeddings must be rank 2 tensors' dim = int(shape[-1]) assert dim >= g and dim % g == 0, 'field embedding dimension %d must be divisible by %d' % ( - dim, g) + dim, g + ) emb_size += dim r = self.config.reduction_ratio field_size = len(input_shape) reduction_size = max(1, field_size * g * 2 // r) self.reduce_layer = Dense( - units=reduction_size, - activation='relu', - kernel_initializer='he_normal', - name='W1') + units=reduction_size, + activation='relu', + kernel_initializer='he_normal', + name='W1' + ) self.excite_layer = Dense( - units=emb_size, kernel_initializer='glorot_normal', name='W2') + units=emb_size, kernel_initializer='glorot_normal', name='W2' + ) super(SENet, self).build(input_shape) # Be sure to call this somewhere! def call(self, inputs, **kwargs): @@ -66,7 +67,7 @@ def call(self, inputs, **kwargs): # Squeeze # embedding dimension 必须能被 g 整除 group_embs = [ - tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs + tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs ] squeezed = [] @@ -96,7 +97,8 @@ def call(self, inputs, **kwargs): def _full_interaction(v_i, v_j): # [bs, 1, dim] x [bs, dim, 1] = [bs, 1] interaction = tf.matmul( - tf.expand_dims(v_i, axis=1), tf.expand_dims(v_j, axis=-1)) + tf.expand_dims(v_i, axis=1), tf.expand_dims(v_j, axis=-1) + ) return tf.squeeze(interaction, axis=1) @@ -132,7 +134,8 @@ def __init__(self, params, name='bilinear', reuse=None, **kwargs): self.bilinear_type = params.get_or_default('type', 'interaction').lower() if self.bilinear_type not in ['all', 'each', 'interaction']: raise NotImplementedError( - "bilinear_type only support: ['all', 'each', 'interaction']") + "bilinear_type only support: ['all', 'each', 'interaction']" + ) if bilinear_plus: self.func = _full_interaction else: @@ -154,7 +157,7 @@ def build(self, input_shape): equal_dim = False if not equal_dim and self.bilinear_type != 'interaction': raise ValueError( - 'all embedding dimensions must be same when not use bilinear type: interaction' + 'all embedding dimensions must be same when not use bilinear type: interaction' ) dim = int(_dim) @@ -162,13 +165,13 @@ def build(self, input_shape): self.dot_layer = Dense(dim, name='all') elif self.bilinear_type == 'each': self.dot_layers = [ - Dense(dim, name='each_%d' % i) for i in range(field_num - 1) + Dense(dim, name='each_%d' % i) for i in range(field_num - 1) ] else: # interaction self.dot_layers = [ - Dense( - units=int(input_shape[j][-1]), name='interaction_%d_%d' % (i, j)) - for i, j in itertools.combinations(range(field_num), 2) + Dense( + units=int(input_shape[j][-1]), name='interaction_%d_%d' % (i, j) + ) for i, j in itertools.combinations(range(field_num), 2) ] super(BiLinear, self).build(input_shape) # Be sure to call this somewhere! @@ -184,20 +187,22 @@ def call(self, inputs, **kwargs): if self.bilinear_type == 'all': v_dot = [self.dot_layer(v_i) for v_i in embeddings[:-1]] p = [ - self.func(v_dot[i], embeddings[j]) - for i, j in itertools.combinations(range(field_num), 2) + self.func(v_dot[i], embeddings[j]) + for i, j in itertools.combinations(range(field_num), 2) ] elif self.bilinear_type == 'each': - v_dot = [self.dot_layers[i](v_i) for i, v_i in enumerate(embeddings[:-1])] + v_dot = [ + self.dot_layers[i](v_i) for i, v_i in enumerate(embeddings[:-1]) + ] p = [ - self.func(v_dot[i], embeddings[j]) - for i, j in itertools.combinations(range(field_num), 2) + self.func(v_dot[i], embeddings[j]) + for i, j in itertools.combinations(range(field_num), 2) ] else: # interaction p = [ - self.func(self.dot_layers[i * field_num + j](embeddings[i]), - embeddings[j]) - for i, j in itertools.combinations(range(field_num), 2) + self.func( + self.dot_layers[i * field_num + j](embeddings[i]), embeddings[j] + ) for i, j in itertools.combinations(range(field_num), 2) ] return self.output_layer(tf.concat(p, axis=-1)) @@ -218,12 +223,14 @@ def __init__(self, params, name='fibinet', reuse=None, **kwargs): se_params = Parameter.make_from_pb(self._config.senet) self.senet_layer = SENet( - se_params, name=self.name + '/senet', reuse=self.reuse) + se_params, name=self.name + '/senet', reuse=self.reuse + ) if self._config.HasField('bilinear'): bi_params = Parameter.make_from_pb(self._config.bilinear) self.bilinear_layer = BiLinear( - bi_params, name=self.name + '/bilinear', reuse=self.reuse) + bi_params, name=self.name + '/bilinear', reuse=self.reuse + ) if self._config.HasField('mlp'): p = Parameter.make_from_pb(self._config.mlp) diff --git a/easy_rec/python/layers/keras/interaction.py b/easy_rec/python/layers/keras/interaction.py index 9b14f254a..d8098f6c0 100644 --- a/easy_rec/python/layers/keras/interaction.py +++ b/easy_rec/python/layers/keras/interaction.py @@ -92,8 +92,10 @@ def call(self, inputs, **kwargs): try: concat_features = tf.stack(inputs, axis=1) except (ValueError, tf.errors.InvalidArgumentError) as e: - raise ValueError('Input tensors` dimensions must be equal, original' - 'error message: {}'.format(e)) + raise ValueError( + 'Input tensors` dimensions must be equal, original' + 'error message: {}'.format(e) + ) else: assert inputs.shape.ndims == 3, 'input of dot func must be a 3D tensor or a list of 2D tensors' concat_features = inputs @@ -118,9 +120,10 @@ def call(self, inputs, **kwargs): if self._skip_gather: # Setting upper triangle part of the interaction matrix to zeros. activations = tf.where( - condition=tf.cast(upper_tri_mask, tf.bool), - x=tf.zeros_like(xactions), - y=xactions) + condition=tf.cast(upper_tri_mask, tf.bool), + x=tf.zeros_like(xactions), + y=xactions + ) out_dim = num_features * num_features else: activations = tf.boolean_mask(xactions, lower_tri_mask) @@ -195,8 +198,9 @@ def __init__(self, params, name='cross', reuse=None, **kwargs): preactivation = params.get_or_default('preactivation', None) preact = get_activation(preactivation) self._preactivation = tf.keras.activations.get(preact) - kernel_initializer = params.get_or_default('kernel_initializer', - 'truncated_normal') + kernel_initializer = params.get_or_default( + 'kernel_initializer', 'truncated_normal' + ) self._kernel_initializer = tf.keras.initializers.get(kernel_initializer) bias_initializer = params.get_or_default('bias_initializer', 'zeros') self._bias_initializer = tf.keras.initializers.get(bias_initializer) @@ -209,40 +213,42 @@ def __init__(self, params, name='cross', reuse=None, **kwargs): if self._diag_scale < 0: # pytype: disable=unsupported-operands raise ValueError( - '`diag_scale` should be non-negative. Got `diag_scale` = {}'.format( - self._diag_scale)) + '`diag_scale` should be non-negative. Got `diag_scale` = {}'.format( + self._diag_scale + ) + ) def build(self, input_shape): last_dim = input_shape[0][-1] if self._projection_dim is None: self._dense = tf.keras.layers.Dense( - last_dim, - kernel_initializer=_clone_initializer(self._kernel_initializer), - bias_initializer=self._bias_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer, - use_bias=self._use_bias, - dtype=self.dtype, - activation=self._preactivation, + last_dim, + kernel_initializer=_clone_initializer(self._kernel_initializer), + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + use_bias=self._use_bias, + dtype=self.dtype, + activation=self._preactivation, ) else: self._dense_u = tf.keras.layers.Dense( - self._projection_dim, - kernel_initializer=_clone_initializer(self._kernel_initializer), - kernel_regularizer=self._kernel_regularizer, - use_bias=False, - dtype=self.dtype, + self._projection_dim, + kernel_initializer=_clone_initializer(self._kernel_initializer), + kernel_regularizer=self._kernel_regularizer, + use_bias=False, + dtype=self.dtype, ) self._dense_v = tf.keras.layers.Dense( - last_dim, - kernel_initializer=_clone_initializer(self._kernel_initializer), - bias_initializer=self._bias_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer, - use_bias=self._use_bias, - dtype=self.dtype, - activation=self._preactivation, + last_dim, + kernel_initializer=_clone_initializer(self._kernel_initializer), + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + use_bias=self._use_bias, + dtype=self.dtype, + activation=self._preactivation, ) super(Cross, self).build(input_shape) # Be sure to call this somewhere! @@ -269,9 +275,11 @@ def call(self, inputs, **kwargs): if x0.shape[-1] != x.shape[-1]: raise ValueError( - '`x0` and `x` dimension mismatch! Got `x0` dimension {}, and x ' - 'dimension {}. This case is not supported yet.'.format( - x0.shape[-1], x.shape[-1])) + '`x0` and `x` dimension mismatch! Got `x0` dimension {}, and x ' + 'dimension {}. This case is not supported yet.'.format( + x0.shape[-1], x.shape[-1] + ) + ) if self._projection_dim is None: prod_output = self._dense(x) @@ -287,22 +295,22 @@ def call(self, inputs, **kwargs): def get_config(self): config = { - 'projection_dim': - self._projection_dim, - 'diag_scale': - self._diag_scale, - 'use_bias': - self._use_bias, - 'preactivation': - tf.keras.activations.serialize(self._preactivation), - 'kernel_initializer': - tf.keras.initializers.serialize(self._kernel_initializer), - 'bias_initializer': - tf.keras.initializers.serialize(self._bias_initializer), - 'kernel_regularizer': - tf.keras.regularizers.serialize(self._kernel_regularizer), - 'bias_regularizer': - tf.keras.regularizers.serialize(self._bias_regularizer), + 'projection_dim': + self._projection_dim, + 'diag_scale': + self._diag_scale, + 'use_bias': + self._use_bias, + 'preactivation': + tf.keras.activations.serialize(self._preactivation), + 'kernel_initializer': + tf.keras.initializers.serialize(self._kernel_initializer), + 'bias_initializer': + tf.keras.initializers.serialize(self._bias_initializer), + 'kernel_regularizer': + tf.keras.regularizers.serialize(self._kernel_regularizer), + 'bias_regularizer': + tf.keras.regularizers.serialize(self._bias_regularizer), } base_config = super(Cross, self).get_config() return dict(list(base_config.items()) + list(config.items())) @@ -324,10 +332,11 @@ def __init__(self, params, name='cin', reuse=None, **kwargs): super(CIN, self).__init__(name=name, **kwargs) self._name = name self._hidden_feature_sizes = list( - params.get_or_default('hidden_feature_sizes', [])) + params.get_or_default('hidden_feature_sizes', []) + ) assert isinstance(self._hidden_feature_sizes, list) and len( - self._hidden_feature_sizes + self._hidden_feature_sizes ) > 0, 'parameter hidden_feature_sizes must be a list of int with length greater than 0' kernel_regularizer = params.get_or_default('kernel_regularizer', None) @@ -338,31 +347,34 @@ def __init__(self, params, name='cin', reuse=None, **kwargs): def build(self, input_shape): if len(input_shape) != 3: raise ValueError( - 'Unexpected inputs dimensions %d, expect to be 3 dimensions' % - (len(input_shape))) + 'Unexpected inputs dimensions %d, expect to be 3 dimensions' % + (len(input_shape)) + ) hidden_feature_sizes = [input_shape[1] - ] + [h for h in self._hidden_feature_sizes] + ] + [h for h in self._hidden_feature_sizes] tfv1 = tf.compat.v1 if tf.__version__ >= '2.0' else tf with tfv1.variable_scope(self._name): self.kernel_list = [ - tfv1.get_variable( - name='cin_kernel_%d' % i, - shape=[ - hidden_feature_sizes[i + 1], hidden_feature_sizes[i], - hidden_feature_sizes[0] - ], - initializer=tf.initializers.he_normal(), - regularizer=self._kernel_regularizer, - trainable=True) for i in range(len(self._hidden_feature_sizes)) + tfv1.get_variable( + name='cin_kernel_%d' % i, + shape=[ + hidden_feature_sizes[i + 1], hidden_feature_sizes[i], + hidden_feature_sizes[0] + ], + initializer=tf.initializers.he_normal(), + regularizer=self._kernel_regularizer, + trainable=True + ) for i in range(len(self._hidden_feature_sizes)) ] self.bias_list = [ - tfv1.get_variable( - name='cin_bias_%d' % i, - shape=[hidden_feature_sizes[i + 1]], - initializer=tf.keras.initializers.Zeros, - regularizer=self._bias_regularizer, - trainable=True) for i in range(len(self._hidden_feature_sizes)) + tfv1.get_variable( + name='cin_bias_%d' % i, + shape=[hidden_feature_sizes[i + 1]], + initializer=tf.keras.initializers.Zeros, + regularizer=self._bias_regularizer, + trainable=True + ) for i in range(len(self._hidden_feature_sizes)) ] super(CIN, self).build(input_shape) @@ -389,24 +401,29 @@ def call(self, input, **kwargs): intermediate_tensor = tf.multiply(x_0_expanded, x_i_expanded) intermediate_tensor_expanded = tf.expand_dims(intermediate_tensor, 1) - intermediate_tensor_expanded = tf.tile(intermediate_tensor_expanded, - [1, hk, 1, 1, 1]) + intermediate_tensor_expanded = tf.tile( + intermediate_tensor_expanded, [1, hk, 1, 1, 1] + ) feature_map_elementwise = tf.multiply( - intermediate_tensor_expanded, - tf.expand_dims(tf.expand_dims(self.kernel_list[i], -1), 0)) + intermediate_tensor_expanded, + tf.expand_dims(tf.expand_dims(self.kernel_list[i], -1), 0) + ) feature_map = tf.reduce_sum( - tf.reduce_sum(feature_map_elementwise, axis=3), axis=2) + tf.reduce_sum(feature_map_elementwise, axis=3), axis=2 + ) feature_map = tf.add( - feature_map, - tf.expand_dims(tf.expand_dims(self.bias_list[i], axis=-1), axis=0)) + feature_map, + tf.expand_dims(tf.expand_dims(self.bias_list[i], axis=-1), axis=0) + ) feature_map = tf.nn.relu(feature_map) x_i = feature_map pooled_feature_map_list.append(tf.reduce_sum(feature_map, axis=-1)) return tf.concat( - pooled_feature_map_list, axis=-1) # shape = (b, h1 + ... + hk) + pooled_feature_map_list, axis=-1 + ) # shape = (b, h1 + ... + hk) def get_config(self): pass diff --git a/easy_rec/python/layers/keras/layer_norm.py b/easy_rec/python/layers/keras/layer_norm.py index 7d6c81d5f..9415ce977 100644 --- a/easy_rec/python/layers/keras/layer_norm.py +++ b/easy_rec/python/layers/keras/layer_norm.py @@ -1,8 +1,6 @@ """Layer Normalization layer.""" import tensorflow as tf -from tensorflow.python.keras import constraints -from tensorflow.python.keras import initializers -from tensorflow.python.keras import regularizers +from tensorflow.python.keras import constraints, initializers, regularizers from tensorflow.python.keras.layers import Layer @@ -21,8 +19,10 @@ def validate_axis(axis, input_shape): rank = input_shape.ndims if not rank: raise ValueError( - 'Input has undefined rank. Received: input_shape={input_shape}'.format( - input_shape=input_shape)) + 'Input has undefined rank. Received: input_shape={input_shape}'.format( + input_shape=input_shape + ) + ) # Convert axis to list and resolve negatives if isinstance(axis, int): @@ -36,10 +36,13 @@ def validate_axis(axis, input_shape): # Validate axes for x in axis: if x < 0 or x >= rank: - raise ValueError('Invalid value for `axis` argument. ' - 'Expected 0 <= axis < inputs.rank (with ' - 'inputs.rank={rank}). Received: axis={axis}'.format( - rank=rank, axis=tuple(axis))) + raise ValueError( + 'Invalid value for `axis` argument. ' + 'Expected 0 <= axis < inputs.rank (with ' + 'inputs.rank={rank}). Received: axis={axis}'.format( + rank=rank, axis=tuple(axis) + ) + ) if len(axis) != len(set(axis)): raise ValueError('Duplicate axis: {axis}'.format(axis=tuple(axis))) return axis @@ -168,26 +171,30 @@ class LayerNormalization(Layer): - [Lei Ba et al., 2016](https://arxiv.org/abs/1607.06450). """ - def __init__(self, - axis=-1, - epsilon=1e-3, - center=True, - scale=True, - beta_initializer='zeros', - gamma_initializer='ones', - beta_regularizer=None, - gamma_regularizer=None, - beta_constraint=None, - gamma_constraint=None, - **kwargs): + def __init__( + self, + axis=-1, + epsilon=1e-3, + center=True, + scale=True, + beta_initializer='zeros', + gamma_initializer='ones', + beta_regularizer=None, + gamma_regularizer=None, + beta_constraint=None, + gamma_constraint=None, + **kwargs + ): super(LayerNormalization, self).__init__(**kwargs) if isinstance(axis, (list, tuple)): self.axis = list(axis) elif isinstance(axis, int): self.axis = axis else: - raise TypeError('Expected an int or a list/tuple of ints for the ' - "argument 'axis', but received: %r" % axis) + raise TypeError( + 'Expected an int or a list/tuple of ints for the ' + "argument 'axis', but received: %r" % axis + ) self.epsilon = epsilon self.center = center @@ -236,24 +243,24 @@ def build(self, input_shape): param_shape = [input_shape[dim] for dim in self.axis] if self.scale: self.gamma = self.add_weight( - name='gamma', - shape=param_shape, - initializer=self.gamma_initializer, - regularizer=self.gamma_regularizer, - constraint=self.gamma_constraint, - trainable=True, + name='gamma', + shape=param_shape, + initializer=self.gamma_initializer, + regularizer=self.gamma_regularizer, + constraint=self.gamma_constraint, + trainable=True, ) else: self.gamma = None if self.center: self.beta = self.add_weight( - name='beta', - shape=param_shape, - initializer=self.beta_initializer, - regularizer=self.beta_regularizer, - constraint=self.beta_constraint, - trainable=True, + name='beta', + shape=param_shape, + initializer=self.beta_initializer, + regularizer=self.beta_regularizer, + constraint=self.beta_constraint, + trainable=True, ) else: self.beta = None @@ -274,7 +281,9 @@ def call(self, inputs): broadcast_shape[dim] = input_shape.dims[dim].value def _broadcast(v): - if (v is not None and len(v.shape) != ndims and self.axis != [ndims - 1]): + if ( + v is not None and len(v.shape) != ndims and self.axis != [ndims - 1] + ): return tf.reshape(v, broadcast_shape) return v @@ -293,12 +302,12 @@ def _broadcast(v): # Compute layer normalization using the batch_normalization # function. outputs = tf.nn.batch_normalization( - inputs, - mean, - variance, - offset=offset, - scale=scale, - variance_epsilon=self.epsilon, + inputs, + mean, + variance, + offset=offset, + scale=scale, + variance_epsilon=self.epsilon, ) outputs = tf.cast(outputs, input_dtype) else: @@ -324,11 +333,11 @@ def _broadcast(v): # Compute layer normalization using the fused_batch_norm function. outputs, _, _ = tf.compat.v1.nn.fused_batch_norm( - inputs, - scale=scale, - offset=offset, - epsilon=self.epsilon, - data_format=data_format, + inputs, + scale=scale, + offset=offset, + epsilon=self.epsilon, + data_format=data_format, ) outputs = tf.reshape(outputs, tensor_shape) @@ -349,16 +358,16 @@ def compute_output_shape(self, input_shape): def get_config(self): config = { - 'axis': self.axis, - 'epsilon': self.epsilon, - 'center': self.center, - 'scale': self.scale, - 'beta_initializer': initializers.serialize(self.beta_initializer), - 'gamma_initializer': initializers.serialize(self.gamma_initializer), - 'beta_regularizer': regularizers.serialize(self.beta_regularizer), - 'gamma_regularizer': regularizers.serialize(self.gamma_regularizer), - 'beta_constraint': constraints.serialize(self.beta_constraint), - 'gamma_constraint': constraints.serialize(self.gamma_constraint), + 'axis': self.axis, + 'epsilon': self.epsilon, + 'center': self.center, + 'scale': self.scale, + 'beta_initializer': initializers.serialize(self.beta_initializer), + 'gamma_initializer': initializers.serialize(self.gamma_initializer), + 'beta_regularizer': regularizers.serialize(self.beta_regularizer), + 'gamma_regularizer': regularizers.serialize(self.gamma_regularizer), + 'beta_constraint': constraints.serialize(self.beta_constraint), + 'gamma_constraint': constraints.serialize(self.gamma_constraint), } base_config = super(LayerNormalization, self).get_config() return dict(list(base_config.items()) + list(config.items())) diff --git a/easy_rec/python/layers/keras/mask_net.py b/easy_rec/python/layers/keras/mask_net.py index bf687154e..1611f2941 100644 --- a/easy_rec/python/layers/keras/mask_net.py +++ b/easy_rec/python/layers/keras/mask_net.py @@ -1,11 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf -from tensorflow.python.keras.layers import Activation -from tensorflow.python.keras.layers import Dense -from tensorflow.python.keras.layers import Layer +from tensorflow.python.keras.layers import Activation, Dense, Layer from easy_rec.python.layers.keras.blocks import MLP from easy_rec.python.layers.keras.layer_norm import LayerNormalization @@ -47,36 +44,42 @@ def build(self, input_shape): aggregation_size = self.config.aggregation_size else: raise ValueError( - 'Need one of reduction factor or aggregation size for MaskBlock.') + 'Need one of reduction factor or aggregation size for MaskBlock.' + ) self.aggr_layer = Dense( - aggregation_size, - activation='relu', - kernel_initializer='he_uniform', - kernel_regularizer=self.l2_reg, - name='aggregation') + aggregation_size, + activation='relu', + kernel_initializer='he_uniform', + kernel_regularizer=self.l2_reg, + name='aggregation' + ) self.weight_layer = Dense(input_dim, name='weights') if self._projection_dim is not None: logging.info('%s project dim is %d', self.name, self._projection_dim) self.project_layer = Dense( - self._projection_dim, - kernel_regularizer=self.l2_reg, - use_bias=False, - name='project') + self._projection_dim, + kernel_regularizer=self.l2_reg, + use_bias=False, + name='project' + ) if self.config.input_layer_norm: # 推荐在调用MaskBlock之前做好 layer norm，否则每一次调用都需要对input做ln if tf.__version__ >= '2.0': self.input_layer_norm = tf.keras.layers.LayerNormalization( - name='input_ln') + name='input_ln' + ) else: self.input_layer_norm = LayerNormalization(name='input_ln') if self.config.HasField('output_size'): self.output_layer = Dense( - self.config.output_size, use_bias=False, name='output') + self.config.output_size, use_bias=False, name='output' + ) if tf.__version__ >= '2.0': self.output_layer_norm = tf.keras.layers.LayerNormalization( - name='output_ln') + name='output_ln' + ) else: self.output_layer_norm = LayerNormalization(name='output_ln') super(MaskBlock, self).build(input_shape) @@ -135,7 +138,8 @@ def __init__(self, params, name='mask_net', reuse=None, **kwargs): if self.config.input_layer_norm: if tf.__version__ >= '2.0': self.input_layer_norm = tf.keras.layers.LayerNormalization( - name='input_ln') + name='input_ln' + ) else: self.input_layer_norm = LayerNormalization(name='input_ln') @@ -145,7 +149,7 @@ def call(self, inputs, training=None, **kwargs): if self.config.use_parallel: mask_outputs = [ - mask_layer((inputs, inputs)) for mask_layer in self.mask_layers + mask_layer((inputs, inputs)) for mask_layer in self.mask_layers ] all_mask_outputs = tf.concat(mask_outputs, axis=1) if self.mlp is not None: diff --git a/easy_rec/python/layers/keras/multi_head_attention.py b/easy_rec/python/layers/keras/multi_head_attention.py index a5ca0b40d..a08d1fcab 100644 --- a/easy_rec/python/layers/keras/multi_head_attention.py +++ b/easy_rec/python/layers/keras/multi_head_attention.py @@ -1,16 +1,11 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import math -import string - import numpy as np +import string import tensorflow as tf -from tensorflow.python.keras import constraints -from tensorflow.python.keras import initializers -from tensorflow.python.keras import regularizers -from tensorflow.python.keras.layers import Dropout -from tensorflow.python.keras.layers import Layer -from tensorflow.python.keras.layers import Softmax +from tensorflow.python.keras import constraints, initializers, regularizers +from tensorflow.python.keras.layers import Dropout, Layer, Softmax from easy_rec.python.layers.keras.activation import MaskedSoftmax from easy_rec.python.layers.keras.einsum_dense import EinsumDense @@ -93,7 +88,9 @@ class MultiHeadAttention(Layer): attention axes. """ - def __init__(self, params, name='multi_head_attention', reuse=None, **kwargs): + def __init__( + self, params, name='multi_head_attention', reuse=None, **kwargs + ): super(MultiHeadAttention, self).__init__(name=name, **kwargs) self.supports_masking = True self._num_heads = params.num_heads @@ -106,23 +103,31 @@ def __init__(self, params, name='multi_head_attention', reuse=None, **kwargs): self._use_bias = params.get_or_default('use_bias', True) self._output_shape = params.get_or_default('output_shape', None) self._kernel_initializer = initializers.get( - params.get_or_default('kernel_initializer', 'glorot_uniform')) + params.get_or_default('kernel_initializer', 'glorot_uniform') + ) self._bias_initializer = initializers.get( - params.get_or_default('bias_initializer', 'zeros')) + params.get_or_default('bias_initializer', 'zeros') + ) self._kernel_regularizer = regularizers.get( - params.get_or_default('kernel_regularizer', None)) + params.get_or_default('kernel_regularizer', None) + ) self._bias_regularizer = regularizers.get( - params.get_or_default('bias_regularizer', None)) + params.get_or_default('bias_regularizer', None) + ) self._activity_regularizer = regularizers.get( - params.get_or_default('activity_regularizer', None)) + params.get_or_default('activity_regularizer', None) + ) self._kernel_constraint = constraints.get( - params.get_or_default('kernel_constraint', None)) + params.get_or_default('kernel_constraint', None) + ) self._bias_constraint = constraints.get( - params.get_or_default('bias_constraint', None)) + params.get_or_default('bias_constraint', None) + ) self._attention_axes = params.get_or_default('attention_axes', None) self._use_causal_mask = params.get_or_default('use_causal_mask', False) self._return_attention_scores = params.get_or_default( - 'return_attention_scores', False) + 'return_attention_scores', False + ) @property def num_heads(self): @@ -155,34 +160,21 @@ def attention_axes(self): def get_config(self): base_config = super(MultiHeadAttention, self).get_config() config = { - 'num_heads': - self._num_heads, - 'key_dim': - self._key_dim, - 'value_dim': - self._value_dim, - 'dropout': - self._dropout, - 'use_bias': - self._use_bias, - 'output_shape': - self._output_shape, - 'attention_axes': - self._attention_axes, - 'kernel_initializer': - initializers.serialize(self._kernel_initializer), - 'bias_initializer': - initializers.serialize(self._bias_initializer), - 'kernel_regularizer': - regularizers.serialize(self._kernel_regularizer), - 'bias_regularizer': - regularizers.serialize(self._bias_regularizer), - 'activity_regularizer': - regularizers.serialize(self._activity_regularizer), - 'kernel_constraint': - constraints.serialize(self._kernel_constraint), - 'bias_constraint': - constraints.serialize(self._bias_constraint), + 'num_heads': self._num_heads, + 'key_dim': self._key_dim, + 'value_dim': self._value_dim, + 'dropout': self._dropout, + 'use_bias': self._use_bias, + 'output_shape': self._output_shape, + 'attention_axes': self._attention_axes, + 'kernel_initializer': initializers.serialize(self._kernel_initializer), + 'bias_initializer': initializers.serialize(self._bias_initializer), + 'kernel_regularizer': regularizers.serialize(self._kernel_regularizer), + 'bias_regularizer': regularizers.serialize(self._bias_regularizer), + 'activity_regularizer': + regularizers.serialize(self._activity_regularizer), + 'kernel_constraint': constraints.serialize(self._kernel_constraint), + 'bias_constraint': constraints.serialize(self._bias_constraint), } config.update(base_config) return config @@ -202,46 +194,56 @@ def build(self, input_shape): value_rank = len(value_shape) key_rank = len(key_shape) einsum_equation, bias_axes, output_rank = _build_proj_equation( - query_rank - 1, bound_dims=1, output_dims=2) + query_rank - 1, bound_dims=1, output_dims=2 + ) self._query_dense = EinsumDense( - einsum_equation, - output_shape=_get_output_shape(output_rank - 1, - [self._num_heads, self._key_dim]), - bias_axes=bias_axes if self._use_bias else None, - name='query', - **self._get_common_kwargs_for_sublayer()) + einsum_equation, + output_shape=_get_output_shape( + output_rank - 1, [self._num_heads, self._key_dim] + ), + bias_axes=bias_axes if self._use_bias else None, + name='query', + **self._get_common_kwargs_for_sublayer() + ) self._query_dense.build(query_shape) einsum_equation, bias_axes, output_rank = _build_proj_equation( - key_rank - 1, bound_dims=1, output_dims=2) + key_rank - 1, bound_dims=1, output_dims=2 + ) self._key_dense = EinsumDense( - einsum_equation, - output_shape=_get_output_shape(output_rank - 1, - [self._num_heads, self._key_dim]), - bias_axes=bias_axes if self._use_bias else None, - name='key', - **self._get_common_kwargs_for_sublayer()) + einsum_equation, + output_shape=_get_output_shape( + output_rank - 1, [self._num_heads, self._key_dim] + ), + bias_axes=bias_axes if self._use_bias else None, + name='key', + **self._get_common_kwargs_for_sublayer() + ) self._key_dense.build(key_shape) einsum_equation, bias_axes, output_rank = _build_proj_equation( - value_rank - 1, bound_dims=1, output_dims=2) + value_rank - 1, bound_dims=1, output_dims=2 + ) self._value_dense = EinsumDense( - einsum_equation, - output_shape=_get_output_shape(output_rank - 1, - [self._num_heads, self._value_dim]), - bias_axes=bias_axes if self._use_bias else None, - name='value', - **self._get_common_kwargs_for_sublayer()) + einsum_equation, + output_shape=_get_output_shape( + output_rank - 1, [self._num_heads, self._value_dim] + ), + bias_axes=bias_axes if self._use_bias else None, + name='value', + **self._get_common_kwargs_for_sublayer() + ) self._value_dense.build(value_shape) # Builds the attention computations for multi-head dot product # attention. These computations could be wrapped into the keras # attention layer once it supports multi-head einsum computations. self._build_attention(output_rank) self._output_dense = self._make_output_dense( - query_shape, - self._get_common_kwargs_for_sublayer(), - 'attention_output', + query_shape, + self._get_common_kwargs_for_sublayer(), + 'attention_output', ) output_dense_input_shape = list( - self._query_dense.compute_output_shape(query_shape)) + self._query_dense.compute_output_shape(query_shape) + ) output_dense_input_shape[-1] = self._value_dim self._output_dense.build(tuple(output_dense_input_shape)) self.built = True @@ -265,20 +267,22 @@ def output_dense(self): def _get_common_kwargs_for_sublayer(self): common_kwargs = dict( - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer, - activity_regularizer=self._activity_regularizer, - kernel_constraint=self._kernel_constraint, - bias_constraint=self._bias_constraint, - dtype=tf.float32, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + dtype=tf.float32, ) # Create new clone of kernel/bias initializer, so that we don't reuse # the initializer instance, which could lead to same init value since # initializer is stateless. kernel_initializer = self._kernel_initializer.__class__.from_config( - self._kernel_initializer.get_config()) + self._kernel_initializer.get_config() + ) bias_initializer = self._bias_initializer.__class__.from_config( - self._bias_initializer.get_config()) + self._bias_initializer.get_config() + ) common_kwargs['kernel_initializer'] = kernel_initializer common_kwargs['bias_initializer'] = bias_initializer return common_kwargs @@ -303,13 +307,15 @@ def _make_output_dense(self, query_shape, common_kwargs, name=None): else: output_shape = [query_shape[-1]] einsum_equation, bias_axes, output_rank = _build_proj_equation( - query_rank - 1, bound_dims=2, output_dims=len(output_shape)) + query_rank - 1, bound_dims=2, output_dims=len(output_shape) + ) return EinsumDense( - einsum_equation, - output_shape=_get_output_shape(output_rank - 1, output_shape), - bias_axes=bias_axes if self._use_bias else None, - name=name, - **common_kwargs) + einsum_equation, + output_shape=_get_output_shape(output_rank - 1, output_shape), + bias_axes=bias_axes if self._use_bias else None, + name=name, + **common_kwargs + ) def _build_attention(self, rank): """Builds multi-head dot-product attention computations. @@ -326,16 +332,16 @@ def _build_attention(self, rank): else: self._attention_axes = tuple(self._attention_axes) ( - self._dot_product_equation, - self._combine_equation, - attn_scores_rank, - ) = _build_attention_equation( - rank, attn_axes=self._attention_axes) + self._dot_product_equation, + self._combine_equation, + attn_scores_rank, + ) = _build_attention_equation(rank, attn_axes=self._attention_axes) norm_axes = tuple( - range(attn_scores_rank - len(self._attention_axes), attn_scores_rank)) + range(attn_scores_rank - len(self._attention_axes), attn_scores_rank) + ) self._softmax = Softmax( - axis=norm_axes) if tf.__version__ >= '2.0' else MaskedSoftmax( - axis=norm_axes) + axis=norm_axes + ) if tf.__version__ >= '2.0' else MaskedSoftmax(axis=norm_axes) self._dropout_layer = Dropout(rate=self._dropout) self._inverse_sqrt_key_dim = 1.0 / math.sqrt(float(self._key_dim)) @@ -349,15 +355,13 @@ def _masked_softmax(self, attention_scores, attention_mask=None): mask_expansion_axis = -len(self._attention_axes) * 2 - 1 for _ in range(len(attention_scores.shape) - len(attention_mask.shape)): attention_mask = tf.expand_dims( - attention_mask, axis=mask_expansion_axis) + attention_mask, axis=mask_expansion_axis + ) return self._softmax(attention_scores, mask=attention_mask) - def _compute_attention(self, - query, - key, - value, - attention_mask=None, - training=None): + def _compute_attention( + self, query, key, value, attention_mask=None, training=None + ): """Applies Dot-product attention with query, key, value tensors. This function defines the computation inside `call` with projected @@ -382,7 +386,9 @@ def _compute_attention(self, # Note: Applying scalar multiply at the smaller end of einsum improves # XLA performance, but may introduce slight numeric differences in # the Transformer attention head. - query = tf.multiply(query, tf.cast(self._inverse_sqrt_key_dim, query.dtype)) + query = tf.multiply( + query, tf.cast(self._inverse_sqrt_key_dim, query.dtype) + ) # Take the dot product between "query" and "key" to get the raw # attention scores. @@ -394,18 +400,21 @@ def _compute_attention(self, # seem a bit unusual, but is taken from the original Transformer paper. if self.dropout: final_attn_scores = self._dropout_layer( - attention_scores, training=training) + attention_scores, training=training + ) else: final_attn_scores = attention_scores # `context_layer` = [B, T, N, H] - attention_output = tf.einsum(self._combine_equation, final_attn_scores, - value) + attention_output = tf.einsum( + self._combine_equation, final_attn_scores, value + ) return attention_output, attention_scores def call(self, inputs, mask=None, training=None, **kwargs): assert isinstance( - inputs, (tuple, list)), 'inputs of MultiHeadAttention must be a list' + inputs, (tuple, list) + ), 'inputs of MultiHeadAttention must be a list' query, value, key = (list(inputs) + [None] * 2)[:3] if key is None: key = value @@ -419,13 +428,13 @@ def call(self, inputs, mask=None, training=None, **kwargs): if attention_mask is None and value_mask is None: value_mask = query_mask attention_mask = self._compute_attention_mask( - query, - value, - query_mask=query_mask, - value_mask=value_mask, - key_mask=key_mask, - attention_mask=attention_mask, - use_causal_mask=self._use_causal_mask, + query, + value, + query_mask=query_mask, + value_mask=value_mask, + key_mask=key_mask, + attention_mask=attention_mask, + use_causal_mask=self._use_causal_mask, ) # N = `num_attention_heads` @@ -439,21 +448,22 @@ def call(self, inputs, mask=None, training=None, **kwargs): # `value` = [B, S, N, H] value = self._value_dense(value) attention_output, attention_scores = self._compute_attention( - query, key, value, attention_mask, training) + query, key, value, attention_mask, training + ) attention_output = self._output_dense(attention_output) if self._return_attention_scores: return attention_output, attention_scores return attention_output def _compute_attention_mask( - self, - query, - value, - query_mask=None, - value_mask=None, - key_mask=None, - attention_mask=None, - use_causal_mask=False, + self, + query, + value, + query_mask=None, + value_mask=None, + key_mask=None, + attention_mask=None, + use_causal_mask=False, ): """Computes the attention mask, using the Keras masks of the inputs. @@ -506,8 +516,9 @@ def _compute_attention_mask( if auto_mask is not None: # merge attention_mask & automatic mask, to shape [B, T, S] attention_mask = ( - auto_mask if attention_mask is None else - tf.cast(attention_mask, tf.bool) & auto_mask) + auto_mask if attention_mask is None else + tf.cast(attention_mask, tf.bool) & auto_mask + ) return attention_mask def _compute_causal_mask(self, query, value=None): @@ -552,21 +563,25 @@ def compute_output_shape(self, input_shape): if query_shape[-1] != value_shape[-1]: raise ValueError( - 'The last dimension of `query_shape` and `value_shape` ' - 'must be equal, but are {query_last_dim}, {value_last_dim}. ' - 'Received: query_shape={query_shape}, value_shape={value_shape}' - .format( - query_shape=query_shape, - value_shape=value_shape, - query_last_dim=query_shape[-1], - value_last_dim=value_shape[-1])) + 'The last dimension of `query_shape` and `value_shape` ' + 'must be equal, but are {query_last_dim}, {value_last_dim}. ' + 'Received: query_shape={query_shape}, value_shape={value_shape}'. + format( + query_shape=query_shape, + value_shape=value_shape, + query_last_dim=query_shape[-1], + value_last_dim=value_shape[-1] + ) + ) if value_shape[1:-1] != key_shape[1:-1]: raise ValueError( - 'All dimensions of `value` and `key`, except the last one, ' - 'must be equal. Received: value_shape={value_shape} and ' - 'key_shape={key_shape}'.format( - key_shape=key_shape, value_shape=value_shape)) + 'All dimensions of `value` and `key`, except the last one, ' + 'must be equal. Received: value_shape={value_shape} and ' + 'key_shape={key_shape}'.format( + key_shape=key_shape, value_shape=value_shape + ) + ) if self._output_shape: if hasattr(self._output_dense, '__len__'): @@ -614,7 +629,7 @@ def _build_attention_equation(rank, attn_axes): for i in range(rank): target_notation += _index_to_einsum_variable(i) # `batch_dims` includes the head dim. - batch_dims = tuple(np.delete(range(rank), attn_axes + (rank - 1,))) + batch_dims = tuple(np.delete(range(rank), attn_axes + (rank - 1, ))) letter_offset = rank source_notation = '' for i in range(rank): @@ -624,19 +639,21 @@ def _build_attention_equation(rank, attn_axes): source_notation += _index_to_einsum_variable(letter_offset) letter_offset += 1 - product_notation = ''.join([target_notation[i] for i in batch_dims] + - [target_notation[i] for i in attn_axes] + - [source_notation[i] for i in attn_axes]) + product_notation = ''.join( + [target_notation[i] + for i in batch_dims] + [target_notation[i] for i in attn_axes] + + [source_notation[i] for i in attn_axes] + ) dot_product_equation = '%s,%s->%s' % ( - source_notation, - target_notation, - product_notation, + source_notation, + target_notation, + product_notation, ) attn_scores_rank = len(product_notation) combine_equation = '%s,%s->%s' % ( - product_notation, - source_notation, - target_notation, + product_notation, + source_notation, + target_notation, ) return dot_product_equation, combine_equation, attn_scores_rank @@ -666,7 +683,8 @@ def _build_proj_equation(free_dims, bound_dims, output_dims): output_str += char bias_axes += char equation = '{input_str},{kernel_str}->{output_str}'.format( - input_str=input_str, kernel_str=kernel_str, output_str=output_str) + input_str=input_str, kernel_str=kernel_str, output_str=output_str + ) return equation, bias_axes, len(output_str) diff --git a/easy_rec/python/layers/keras/multi_task.py b/easy_rec/python/layers/keras/multi_task.py index dbb26ee86..24ec973f5 100644 --- a/easy_rec/python/layers/keras/multi_task.py +++ b/easy_rec/python/layers/keras/multi_task.py @@ -1,10 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf -from tensorflow.python.keras.layers import Dense -from tensorflow.python.keras.layers import Layer +from tensorflow.python.keras.layers import Dense, Layer from easy_rec.python.layers.keras.attention import Attention from easy_rec.python.layers.keras.blocks import MLP @@ -29,8 +27,8 @@ def __init__(self, params, name='MMoE', reuse=None, **kwargs): expert_params.l2_regularizer = params.l2_regularizer self._has_experts = True self._experts = [ - MLP(expert_params, 'expert_%d' % i, reuse=reuse) - for i in range(self._num_expert) + MLP(expert_params, 'expert_%d' % i, reuse=reuse) + for i in range(self._num_expert) ] else: self._has_experts = False @@ -38,10 +36,11 @@ def __init__(self, params, name='MMoE', reuse=None, **kwargs): self._gates = [] for task_id in range(self._num_task): dense = Dense( - self._num_expert, - activation='softmax', - name='gate_%d' % task_id, - kernel_regularizer=params.l2_regularizer) + self._num_expert, + activation='softmax', + name='gate_%d' % task_id, + kernel_regularizer=params.l2_regularizer + ) self._gates.append(dense) def call(self, inputs, training=None, **kwargs): @@ -50,7 +49,7 @@ def call(self, inputs, training=None, **kwargs): return inputs if self._has_experts: expert_fea_list = [ - expert(inputs, training=training) for expert in self._experts + expert(inputs, training=training) for expert in self._experts ] else: expert_fea_list = inputs diff --git a/easy_rec/python/layers/keras/numerical_embedding.py b/easy_rec/python/layers/keras/numerical_embedding.py index 65cc77d52..fef1c0f0b 100644 --- a/easy_rec/python/layers/keras/numerical_embedding.py +++ b/easy_rec/python/layers/keras/numerical_embedding.py @@ -3,7 +3,6 @@ import logging import math import os - import tensorflow as tf from tensorflow.python.framework import ops from tensorflow.python.keras.layers import Layer @@ -34,8 +33,9 @@ custom_ops = tf.load_op_library(custom_op_path) logging.info('load custom op from %s succeed' % custom_op_path) except Exception as ex: - logging.warning('load custom op from %s failed: %s' % - (custom_op_path, str(ex))) + logging.warning( + 'load custom op from %s failed: %s' % (custom_op_path, str(ex)) + ) custom_ops = None @@ -68,13 +68,9 @@ class NLinear(Layer): assert m(x).shape == (batch_size, n_features, d_embedding_out) """ - def __init__(self, - n_tokens, - d_in, - d_out, - bias=True, - name='nd_linear', - **kwargs): + def __init__( + self, n_tokens, d_in, d_out, bias=True, name='nd_linear', **kwargs + ): """Init with input shapes. Args: @@ -86,24 +82,28 @@ def __init__(self, """ super(NLinear, self).__init__(name=name, **kwargs) self.weight = self.add_weight( - 'weights', [1, n_tokens, d_in, d_out], dtype=tf.float32) + 'weights', [1, n_tokens, d_in, d_out], dtype=tf.float32 + ) if bias: initializer = tf.constant_initializer(0.0) self.bias = self.add_weight( - 'bias', [1, n_tokens, d_out], - dtype=tf.float32, - initializer=initializer) + 'bias', [1, n_tokens, d_out], + dtype=tf.float32, + initializer=initializer + ) else: self.bias = None def call(self, x, **kwargs): if x.shape.ndims != 3: raise ValueError( - 'The input must have three dimensions (batch_size, n_tokens, d_embedding)' + 'The input must have three dimensions (batch_size, n_tokens, d_embedding)' ) if x.shape[2] != self.weight.shape[2]: - raise ValueError('invalid input embedding dimension %d, expect %d' % - (int(x.shape[2]), int(self.weight.shape[2]))) + raise ValueError( + 'invalid input embedding dimension %d, expect %d' % + (int(x.shape[2]), int(self.weight.shape[2])) + ) x = x[..., None] * self.weight # [B, N, D, D_out] x = tf.reduce_sum(x, axis=-2) # [B, N, D_out] @@ -145,7 +145,9 @@ def __init__(self, params, name='periodic_embedding', reuse=None, **kwargs): self.initializer = tf.random_normal_initializer(stddev=sigma) self.add_linear_layer = params.get_or_default('add_linear_layer', True) self.linear_activation = params.get_or_default('linear_activation', 'relu') - self.output_tensor_list = params.get_or_default('output_tensor_list', False) + self.output_tensor_list = params.get_or_default( + 'output_tensor_list', False + ) self.output_3d_tensor = params.get_or_default('output_3d_tensor', False) def build(self, input_shape): @@ -158,16 +160,18 @@ def build(self, input_shape): partitioner = tf.fixed_size_partitioner(num_shards=num_ps) emb_dim = self.embedding_dim // 2 self.coef = self.add_weight( - 'coefficients', - shape=[1, self.num_features, emb_dim], - partitioner=partitioner, - initializer=self.initializer) + 'coefficients', + shape=[1, self.num_features, emb_dim], + partitioner=partitioner, + initializer=self.initializer + ) if self.add_linear_layer: self.linear = NLinear( - self.num_features, - self.embedding_dim, - self.embedding_dim, - name='nd_linear') + self.num_features, + self.embedding_dim, + self.embedding_dim, + name='nd_linear' + ) super(PeriodicEmbedding, self).build(input_shape) def call(self, inputs, **kwargs): @@ -204,7 +208,9 @@ def __init__(self, params, name='auto_dis_embedding', reuse=None, **kwargs): self.num_bins = int(params.num_bins) self.temperature = params.temperature self.keep_prob = params.get_or_default('keep_prob', 0.8) - self.output_tensor_list = params.get_or_default('output_tensor_list', False) + self.output_tensor_list = params.get_or_default( + 'output_tensor_list', False + ) self.output_3d_tensor = params.get_or_default('output_3d_tensor', False) def build(self, input_shape): @@ -216,17 +222,20 @@ def build(self, input_shape): if num_ps > 0: partitioner = tf.fixed_size_partitioner(num_shards=num_ps) self.meta_emb = self.add_weight( - 'meta_embedding', - shape=[self.num_features, self.num_bins, self.emb_dim], - partitioner=partitioner) + 'meta_embedding', + shape=[self.num_features, self.num_bins, self.emb_dim], + partitioner=partitioner + ) self.proj_w = self.add_weight( - 'project_w', - shape=[1, self.num_features, self.num_bins], - partitioner=partitioner) + 'project_w', + shape=[1, self.num_features, self.num_bins], + partitioner=partitioner + ) self.proj_mat = self.add_weight( - 'project_mat', - shape=[self.num_features, self.num_bins, self.num_bins], - partitioner=partitioner) + 'project_mat', + shape=[self.num_features, self.num_bins, self.num_bins], + partitioner=partitioner + ) super(AutoDisEmbedding, self).build(input_shape) def call(self, inputs, **kwargs): @@ -245,7 +254,9 @@ def call(self, inputs, **kwargs): # emb = tf.matmul(x_hat[:, :, None, :], meta_emb) # [B, N, 1, D] # emb = tf.squeeze(emb, axis=2) # [B, N, D] emb = tf.einsum('bnk,nkd->bnd', x_hat, self.meta_emb) - output = tf.reshape(emb, [-1, self.emb_dim * self.num_features]) # [B, N*D] + output = tf.reshape( + emb, [-1, self.emb_dim * self.num_features] + ) # [B, N*D] if self.output_tensor_list: return output, tf.unstack(emb, axis=1) @@ -274,12 +285,17 @@ def __init__(self, params, name='nary_dis_embedding', reuse=None, **kwargs): self.multiplier = params.get_or_default('multiplier', 1.0) self.intra_ary_pooling = params.get_or_default('intra_ary_pooling', 'sum') self.output_3d_tensor = params.get_or_default('output_3d_tensor', False) - self.output_tensor_list = params.get_or_default('output_tensor_list', False) + self.output_tensor_list = params.get_or_default( + 'output_tensor_list', False + ) logging.info( - '{} carries: {}, lengths: {}, vocab_size: {}, intra_ary: {}, replicas: {}, multiplier: {}' - .format(self.name, ','.join(map(str, self.carries)), - ','.join(map(str, self.lengths)), self.vocab_size, - self.intra_ary_pooling, self.num_replicas, self.multiplier)) + '{} carries: {}, lengths: {}, vocab_size: {}, intra_ary: {}, replicas: {}, multiplier: {}' + .format( + self.name, ','.join(map(str, self.carries)), + ','.join(map(str, self.lengths)), self.vocab_size, + self.intra_ary_pooling, self.num_replicas, self.multiplier + ) + ) @staticmethod def max_length(carry): @@ -287,8 +303,9 @@ def max_length(carry): return (math.floor(bits) + 1) * carry def build(self, input_shape): - assert isinstance(input_shape, - tf.TensorShape), 'NaryDisEmbedding only takes 1 input' + assert isinstance( + input_shape, tf.TensorShape + ), 'NaryDisEmbedding only takes 1 input' self.num_features = int(input_shape[-1]) logging.info('%s has %d input features', self.name, self.num_features) vocab_size = self.num_features * self.vocab_size @@ -298,7 +315,8 @@ def build(self, input_shape): if num_ps > 0: partitioner = tf.fixed_size_partitioner(num_shards=num_ps) self.embedding_table = self.add_weight( - 'embed_table', shape=[vocab_size, emb_dim], partitioner=partitioner) + 'embed_table', shape=[vocab_size, emb_dim], partitioner=partitioner + ) super(NaryDisEmbedding, self).build(input_shape) def call(self, inputs, **kwargs): @@ -333,8 +351,9 @@ def call(self, inputs, **kwargs): segment_ids = repeat(tf.range(total_length), repeats=splits) embedding = tf.math.segment_mean(embedding, segment_ids) else: - raise ValueError('Unsupported intra ary pooling method %s' % - self.intra_ary_pooling) + raise ValueError( + 'Unsupported intra ary pooling method %s' % self.intra_ary_pooling + ) # B: batch size # N: num features # C: num carries diff --git a/easy_rec/python/layers/keras/ppnet.py b/easy_rec/python/layers/keras/ppnet.py index 431034924..204b700b2 100644 --- a/easy_rec/python/layers/keras/ppnet.py +++ b/easy_rec/python/layers/keras/ppnet.py @@ -2,7 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Convenience blocks for building models.""" import logging - import tensorflow as tf from easy_rec.python.layers.keras.activation import activation_layer @@ -14,12 +13,9 @@ class GateNN(tf.keras.layers.Layer): - def __init__(self, - params, - output_units=None, - name='gate_nn', - reuse=None, - **kwargs): + def __init__( + self, params, output_units=None, name='gate_nn', reuse=None, **kwargs + ): super(GateNN, self).__init__(name=name, **kwargs) output_dim = output_units if output_units is not None else params.output_dim hidden_dim = params.get_or_default('hidden_dim', output_dim) @@ -30,9 +26,10 @@ def __init__(self, self._sub_layers = [] dense = tf.keras.layers.Dense( - units=hidden_dim, - use_bias=not do_batch_norm, - kernel_initializer=initializer) + units=hidden_dim, + use_bias=not do_batch_norm, + kernel_initializer=initializer + ) self._sub_layers.append(dense) if do_batch_norm: @@ -49,11 +46,12 @@ def __init__(self, raise ValueError('invalid dropout_ratio: %.3f' % dropout_rate) dense = tf.keras.layers.Dense( - units=output_dim, - activation='sigmoid', - use_bias=not do_batch_norm, - kernel_initializer=initializer, - name='weight') + units=output_dim, + activation='sigmoid', + use_bias=not do_batch_norm, + kernel_initializer=initializer, + name='weight' + ) self._sub_layers.append(dense) self._sub_layers.append(lambda x: x * 2) @@ -100,10 +98,12 @@ def __init__(self, params, name='ppnet', reuse=None, **kwargs): use_bn_after_act = params.get_or_default('use_bn_after_activation', False) units = list(params.hidden_units) logging.info( - 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' - ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' % - (name, units, dropout_rate, activation, use_bn, use_final_bn, - final_activation, use_bias, initializer, use_bn_after_act)) + 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' + ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' % ( + name, units, dropout_rate, activation, use_bn, use_final_bn, + final_activation, use_bias, initializer, use_bn_after_act + ) + ) assert len(units) > 0, 'MLP(%s) takes at least one hidden units' % name self.reuse = reuse @@ -115,52 +115,61 @@ def __init__(self, params, name='ppnet', reuse=None, **kwargs): for i, num_units in enumerate(units[:-1]): name = 'layer_%d' % i drop_rate = dropout_rate[i] if i < num_dropout else 0.0 - self.add_rich_layer(num_units, use_bn, drop_rate, activation, initializer, - use_bias, use_bn_after_act, name, - params.l2_regularizer) + self.add_rich_layer( + num_units, use_bn, drop_rate, activation, initializer, use_bias, + use_bn_after_act, name, params.l2_regularizer + ) self._sub_layers.append( - GateNN(gate_params, num_units, 'gate_%d' % (i + 1))) + GateNN(gate_params, num_units, 'gate_%d' % (i + 1)) + ) n = len(units) - 1 drop_rate = dropout_rate[n] if num_dropout > n else 0.0 name = 'layer_%d' % n - self.add_rich_layer(units[-1], use_final_bn, drop_rate, final_activation, - initializer, use_final_bias, use_bn_after_act, name, - params.l2_regularizer) + self.add_rich_layer( + units[-1], use_final_bn, drop_rate, final_activation, initializer, + use_final_bias, use_bn_after_act, name, params.l2_regularizer + ) if mode == 'lazy': self._sub_layers.append( - GateNN(gate_params, units[-1], 'gate_%d' % (n + 1))) - - def add_rich_layer(self, - num_units, - use_bn, - dropout_rate, - activation, - initializer, - use_bias, - use_bn_after_activation, - name, - l2_reg=None): + GateNN(gate_params, units[-1], 'gate_%d' % (n + 1)) + ) + + def add_rich_layer( + self, + num_units, + use_bn, + dropout_rate, + activation, + initializer, + use_bias, + use_bn_after_activation, + name, + l2_reg=None + ): act_layer = activation_layer(activation, name='%s/act' % name) if use_bn and not use_bn_after_activation: dense = tf.keras.layers.Dense( - units=num_units, - use_bias=use_bias, - kernel_initializer=initializer, - kernel_regularizer=l2_reg, - name='%s/dense' % name) + units=num_units, + use_bias=use_bias, + kernel_initializer=initializer, + kernel_regularizer=l2_reg, + name='%s/dense' % name + ) self._sub_layers.append(dense) bn = tf.keras.layers.BatchNormalization( - name='%s/bn' % name, trainable=True) + name='%s/bn' % name, trainable=True + ) self._sub_layers.append(bn) self._sub_layers.append(act_layer) else: dense = tf.keras.layers.Dense( - num_units, - use_bias=use_bias, - kernel_initializer=initializer, - kernel_regularizer=l2_reg, - name='%s/dense' % name) + num_units, + use_bias=use_bias, + kernel_initializer=initializer, + kernel_regularizer=l2_reg, + name='%s/dense' % name + ) self._sub_layers.append(dense) self._sub_layers.append(act_layer) if use_bn and use_bn_after_activation: diff --git a/easy_rec/python/layers/keras/transformer.py b/easy_rec/python/layers/keras/transformer.py index d71a02831..c2cf8fd84 100644 --- a/easy_rec/python/layers/keras/transformer.py +++ b/easy_rec/python/layers/keras/transformer.py @@ -1,13 +1,9 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import numpy as np import tensorflow as tf -from tensorflow.python.keras.layers import Dense -from tensorflow.python.keras.layers import Dropout -from tensorflow.python.keras.layers import Embedding -from tensorflow.python.keras.layers import Layer +from tensorflow.python.keras.layers import Dense, Dropout, Embedding, Layer from easy_rec.python.layers.keras import MultiHeadAttention from easy_rec.python.layers.keras.layer_norm import LayerNormalization @@ -30,7 +26,9 @@ def __init__(self, params, name='transformer_block', reuse=None, **kwargs): mha_cfg = seq_encoder_pb2.MultiHeadAttention() mha_cfg.num_heads = num_heads mha_cfg.key_dim = d_model // num_heads - mha_cfg.dropout = params.get_or_default('attention_probs_dropout_prob', 0.0) + mha_cfg.dropout = params.get_or_default( + 'attention_probs_dropout_prob', 0.0 + ) mha_cfg.return_attention_scores = False args = Parameter.make_from_pb(mha_cfg) self.mha = MultiHeadAttention(args, 'multi_head_attn') @@ -68,7 +66,8 @@ def positional_encoding(length, depth): angle_rates = 1 / (10000**depths) # (1, depth) angle_rads = positions * angle_rates # (pos, depth) pos_encoding = np.concatenate( - [np.sin(angle_rads), np.cos(angle_rads)], axis=-1) + [np.sin(angle_rads), np.cos(angle_rads)], axis=-1 + ) return tf.cast(pos_encoding, dtype=tf.float32) @@ -105,12 +104,16 @@ def __init__(self, params, name='transformer_encoder', reuse=None, **kwargs): max_position = params.get_or_default('max_position_embeddings', 512) num_layers = params.get_or_default('num_hidden_layers', 1) vocab_size = params.vocab_size - logging.info('vocab size of TransformerEncoder(%s) is %d', name, vocab_size) - self.output_all = params.get_or_default('output_all_token_embeddings', True) + logging.info( + 'vocab size of TransformerEncoder(%s) is %d', name, vocab_size + ) + self.output_all = params.get_or_default( + 'output_all_token_embeddings', True + ) self.pos_encoding = PositionalEmbedding(vocab_size, d_model, max_position) self.dropout = Dropout(dropout_rate) self.enc_layers = [ - TransformerBlock(params, 'layer_%d' % i) for i in range(num_layers) + TransformerBlock(params, 'layer_%d' % i) for i in range(num_layers) ] self._vocab_size = vocab_size self._max_position = max_position @@ -148,9 +151,10 @@ def __init__(self, params, name='text_encoder', reuse=None, **kwargs): self.default_token_id = params.get_or_default('default_token_id', 0) if vocab_file is not None: self.vocab = tf.feature_column.categorical_column_with_vocabulary_file( - 'tokens', - vocabulary_file=vocab_file, - default_value=self.default_token_id) + 'tokens', + vocabulary_file=vocab_file, + default_value=self.default_token_id + ) logging.info('vocab file of TextEncoder(%s) is %s', name, vocab_file) trans_params.vocab_size = self.vocab.vocabulary_size self.encoder = TransformerEncoder(trans_params, name='transformer') @@ -172,20 +176,24 @@ def call(self, inputs, training=None, **kwargs): features = {'tokens': tokens} token_ids = self.vocab._transform_feature(features) token_ids = tf.sparse.to_dense( - token_ids, default_value=self.default_token_id, name='token_ids') + token_ids, default_value=self.default_token_id, name='token_ids' + ) length = tf.shape(token_ids)[-1] token_ids = tf.cond( - tf.less_equal(length, self.encoder.max_position), lambda: token_ids, - lambda: tf.slice(token_ids, [0, 0], [-1, self.encoder.max_position])) + tf.less_equal(length, self.encoder.max_position), lambda: token_ids, + lambda: tf.slice(token_ids, [0, 0], [-1, self.encoder.max_position]) + ) mask = tf.not_equal(token_ids, self.default_token_id, name='mask') else: tokens = tf.sparse.to_dense(tokens, default_value='') length = tf.shape(tokens)[-1] tokens = tf.cond( - tf.less_equal(length, self.encoder.max_position), lambda: tokens, - lambda: tf.slice(tokens, [0, 0], [-1, self.encoder.max_position])) + tf.less_equal(length, self.encoder.max_position), lambda: tokens, + lambda: tf.slice(tokens, [0, 0], [-1, self.encoder.max_position]) + ) token_ids = tf.string_to_hash_bucket_fast( - tokens, self.encoder.vocab_size, name='token_ids') + tokens, self.encoder.vocab_size, name='token_ids' + ) mask = tf.not_equal(tokens, '', name='mask') encoding = self.encoder([token_ids, mask], training=training) diff --git a/easy_rec/python/layers/layer_norm.py b/easy_rec/python/layers/layer_norm.py index 7e86963ad..c758ec06d 100644 --- a/easy_rec/python/layers/layer_norm.py +++ b/easy_rec/python/layers/layer_norm.py @@ -17,13 +17,15 @@ def __init__(self, hidden_size, params={}): def build(self, _): self.scale = tf.get_variable( - 'layer_norm_scale', [self.hidden_size], - initializer=tf.keras.initializers.Ones(), - dtype=tf.float32) + 'layer_norm_scale', [self.hidden_size], + initializer=tf.keras.initializers.Ones(), + dtype=tf.float32 + ) self.bias = tf.get_variable( - 'layer_norm_bias', [self.hidden_size], - initializer=tf.keras.initializers.Zeros(), - dtype=tf.float32) + 'layer_norm_bias', [self.hidden_size], + initializer=tf.keras.initializers.Zeros(), + dtype=tf.float32 + ) self.built = True def call(self, x): diff --git a/easy_rec/python/layers/mmoe.py b/easy_rec/python/layers/mmoe.py index 4e409a7c8..bfac751a1 100644 --- a/easy_rec/python/layers/mmoe.py +++ b/easy_rec/python/layers/mmoe.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.layers import dnn @@ -12,13 +11,15 @@ class MMOE: - def __init__(self, - expert_dnn_config, - l2_reg, - num_task, - num_expert=None, - name='mmoe', - is_training=False): + def __init__( + self, + expert_dnn_config, + l2_reg, + num_task, + num_expert=None, + name='mmoe', + is_training=False + ): """Initializes a `DNN` Layer. Args: @@ -52,10 +53,11 @@ def num_expert(self): def gate(self, unit, deep_fea, name): fea = tf.layers.dense( - inputs=deep_fea, - units=unit, - kernel_regularizer=self._l2_reg, - name='%s/dnn' % name) + inputs=deep_fea, + units=unit, + kernel_regularizer=self._l2_reg, + name='%s/dnn' % name + ) fea = tf.nn.softmax(fea, axis=1) return fea @@ -64,10 +66,11 @@ def __call__(self, deep_fea): for expert_id in range(self._num_expert): expert_dnn_config = self._expert_dnn_configs[expert_id] expert_dnn = dnn.DNN( - expert_dnn_config, - self._l2_reg, - name='%s/expert_%d' % (self._name, expert_id), - is_training=self._is_training) + expert_dnn_config, + self._l2_reg, + name='%s/expert_%d' % (self._name, expert_id), + is_training=self._is_training + ) expert_fea = expert_dnn(deep_fea) expert_fea_list.append(expert_fea) experts_fea = tf.stack(expert_fea_list, axis=1) @@ -75,7 +78,8 @@ def __call__(self, deep_fea): task_input_list = [] for task_id in range(self._num_task): gate = self.gate( - self._num_expert, deep_fea, name='%s/gate_%d' % (self._name, task_id)) + self._num_expert, deep_fea, name='%s/gate_%d' % (self._name, task_id) + ) gate = tf.expand_dims(gate, -1) task_input = tf.multiply(experts_fea, gate) task_input = tf.reduce_sum(task_input, axis=1) diff --git a/easy_rec/python/layers/multihead_attention.py b/easy_rec/python/layers/multihead_attention.py index 1da28e5d4..b68429164 100644 --- a/easy_rec/python/layers/multihead_attention.py +++ b/easy_rec/python/layers/multihead_attention.py @@ -38,13 +38,16 @@ def _split_multihead_qkv(self, q, k, v): v: Value matrix of shape [bs, head_num, feature_num, head_size]. """ reshaped_q = tf.reshape( - q, shape=[-1, q.shape[1], self._head_num, self._head_size]) + q, shape=[-1, q.shape[1], self._head_num, self._head_size] + ) q = tf.transpose(reshaped_q, perm=[0, 2, 1, 3]) reshaped_k = tf.reshape( - k, shape=[-1, k.shape[1], self._head_num, self._head_size]) + k, shape=[-1, k.shape[1], self._head_num, self._head_size] + ) k = tf.transpose(reshaped_k, perm=[0, 2, 1, 3]) reshaped_v = tf.reshape( - v, shape=[-1, v.shape[1], self._head_num, self._head_size]) + v, shape=[-1, v.shape[1], self._head_num, self._head_size] + ) v = tf.transpose(reshaped_v, perm=[0, 2, 1, 3]) return q, k, v @@ -61,9 +64,8 @@ def _scaled_dot_product_attention(self, q, k, v): k: Key matrix of shape [bs, head_num, feature_num, head_size]. v: Value matrix of shape [bs, head_num, feature_num, head_size]. """ - product = tf.linalg.matmul( - a=q, b=k, transpose_b=True) / ( - self._head_size**-0.5) + product = tf.linalg.matmul(a=q, b=k, + transpose_b=True) / (self._head_size**-0.5) weights = tf.nn.softmax(product) out = tf.linalg.matmul(weights, v) return out @@ -82,23 +84,26 @@ def _compute_qkv(self, q, k, v): v: Value matrix of shape [bs, feature_num, head_size * n_head]. """ q = tf.layers.dense( - q, - self._head_num * self._head_size, - use_bias=False, - kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'query')) + q, + self._head_num * self._head_size, + use_bias=False, + kernel_regularizer=self._l2_reg, + name='%s/%s/dnn' % (self._name, 'query') + ) k = tf.layers.dense( - k, - self._head_num * self._head_size, - use_bias=False, - kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'key')) + k, + self._head_num * self._head_size, + use_bias=False, + kernel_regularizer=self._l2_reg, + name='%s/%s/dnn' % (self._name, 'key') + ) v = tf.layers.dense( - v, - self._head_num * self._head_size, - use_bias=False, - kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'value')) + v, + self._head_num * self._head_size, + use_bias=False, + kernel_regularizer=self._l2_reg, + name='%s/%s/dnn' % (self._name, 'value') + ) return q, k, v def _combine_heads(self, multi_head_tensor): @@ -147,11 +152,12 @@ def _multi_head_attention(self, attention_input): if self._use_res: W_0_x = tf.layers.dense( - ori_v, - out.shape[2], - use_bias=False, - kernel_regularizer=self._l2_reg, - name='%s/dnn' % (self._name)) + ori_v, + out.shape[2], + use_bias=False, + kernel_regularizer=self._l2_reg, + name='%s/dnn' % (self._name) + ) res_out = tf.nn.relu(out + W_0_x) return res_out else: diff --git a/easy_rec/python/layers/multihead_cross_attention.py b/easy_rec/python/layers/multihead_cross_attention.py index f230ac974..6b62ce449 100644 --- a/easy_rec/python/layers/multihead_cross_attention.py +++ b/easy_rec/python/layers/multihead_cross_attention.py @@ -1,11 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import math - import tensorflow as tf from easy_rec.python.compat.layers import layer_norm as tf_layer_norm @@ -39,21 +36,23 @@ def dropout(input_tensor, dropout_prob): return output -def attention_layer(from_tensor, - to_tensor, - size_per_head, - num_attention_heads=1, - attention_mask=None, - query_act=None, - key_act=None, - value_act=None, - attention_probs_dropout_prob=0.0, - initializer_range=0.02, - do_return_2d_tensor=False, - batch_size=None, - from_seq_length=None, - to_seq_length=None, - reuse=None): +def attention_layer( + from_tensor, + to_tensor, + size_per_head, + num_attention_heads=1, + attention_mask=None, + query_act=None, + key_act=None, + value_act=None, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + do_return_2d_tensor=False, + batch_size=None, + from_seq_length=None, + to_seq_length=None, + reuse=None +): """Performs multi-headed attention from `from_tensor` to `to_tensor`. This is an implementation of multi-headed attention based on "Attention is all you Need". @@ -108,10 +107,12 @@ def attention_layer(from_tensor, ValueError: Any of the arguments or tensor shapes are invalid. """ - def transpose_for_scores(input_tensor, batch_size, num_attention_heads, - seq_length, width): + def transpose_for_scores( + input_tensor, batch_size, num_attention_heads, seq_length, width + ): output_tensor = tf.reshape( - input_tensor, [batch_size, seq_length, num_attention_heads, width]) + input_tensor, [batch_size, seq_length, num_attention_heads, width] + ) output_tensor = tf.transpose(output_tensor, [0, 2, 1, 3]) return output_tensor @@ -121,18 +122,22 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, if len(from_shape) != len(to_shape): raise ValueError( - 'The rank of `from_tensor` must match the rank of `to_tensor`.') + 'The rank of `from_tensor` must match the rank of `to_tensor`.' + ) if len(from_shape) == 3: batch_size = from_shape[0] from_seq_length = from_shape[1] to_seq_length = to_shape[1] elif len(from_shape) == 2: - if (batch_size is None or from_seq_length is None or to_seq_length is None): + if ( + batch_size is None or from_seq_length is None or to_seq_length is None + ): raise ValueError( - 'When passing in rank 2 tensors to attention_layer, the values ' - 'for `batch_size`, `from_seq_length`, and `to_seq_length` ' - 'must all be specified.') + 'When passing in rank 2 tensors to attention_layer, the values ' + 'for `batch_size`, `from_seq_length`, and `to_seq_length` ' + 'must all be specified.' + ) # Scalar dimensions referenced here: # B = batch size (number of sequences) @@ -146,46 +151,52 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, # `query_layer` = [B*F, N*H] query_layer = tf.layers.dense( - from_tensor_2d, - num_attention_heads * size_per_head, - activation=query_act, - name='query', - kernel_initializer=create_initializer(initializer_range), - reuse=reuse) + from_tensor_2d, + num_attention_heads * size_per_head, + activation=query_act, + name='query', + kernel_initializer=create_initializer(initializer_range), + reuse=reuse + ) # `key_layer` = [B*T, N*H] key_layer = tf.layers.dense( - to_tensor_2d, - num_attention_heads * size_per_head, - activation=key_act, - name='key', - kernel_initializer=create_initializer(initializer_range), - reuse=reuse) + to_tensor_2d, + num_attention_heads * size_per_head, + activation=key_act, + name='key', + kernel_initializer=create_initializer(initializer_range), + reuse=reuse + ) # `value_layer` = [B*T, N*H] value_layer = tf.layers.dense( - to_tensor_2d, - num_attention_heads * size_per_head, - activation=value_act, - name='value', - kernel_initializer=create_initializer(initializer_range), - reuse=reuse) + to_tensor_2d, + num_attention_heads * size_per_head, + activation=value_act, + name='value', + kernel_initializer=create_initializer(initializer_range), + reuse=reuse + ) # `query_layer` = [B, N, F, H] - query_layer = transpose_for_scores(query_layer, batch_size, - num_attention_heads, from_seq_length, - size_per_head) + query_layer = transpose_for_scores( + query_layer, batch_size, num_attention_heads, from_seq_length, + size_per_head + ) # `key_layer` = [B, N, T, H] - key_layer = transpose_for_scores(key_layer, batch_size, num_attention_heads, - to_seq_length, size_per_head) + key_layer = transpose_for_scores( + key_layer, batch_size, num_attention_heads, to_seq_length, size_per_head + ) # Take the dot product between "query" and "key" to get the raw # attention scores. # `attention_scores` = [B, N, F, T] attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) - attention_scores = tf.multiply(attention_scores, - 1.0 / math.sqrt(float(size_per_head))) + attention_scores = tf.multiply( + attention_scores, 1.0 / math.sqrt(float(size_per_head)) + ) if attention_mask is not None: # `attention_mask` = [B, 1, F, T] @@ -210,8 +221,9 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, # `value_layer` = [B, T, N, H] value_layer = tf.reshape( - value_layer, - [batch_size, to_seq_length, num_attention_heads, size_per_head]) + value_layer, + [batch_size, to_seq_length, num_attention_heads, size_per_head] + ) # `value_layer` = [B, N, T, H] value_layer = tf.transpose(value_layer, [0, 2, 1, 3]) @@ -225,29 +237,33 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, if do_return_2d_tensor: # `context_layer` = [B*F, N*H] context_layer = tf.reshape( - context_layer, - [batch_size * from_seq_length, num_attention_heads * size_per_head]) + context_layer, + [batch_size * from_seq_length, num_attention_heads * size_per_head] + ) else: # `context_layer` = [B, F, N*H] context_layer = tf.reshape( - context_layer, - [batch_size, from_seq_length, num_attention_heads * size_per_head]) + context_layer, + [batch_size, from_seq_length, num_attention_heads * size_per_head] + ) return context_layer -def transformer_encoder(input_tensor, - attention_mask=None, - hidden_size=768, - num_hidden_layers=12, - num_attention_heads=12, - intermediate_size=3072, - intermediate_act_fn=gelu, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - initializer_range=0.02, - reuse=None, - name='transformer'): +def transformer_encoder( + input_tensor, + attention_mask=None, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + intermediate_act_fn=gelu, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + reuse=None, + name='transformer' +): """Multi-headed, multi-layer Transformer from "Attention is All You Need". This is almost an exact implementation of the original Transformer encoder. @@ -282,8 +298,9 @@ def transformer_encoder(input_tensor, """ if hidden_size % num_attention_heads != 0: raise ValueError( - 'The hidden size (%d) is not a multiple of the number of attention ' - 'heads (%d)' % (hidden_size, num_attention_heads)) + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % (hidden_size, num_attention_heads) + ) attention_head_size = int(hidden_size / num_attention_heads) input_shape = get_shape_list(input_tensor, expected_rank=3) @@ -294,8 +311,10 @@ def transformer_encoder(input_tensor, # The Transformer performs sum residuals on all layers so the input needs # to be the same as the hidden size. if input_width != hidden_size: - raise ValueError('The width of the input tensor (%d) != hidden size (%d)' % - (input_width, hidden_size)) + raise ValueError( + 'The width of the input tensor (%d) != hidden size (%d)' % + (input_width, hidden_size) + ) # We keep the representation as a 2D tensor to avoid re-shaping it back and # forth from a 3D tensor to a 2D tensor. Re-shapes are normally free on @@ -311,43 +330,47 @@ def transformer_encoder(input_tensor, with tf.variable_scope('self'): # [batch_size * from_seq_length, num_attention_heads * size_per_head] attention_output = attention_layer( - from_tensor=layer_input, - to_tensor=layer_input, - size_per_head=attention_head_size, - num_attention_heads=num_attention_heads, - attention_mask=attention_mask, - attention_probs_dropout_prob=attention_probs_dropout_prob, - initializer_range=initializer_range, - do_return_2d_tensor=True, - batch_size=batch_size, - from_seq_length=seq_length, - to_seq_length=seq_length, - reuse=reuse) + from_tensor=layer_input, + to_tensor=layer_input, + size_per_head=attention_head_size, + num_attention_heads=num_attention_heads, + attention_mask=attention_mask, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + batch_size=batch_size, + from_seq_length=seq_length, + to_seq_length=seq_length, + reuse=reuse + ) # Run a linear projection of `hidden_size` then add a residual # with `layer_input`. with tf.variable_scope('output', reuse=reuse): attention_output = tf.layers.dense( - attention_output, - hidden_size, - kernel_initializer=create_initializer(initializer_range)) + attention_output, + hidden_size, + kernel_initializer=create_initializer(initializer_range) + ) attention_output = dropout(attention_output, hidden_dropout_prob) attention_output = layer_norm(attention_output + layer_input) # The activation is only applied to the "intermediate" hidden layer. with tf.variable_scope('intermediate', reuse=reuse): intermediate_output = tf.layers.dense( - attention_output, - intermediate_size, - activation=intermediate_act_fn, - kernel_initializer=create_initializer(initializer_range)) + attention_output, + intermediate_size, + activation=intermediate_act_fn, + kernel_initializer=create_initializer(initializer_range) + ) # Down-project back to `hidden_size` then add the residual. with tf.variable_scope('output', reuse=reuse): layer_output = tf.layers.dense( - intermediate_output, - hidden_size, - kernel_initializer=create_initializer(initializer_range)) + intermediate_output, + hidden_size, + kernel_initializer=create_initializer(initializer_range) + ) layer_output = dropout(layer_output, hidden_dropout_prob) layer_output = layer_norm(layer_output + attention_output) prev_output = layer_output @@ -356,18 +379,20 @@ def transformer_encoder(input_tensor, return final_output -def cross_attention_block(from_tensor, - to_tensor, - layer_idx, - size_per_head, - cross_attention_mask=None, - self_attention_mask=None, - num_attention_heads=1, - intermediate_size=512, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - initializer_range=0.02, - name=''): +def cross_attention_block( + from_tensor, + to_tensor, + layer_idx, + size_per_head, + cross_attention_mask=None, + self_attention_mask=None, + num_attention_heads=1, + intermediate_size=512, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + name='' +): """Multi-headed cross attention block. Args: @@ -411,75 +436,84 @@ def cross_attention_block(from_tensor, with tf.variable_scope('cross'): # [batch_size * from_seq_length, num_attention_heads * size_per_head] cross_attention_output = attention_layer( - from_tensor=from_tensor, - to_tensor=to_tensor, - size_per_head=size_per_head, - num_attention_heads=num_attention_heads, - attention_mask=cross_attention_mask, - attention_probs_dropout_prob=attention_probs_dropout_prob, - initializer_range=initializer_range, - do_return_2d_tensor=True, - batch_size=batch_size, - from_seq_length=from_seq_length, - to_seq_length=to_seq_length) + from_tensor=from_tensor, + to_tensor=to_tensor, + size_per_head=size_per_head, + num_attention_heads=num_attention_heads, + attention_mask=cross_attention_mask, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + batch_size=batch_size, + from_seq_length=from_seq_length, + to_seq_length=to_seq_length + ) with tf.variable_scope('self'): # [batch_size * from_seq_length, num_attention_heads * size_per_head] self_attention_output = attention_layer( - from_tensor=cross_attention_output, - to_tensor=cross_attention_output, - size_per_head=size_per_head, - num_attention_heads=num_attention_heads, - attention_mask=self_attention_mask, - attention_probs_dropout_prob=attention_probs_dropout_prob, - initializer_range=initializer_range, - do_return_2d_tensor=True, - batch_size=batch_size, - from_seq_length=from_seq_length, - to_seq_length=from_seq_length) + from_tensor=cross_attention_output, + to_tensor=cross_attention_output, + size_per_head=size_per_head, + num_attention_heads=num_attention_heads, + attention_mask=self_attention_mask, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + batch_size=batch_size, + from_seq_length=from_seq_length, + to_seq_length=from_seq_length + ) with tf.variable_scope('output'): attention_output = dropout(self_attention_output, hidden_dropout_prob) - attention_output = layer_norm(attention_output + cross_attention_output) + attention_output = layer_norm( + attention_output + cross_attention_output + ) # The activation is only applied to the "intermediate" hidden layer. with tf.variable_scope('intermediate'): intermediate_output = tf.layers.dense( - attention_output, - intermediate_size, - activation=tf.nn.relu, - kernel_initializer=create_initializer(initializer_range)) + attention_output, + intermediate_size, + activation=tf.nn.relu, + kernel_initializer=create_initializer(initializer_range) + ) # Down-project back to `hidden_size` then add the residual. with tf.variable_scope('output'): layer_output = tf.layers.dense( - intermediate_output, - num_attention_heads * size_per_head, - kernel_initializer=create_initializer(initializer_range)) + intermediate_output, + num_attention_heads * size_per_head, + kernel_initializer=create_initializer(initializer_range) + ) layer_output = dropout(layer_output, hidden_dropout_prob) # [batch_size * from_seq_length, num_attention_heads * size_per_head] layer_output = layer_norm(layer_output + attention_output) final_output = reshape_from_matrix( - layer_output, - [batch_size, from_seq_length, num_attention_heads * size_per_head]) + layer_output, + [batch_size, from_seq_length, num_attention_heads * size_per_head] + ) return final_output # [batch_size, from_seq_length, num_attention_heads * size_per_head] -def cross_attention_tower(left_tensor, - right_tensor, - num_hidden_layers=1, - num_attention_heads=12, - left_size_per_head=64, - right_size_per_head=64, - left_intermediate_size=0, - right_intermediate_size=0, - left_input_mask=None, - right_input_mask=None, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - initializer_range=0.02, - name=''): +def cross_attention_tower( + left_tensor, + right_tensor, + num_hidden_layers=1, + num_attention_heads=12, + left_size_per_head=64, + right_size_per_head=64, + left_intermediate_size=0, + right_intermediate_size=0, + left_input_mask=None, + right_input_mask=None, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + name='' +): """Multi-headed, multi layer cross attention block. Args: @@ -521,52 +555,58 @@ def cross_attention_tower(left_tensor, left_attention_mask = None if left_input_mask is not None: left_attention_mask = create_attention_mask_from_input_mask( - left_tensor, left_attention_mask) + left_tensor, left_attention_mask + ) left_2_right_attention_mask = None if right_input_mask is not None: left_2_right_attention_mask = create_attention_mask_from_input_mask( - left_tensor, right_input_mask) + left_tensor, right_input_mask + ) right_attention_mask = None if right_input_mask is not None: right_attention_mask = create_attention_mask_from_input_mask( - right_tensor, right_input_mask) + right_tensor, right_input_mask + ) right_2_left_attention_mask = None if left_input_mask is not None: right_2_left_attention_mask = create_attention_mask_from_input_mask( - right_tensor, left_input_mask) + right_tensor, left_input_mask + ) prev_left_output = left_tensor prev_right_output = right_tensor for layer_idx in range(num_hidden_layers): left_output = cross_attention_block( - prev_left_output, - prev_right_output, - layer_idx, - num_attention_heads=num_attention_heads, - size_per_head=left_size_per_head, - intermediate_size=left_intermediate_size, - hidden_dropout_prob=hidden_dropout_prob, - cross_attention_mask=left_2_right_attention_mask, - self_attention_mask=left_attention_mask, - attention_probs_dropout_prob=attention_probs_dropout_prob, - initializer_range=initializer_range, - name='%sleft_to_right_' % name) + prev_left_output, + prev_right_output, + layer_idx, + num_attention_heads=num_attention_heads, + size_per_head=left_size_per_head, + intermediate_size=left_intermediate_size, + hidden_dropout_prob=hidden_dropout_prob, + cross_attention_mask=left_2_right_attention_mask, + self_attention_mask=left_attention_mask, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + name='%sleft_to_right_' % name + ) right_output = cross_attention_block( - prev_right_output, - prev_left_output, - layer_idx, - num_attention_heads=num_attention_heads, - size_per_head=right_size_per_head, - intermediate_size=right_intermediate_size, - hidden_dropout_prob=hidden_dropout_prob, - cross_attention_mask=right_2_left_attention_mask, - self_attention_mask=right_attention_mask, - attention_probs_dropout_prob=attention_probs_dropout_prob, - initializer_range=initializer_range, - name='%sright_to_left_' % name) + prev_right_output, + prev_left_output, + layer_idx, + num_attention_heads=num_attention_heads, + size_per_head=right_size_per_head, + intermediate_size=right_intermediate_size, + hidden_dropout_prob=hidden_dropout_prob, + cross_attention_mask=right_2_left_attention_mask, + self_attention_mask=right_attention_mask, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + name='%sright_to_left_' % name + ) prev_left_output = left_output prev_right_output = right_output return prev_left_output, prev_right_output @@ -575,15 +615,18 @@ def cross_attention_tower(left_tensor, def layer_norm(input_tensor, name=None): """Run layer normalization on the last dimension of the tensor.""" return tf_layer_norm( - inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name) + inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name + ) def reshape_to_matrix(input_tensor): """Reshapes a >= rank 2 tensor to a rank 2 tensor (i.e., a matrix).""" ndims = input_tensor.shape.ndims if ndims < 2: - raise ValueError('Input tensor must have at least rank 2. Shape = %s' % - (input_tensor.shape)) + raise ValueError( + 'Input tensor must have at least rank 2. Shape = %s' % + (input_tensor.shape) + ) if ndims == 2: return input_tensor @@ -623,7 +666,8 @@ def create_attention_mask_from_input_mask(from_tensor, to_mask): to_seq_length = to_shape[1] to_mask = tf.cast( - tf.reshape(to_mask, [batch_size, 1, to_seq_length]), tf.float32) + tf.reshape(to_mask, [batch_size, 1, to_seq_length]), tf.float32 + ) # We don't assume that `from_tensor` is a mask (although it could be). We # don't actually care if we attend *from* padding tokens (only *to* padding) @@ -631,7 +675,8 @@ def create_attention_mask_from_input_mask(from_tensor, to_mask): # # `broadcast_ones` = [batch_size, from_seq_length, 1] broadcast_ones = tf.ones( - shape=tf.stack([batch_size, from_seq_length, 1]), dtype=tf.float32) + shape=tf.stack([batch_size, from_seq_length, 1]), dtype=tf.float32 + ) # Here we broadcast along two dimensions to create the mask. mask = broadcast_ones * to_mask @@ -639,18 +684,20 @@ def create_attention_mask_from_input_mask(from_tensor, to_mask): return mask -def embedding_postprocessor(input_tensor, - use_token_type=False, - token_type_ids=None, - token_type_vocab_size=16, - token_type_embedding_name='token_type_embeddings', - reuse_token_type=None, - use_position_embeddings=True, - position_embedding_name='position_embeddings', - reuse_position_embedding=None, - initializer_range=0.02, - max_position_embeddings=512, - dropout_prob=0.1): +def embedding_postprocessor( + input_tensor, + use_token_type=False, + token_type_ids=None, + token_type_vocab_size=16, + token_type_embedding_name='token_type_embeddings', + reuse_token_type=None, + use_position_embeddings=True, + position_embedding_name='position_embeddings', + reuse_position_embedding=None, + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.1 +): """Performs various post-processing on a word embedding tensor. Args: @@ -689,31 +736,37 @@ def embedding_postprocessor(input_tensor, if use_token_type: if token_type_ids is None: - raise ValueError('`token_type_ids` must be specified if' - '`use_token_type` is True.') + raise ValueError( + '`token_type_ids` must be specified if' + '`use_token_type` is True.' + ) with tf.variable_scope('token_type', reuse=reuse_token_type): token_type_table = tf.get_variable( - name=token_type_embedding_name, - shape=[token_type_vocab_size, width], - initializer=create_initializer(initializer_range)) + name=token_type_embedding_name, + shape=[token_type_vocab_size, width], + initializer=create_initializer(initializer_range) + ) # This vocab will be small so we always do one-hot here, since it is always # faster for a small vocabulary. flat_token_type_ids = tf.reshape(token_type_ids, [-1]) one_hot_ids = tf.one_hot(flat_token_type_ids, depth=token_type_vocab_size) token_type_embeddings = tf.matmul(one_hot_ids, token_type_table) - token_type_embeddings = tf.reshape(token_type_embeddings, - [batch_size, seq_length, width]) + token_type_embeddings = tf.reshape( + token_type_embeddings, [batch_size, seq_length, width] + ) output += token_type_embeddings if use_position_embeddings: assert_op = tf.assert_less_equal(seq_length, max_position_embeddings) with tf.control_dependencies([assert_op]): with tf.variable_scope( - 'position_embedding', reuse=reuse_position_embedding): + 'position_embedding', reuse=reuse_position_embedding + ): full_position_embeddings = tf.get_variable( - name=position_embedding_name, - shape=[max_position_embeddings, width], - initializer=create_initializer(initializer_range)) + name=position_embedding_name, + shape=[max_position_embeddings, width], + initializer=create_initializer(initializer_range) + ) # Since the position embedding table is a learned variable, we create it # using a (long) sequence length `max_position_embeddings`. The actual # sequence length might be shorter than this, for faster training of @@ -723,8 +776,9 @@ def embedding_postprocessor(input_tensor, # for position [0, 1, 2, ..., max_position_embeddings-1], and the current # sequence has positions [0, 1, 2, ... seq_length-1], so we can just # perform a slice. - position_embeddings = tf.slice(full_position_embeddings, [0, 0], - [seq_length, -1]) + position_embeddings = tf.slice( + full_position_embeddings, [0, 0], [seq_length, -1] + ) num_dims = len(output.shape.as_list()) # Only the last two dimensions are relevant (`seq_length` and `width`), so @@ -734,8 +788,9 @@ def embedding_postprocessor(input_tensor, for _ in range(num_dims - 2): position_broadcast_shape.append(1) position_broadcast_shape.extend([seq_length, width]) - position_embeddings = tf.reshape(position_embeddings, - position_broadcast_shape) + position_embeddings = tf.reshape( + position_embeddings, position_broadcast_shape + ) output += position_embeddings output = layer_norm_and_dropout(output, dropout_prob) diff --git a/easy_rec/python/layers/senet.py b/easy_rec/python/layers/senet.py index 777079341..8add70e0b 100644 --- a/easy_rec/python/layers/senet.py +++ b/easy_rec/python/layers/senet.py @@ -21,12 +21,14 @@ class SENet: name: str, name of the layer. """ - def __init__(self, - num_fields, - num_squeeze_group, - reduction_ratio, - l2_reg, - name='SENet'): + def __init__( + self, + num_fields, + num_squeeze_group, + reduction_ratio, + l2_reg, + name='SENet' + ): self.num_fields = num_fields self.num_squeeze_group = num_squeeze_group self.reduction_ratio = reduction_ratio @@ -44,7 +46,7 @@ def __call__(self, inputs): emb_size += int(input.shape[-1]) group_embs = [ - tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs + tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs ] squeezed = [] @@ -54,17 +56,19 @@ def __call__(self, inputs): z = tf.concat(squeezed, axis=1) # [bs, field_size * num_groups * 2] reduced = tf.layers.dense( - inputs=z, - units=reduction_size, - kernel_regularizer=self._l2_reg, - activation='relu', - name='%s/reduce' % self._name) + inputs=z, + units=reduction_size, + kernel_regularizer=self._l2_reg, + activation='relu', + name='%s/reduce' % self._name + ) excited_weights = tf.layers.dense( - inputs=reduced, - units=emb_size, - kernel_initializer='glorot_normal', - name='%s/excite' % self._name) + inputs=reduced, + units=emb_size, + kernel_initializer='glorot_normal', + name='%s/excite' % self._name + ) # Re-weight inputs = tf.concat(inputs, axis=-1) diff --git a/easy_rec/python/layers/seq_input_layer.py b/easy_rec/python/layers/seq_input_layer.py index a52904dd1..140278876 100644 --- a/easy_rec/python/layers/seq_input_layer.py +++ b/easy_rec/python/layers/seq_input_layer.py @@ -2,7 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from tensorflow.python.framework import ops from tensorflow.python.ops import variable_scope @@ -18,25 +17,31 @@ class SeqInputLayer(object): - def __init__(self, - feature_configs, - feature_groups_config, - embedding_regularizer=None, - ev_params=None): + def __init__( + self, + feature_configs, + feature_groups_config, + embedding_regularizer=None, + ev_params=None + ): self._feature_groups_config = { - x.group_name: x for x in feature_groups_config + x.group_name: x + for x in feature_groups_config } wide_and_deep_dict = self.get_wide_deep_dict() self._fc_parser = FeatureColumnParser( - feature_configs, wide_and_deep_dict, ev_params=ev_params) + feature_configs, wide_and_deep_dict, ev_params=ev_params + ) self._embedding_regularizer = embedding_regularizer - def __call__(self, - features, - group_name, - feature_name_to_output_tensors={}, - allow_key_search=True, - scope_name=None): + def __call__( + self, + features, + group_name, + feature_name_to_output_tensors={}, + allow_key_search=True, + scope_name=None + ): feature_column_dict = self._fc_parser.deep_columns feature_column_dict.update(self._fc_parser.sequence_columns) @@ -56,39 +61,45 @@ def _seq_embed_summary_name(input_name): scope_name = group_name # name_scope is specified to avoid adding _1 _2 after scope_name with variable_scope.variable_scope( - scope_name, - reuse=variable_scope.AUTO_REUSE), ops.name_scope(scope_name + '/'): + scope_name, reuse=variable_scope.AUTO_REUSE + ), ops.name_scope(scope_name + '/'): key_tensors = [] hist_tensors = [] check_op_list = [] for x in feature_dict.seq_att_map: for key in x.key: if key not in feature_name_to_output_tensors or ( - feature_name_to_output_tensors[key] is None and allow_key_search): + feature_name_to_output_tensors[key] is None and allow_key_search + ): qfc = feature_column_dict[key] with variable_scope.variable_scope(qfc._var_scope_name): tmp_key_tensor = feature_column_dict[key]._get_dense_tensor( - builder) + builder + ) regularizers.apply_regularization( - self._embedding_regularizer, weights_list=[tmp_key_tensor]) + self._embedding_regularizer, weights_list=[tmp_key_tensor] + ) key_tensors.append(tmp_key_tensor) elif feature_name_to_output_tensors[key] is None: assert feature_name_to_output_tensors[ - key] is not None, 'When allow_key_search is False, key: %s should defined in same feature group.' % key + key + ] is not None, 'When allow_key_search is False, key: %s should defined in same feature group.' % key else: key_tensors.append(feature_name_to_output_tensors[key]) if tf_summary: for key_tensor in key_tensors: tf.summary.histogram( - _seq_embed_summary_name(key_tensor.name), key_tensor) + _seq_embed_summary_name(key_tensor.name), key_tensor + ) cur_hist_seqs = [] for hist_seq in x.hist_seq: seq_fc = feature_column_dict[hist_seq] with variable_scope.variable_scope(seq_fc._var_scope_name): cur_hist_seqs.append( - feature_column_dict[hist_seq]._get_sequence_dense_tensor( - builder)) + feature_column_dict[hist_seq]. + _get_sequence_dense_tensor(builder) + ) hist_tensors.extend(cur_hist_seqs) aux_hist_emb_list = [] @@ -96,30 +107,34 @@ def _seq_embed_summary_name(input_name): seq_fc = feature_column_dict[aux_hist_seq] with variable_scope.variable_scope(seq_fc._var_scope_name): aux_hist_embedding, _ = feature_column_dict[ - aux_hist_seq]._get_sequence_dense_tensor(builder) + aux_hist_seq]._get_sequence_dense_tensor(builder) aux_hist_emb_list.append(aux_hist_embedding) if tf_summary: for hist_embed, hist_seq_len in hist_tensors: tf.summary.histogram( - _seq_embed_summary_name(hist_embed.name), hist_embed) + _seq_embed_summary_name(hist_embed.name), hist_embed + ) tf.summary.histogram( - _seq_embed_summary_name(hist_seq_len.name), hist_seq_len) + _seq_embed_summary_name(hist_seq_len.name), hist_seq_len + ) for idx in range(1, len(cur_hist_seqs)): check_op = tf.assert_equal( - cur_hist_seqs[0][1], - cur_hist_seqs[idx][1], - message='SequenceFeature Error: The size of %s not equal to the size of %s.' - % (x.hist_seq[idx], x.hist_seq[0])) + cur_hist_seqs[0][1], + cur_hist_seqs[idx][1], + message= + 'SequenceFeature Error: The size of %s not equal to the size of %s.' + % (x.hist_seq[idx], x.hist_seq[0]) + ) check_op_list.append(check_op) with tf.control_dependencies(check_op_list): features = { - 'key': tf.concat(key_tensors, axis=-1), - 'hist_seq_emb': tf.concat([x[0] for x in hist_tensors], axis=-1), - 'hist_seq_len': hist_tensors[0][1], - 'aux_hist_seq_emb_list': aux_hist_emb_list + 'key': tf.concat(key_tensors, axis=-1), + 'hist_seq_emb': tf.concat([x[0] for x in hist_tensors], axis=-1), + 'hist_seq_len': hist_tensors[0][1], + 'aux_hist_seq_emb_list': aux_hist_emb_list } return features diff --git a/easy_rec/python/layers/sequence_feature_layer.py b/easy_rec/python/layers/sequence_feature_layer.py index fd01b5b2c..48e6a94b1 100644 --- a/easy_rec/python/layers/sequence_feature_layer.py +++ b/easy_rec/python/layers/sequence_feature_layer.py @@ -1,12 +1,10 @@ import logging import os - import tensorflow as tf from tensorflow.python.framework import ops from easy_rec.python.compat import regularizers -from easy_rec.python.layers import dnn -from easy_rec.python.layers import seq_input_layer +from easy_rec.python.layers import dnn, seq_input_layer from easy_rec.python.utils import conditional if tf.__version__ >= '2.0': @@ -15,14 +13,16 @@ class SequenceFeatureLayer(object): - def __init__(self, - feature_configs, - feature_groups_config, - ev_params=None, - embedding_regularizer=None, - kernel_regularizer=None, - is_training=False, - is_predicting=False): + def __init__( + self, + feature_configs, + feature_groups_config, + ev_params=None, + embedding_regularizer=None, + kernel_regularizer=None, + is_training=False, + is_predicting=False + ): self._seq_feature_groups_config = [] for x in feature_groups_config: for y in x.sequence_features: @@ -30,25 +30,28 @@ def __init__(self, self._seq_input_layer = None if len(self._seq_feature_groups_config) > 0: self._seq_input_layer = seq_input_layer.SeqInputLayer( - feature_configs, - self._seq_feature_groups_config, - embedding_regularizer=embedding_regularizer, - ev_params=ev_params) + feature_configs, + self._seq_feature_groups_config, + embedding_regularizer=embedding_regularizer, + ev_params=ev_params + ) self._embedding_regularizer = embedding_regularizer self._kernel_regularizer = kernel_regularizer self._is_training = is_training self._is_predicting = is_predicting - def negative_sampler_target_attention(self, - dnn_config, - deep_fea, - concat_features, - name, - need_key_feature=True, - allow_key_transform=False): - cur_id, hist_id_col, seq_len, aux_hist_emb_list = deep_fea['key'], deep_fea[ - 'hist_seq_emb'], deep_fea['hist_seq_len'], deep_fea[ - 'aux_hist_seq_emb_list'] + def negative_sampler_target_attention( + self, + dnn_config, + deep_fea, + concat_features, + name, + need_key_feature=True, + allow_key_transform=False + ): + cur_id, hist_id_col, seq_len, aux_hist_emb_list = deep_fea[ + 'key'], deep_fea['hist_seq_emb'], deep_fea['hist_seq_len'], deep_fea[ + 'aux_hist_seq_emb_list'] seq_max_len = tf.shape(hist_id_col)[1] seq_emb_dim = hist_id_col.shape[2] @@ -57,57 +60,67 @@ def negative_sampler_target_attention(self, pos_feature = cur_id[:batch_size] neg_feature = cur_id[batch_size:] - cur_id = tf.concat([ + cur_id = tf.concat( + [ pos_feature[:, tf.newaxis, :], tf.tile(neg_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1]) - ], - axis=1) # noqa: E126 + ], + axis=1 + ) # noqa: E126 neg_num_add_1 = tf.shape(cur_id)[1] hist_id_col_tmp = tf.tile( - hist_id_col[:, :, :], multiples=[1, neg_num_add_1, 1]) + hist_id_col[:, :, :], multiples=[1, neg_num_add_1, 1] + ) hist_id_col = tf.reshape( - hist_id_col_tmp, [batch_size * neg_num_add_1, seq_max_len, seq_emb_dim]) + hist_id_col_tmp, [batch_size * neg_num_add_1, seq_max_len, seq_emb_dim] + ) concat_features = tf.tile( - concat_features[:, tf.newaxis, :], multiples=[1, neg_num_add_1, 1]) + concat_features[:, tf.newaxis, :], multiples=[1, neg_num_add_1, 1] + ) seq_len = tf.tile(seq_len, multiples=[neg_num_add_1]) if allow_key_transform and (cur_id_dim != seq_emb_dim): cur_id = tf.layers.dense( - cur_id, seq_emb_dim, name='sequence_key_transform_layer') + cur_id, seq_emb_dim, name='sequence_key_transform_layer' + ) cur_ids = tf.tile(cur_id, [1, 1, seq_max_len]) cur_ids = tf.reshape( - cur_ids, - tf.shape(hist_id_col)) # (B * neg_num_add_1, seq_max_len, seq_emb_dim) + cur_ids, tf.shape(hist_id_col) + ) # (B * neg_num_add_1, seq_max_len, seq_emb_dim) din_net = tf.concat( - [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1) # (B * neg_num_add_1, seq_max_len, seq_emb_dim*4) + [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], + axis=-1 + ) # (B * neg_num_add_1, seq_max_len, seq_emb_dim*4) din_layer = dnn.DNN( - dnn_config, - self._kernel_regularizer, - name, - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True) + dnn_config, + self._kernel_regularizer, + name, + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True + ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) seq_len = tf.expand_dims(seq_len, 1) mask = tf.sequence_mask(seq_len) padding = tf.ones_like(scores) * (-2**32 + 1) - scores = tf.where(mask, scores, - padding) # [B*neg_num_add_1, 1, seq_max_len] + scores = tf.where( + mask, scores, padding + ) # [B*neg_num_add_1, 1, seq_max_len] # Scale scores = tf.nn.softmax(scores) # (B * neg_num_add_1, 1, seq_max_len) - hist_din_emb = tf.matmul(scores, - hist_id_col) # [B * neg_num_add_1, 1, seq_emb_dim] - hist_din_emb = tf.reshape(hist_din_emb, - [batch_size, neg_num_add_1, seq_emb_dim - ]) # [B * neg_num_add_1, seq_emb_dim] + hist_din_emb = tf.matmul( + scores, hist_id_col + ) # [B * neg_num_add_1, 1, seq_emb_dim] + hist_din_emb = tf.reshape( + hist_din_emb, [batch_size, neg_num_add_1, seq_emb_dim] + ) # [B * neg_num_add_1, seq_emb_dim] if len(aux_hist_emb_list) > 0: all_hist_dim_emb = [hist_din_emb] for hist_col in aux_hist_emb_list: @@ -120,16 +133,18 @@ def negative_sampler_target_attention(self, din_output = tf.concat([hist_din_emb, cur_id], axis=2) return din_output, concat_features - def target_attention(self, - dnn_config, - deep_fea, - name, - need_key_feature=True, - allow_key_transform=False, - transform_dnn=False): - cur_id, hist_id_col, seq_len, aux_hist_emb_list = deep_fea['key'], deep_fea[ - 'hist_seq_emb'], deep_fea['hist_seq_len'], deep_fea[ - 'aux_hist_seq_emb_list'] + def target_attention( + self, + dnn_config, + deep_fea, + name, + need_key_feature=True, + allow_key_transform=False, + transform_dnn=False + ): + cur_id, hist_id_col, seq_len, aux_hist_emb_list = deep_fea[ + 'key'], deep_fea['hist_seq_emb'], deep_fea['hist_seq_len'], deep_fea[ + 'aux_hist_seq_emb_list'] seq_max_len = tf.shape(hist_id_col)[1] seq_emb_dim = hist_id_col.shape[2] @@ -143,25 +158,29 @@ def target_attention(self, cur_id = tf.layers.dense(cur_id, seq_emb_dim, name=cur_key_layer_name) cur_fea_layer_name = 'sequence_fea_transform_layer_' + name hist_id_col = tf.layers.dense( - hist_id_col, seq_emb_dim, name=cur_fea_layer_name) + hist_id_col, seq_emb_dim, name=cur_fea_layer_name + ) else: cur_id = cur_id[:tf.shape(hist_id_col)[0], ...] # for negative sampler cur_ids = tf.tile(cur_id, [1, seq_max_len]) - cur_ids = tf.reshape(cur_ids, - tf.shape(hist_id_col)) # (B, seq_max_len, seq_emb_dim) + cur_ids = tf.reshape( + cur_ids, tf.shape(hist_id_col) + ) # (B, seq_max_len, seq_emb_dim) din_net = tf.concat( - [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1) # (B, seq_max_len, seq_emb_dim*4) + [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], + axis=-1 + ) # (B, seq_max_len, seq_emb_dim*4) din_layer = dnn.DNN( - dnn_config, - self._kernel_regularizer, - name, - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True) + dnn_config, + self._kernel_regularizer, + name, + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True + ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) @@ -173,8 +192,9 @@ def target_attention(self, # Scale scores = tf.nn.softmax(scores) # (B, 1, seq_max_len) hist_din_emb = tf.matmul(scores, hist_id_col) # [B, 1, seq_emb_dim] - hist_din_emb = tf.reshape(hist_din_emb, - [-1, seq_emb_dim]) # [B, seq_emb_dim] + hist_din_emb = tf.reshape( + hist_din_emb, [-1, seq_emb_dim] + ) # [B, seq_emb_dim] if len(aux_hist_emb_list) > 0: all_hist_dim_emb = [hist_din_emb] for hist_col in aux_hist_emb_list: @@ -188,13 +208,15 @@ def target_attention(self, din_output = tf.concat([hist_din_emb, cur_id], axis=1) return din_output - def __call__(self, - features, - concat_features, - all_seq_att_map_config, - feature_name_to_output_tensors=None, - negative_sampler=False, - scope_name=None): + def __call__( + self, + features, + concat_features, + all_seq_att_map_config, + feature_name_to_output_tensors=None, + negative_sampler=False, + scope_name=None + ): logging.info('use sequence feature layer.') all_seq_fea = [] # process all sequence features @@ -207,23 +229,27 @@ def __call__(self, place_on_cpu = os.getenv('place_embedding_on_cpu') place_on_cpu = eval(place_on_cpu) if place_on_cpu else False - with conditional(self._is_predicting and place_on_cpu, - ops.device('/CPU:0')): - seq_features = self._seq_input_layer(features, group_name, - feature_name_to_output_tensors, - allow_key_search, scope_name) + with conditional( + self._is_predicting and place_on_cpu, ops.device('/CPU:0') + ): + seq_features = self._seq_input_layer( + features, group_name, feature_name_to_output_tensors, + allow_key_search, scope_name + ) # apply regularization for sequence feature key in seq_input_layer. regularizers.apply_regularization( - self._embedding_regularizer, - weights_list=[seq_features['hist_seq_emb']]) + self._embedding_regularizer, + weights_list=[seq_features['hist_seq_emb']] + ) seq_dnn_config = None if seq_att_map_config.HasField('seq_dnn'): seq_dnn_config = seq_att_map_config.seq_dnn else: logging.info( - 'seq_dnn not set in seq_att_groups, will use default settings') + 'seq_dnn not set in seq_att_groups, will use default settings' + ) # If not set seq_dnn, will use default settings from easy_rec.python.protos.dnn_pb2 import DNN seq_dnn_config = DNN() @@ -231,19 +257,21 @@ def __call__(self, cur_target_attention_name = 'seq_dnn' + group_name if negative_sampler: seq_fea, concat_features = self.negative_sampler_target_attention( - seq_dnn_config, - seq_features, - concat_features, - name=cur_target_attention_name, - need_key_feature=need_key_feature, - allow_key_transform=allow_key_transform) + seq_dnn_config, + seq_features, + concat_features, + name=cur_target_attention_name, + need_key_feature=need_key_feature, + allow_key_transform=allow_key_transform + ) else: seq_fea = self.target_attention( - seq_dnn_config, - seq_features, - name=cur_target_attention_name, - need_key_feature=need_key_feature, - allow_key_transform=allow_key_transform, - transform_dnn=transform_dnn) + seq_dnn_config, + seq_features, + name=cur_target_attention_name, + need_key_feature=need_key_feature, + allow_key_transform=allow_key_transform, + transform_dnn=transform_dnn + ) all_seq_fea.append(seq_fea) return concat_features, all_seq_fea diff --git a/easy_rec/python/layers/uniter.py b/easy_rec/python/layers/uniter.py index 3018bad61..e8f119dea 100644 --- a/easy_rec/python/layers/uniter.py +++ b/easy_rec/python/layers/uniter.py @@ -2,8 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import dnn -from easy_rec.python.layers import multihead_cross_attention +from easy_rec.python.layers import dnn, multihead_cross_attention from easy_rec.python.utils.activation import get_activation from easy_rec.python.utils.shape_utils import get_shape_list @@ -18,8 +17,9 @@ class Uniter(object): https://arxiv.org/abs/1909.11740 """ - def __init__(self, model_config, feature_configs, features, uniter_config, - input_layer): + def __init__( + self, model_config, feature_configs, features, uniter_config, input_layer + ): self._model_config = uniter_config tower_num = 0 self._img_features = None @@ -33,7 +33,8 @@ def __init__(self, model_config, feature_configs, features, uniter_config, self._txt_seq_features = None if input_layer.has_group('text'): self._txt_seq_features, _, _ = input_layer( - features, 'text', is_combine=False) + features, 'text', is_combine=False + ) tower_num += 1 self._use_token_type = True if tower_num > 1 else False self._other_features = None @@ -51,17 +52,20 @@ def __init__(self, model_config, feature_configs, features, uniter_config, self._general_feature_num = len(fea_group.feature_names) general_feature_names = set(fea_group.feature_names) assert self._general_feature_num == len(general_feature_names), ( - 'there are duplicate features in `general` feature group') + 'there are duplicate features in `general` feature group' + ) elif fea_group.group_name == 'image': self._img_feature_num = len(fea_group.feature_names) img_feature_names = set(fea_group.feature_names) assert self._img_feature_num == len(img_feature_names), ( - 'there are duplicate features in `image` feature group') + 'there are duplicate features in `image` feature group' + ) elif fea_group.group_name == 'text': self._txt_feature_num = len(fea_group.feature_names) txt_feature_names = set(fea_group.feature_names) assert self._txt_feature_num == len(txt_feature_names), ( - 'there are duplicate features in `text` feature group') + 'there are duplicate features in `text` feature group' + ) if self._txt_feature_num > 1 or self._img_feature_num > 1: self._use_token_type = True @@ -87,41 +91,44 @@ def __init__(self, model_config, feature_configs, features, uniter_config, txt_fea_emb_dim_list.append(feature_config.embedding_dim) if feature_config.HasField('max_seq_len'): assert feature_config.max_seq_len > 0, ( - 'feature config `max_seq_len` must be greater than 0 for feature: ' - + fea_name) + 'feature config `max_seq_len` must be greater than 0 for feature: ' + + fea_name + ) if feature_config.max_seq_len > max_seq_len: max_seq_len = feature_config.max_seq_len unique_dim_num = len(set(txt_fea_emb_dim_list)) assert unique_dim_num <= 1 and len( - txt_fea_emb_dim_list + txt_fea_emb_dim_list ) == self._txt_feature_num, ( - 'Uniter requires that all `text` feature dimensions must be consistent.' + 'Uniter requires that all `text` feature dimensions must be consistent.' ) unique_dim_num = len(set(img_fea_emb_dim_list)) assert unique_dim_num <= 1 and len( - img_fea_emb_dim_list + img_fea_emb_dim_list ) == self._img_feature_num, ( - 'Uniter requires that all `image` feature dimensions must be consistent.' + 'Uniter requires that all `image` feature dimensions must be consistent.' ) unique_dim_num = len(set(general_emb_dim_list)) assert unique_dim_num <= 1 and len( - general_emb_dim_list + general_emb_dim_list ) == self._general_feature_num, ( - 'Uniter requires that all `general` feature dimensions must be consistent.' + 'Uniter requires that all `general` feature dimensions must be consistent.' ) if self._txt_feature_num > 0 and uniter_config.use_position_embeddings: assert uniter_config.max_position_embeddings > 0, ( - 'model config `max_position_embeddings` must be greater than 0. ') + 'model config `max_position_embeddings` must be greater than 0. ' + ) assert uniter_config.max_position_embeddings >= max_seq_len, ( - 'model config `max_position_embeddings` must be greater than or equal to the maximum of all feature config ' - '`max_seq_len`, which is %d' % max_seq_len) + 'model config `max_position_embeddings` must be greater than or equal to the maximum of all feature config ' + '`max_seq_len`, which is %d' % max_seq_len + ) self._img_emb_size = img_fea_emb_dim_list[0] if img_fea_emb_dim_list else 0 self._txt_emb_size = txt_fea_emb_dim_list[0] if txt_fea_emb_dim_list else 0 self._general_emb_size = general_emb_dim_list[ - 0] if general_emb_dim_list else 0 + 0] if general_emb_dim_list else 0 if self._img_features is not None: assert self._img_emb_size > 0, '`image` feature dimensions must be greater than 0, set by `raw_input_dim`' @@ -134,28 +141,34 @@ def text_embeddings(self, token_type_id): if self._general_emb_size != hidden_size: # Run a linear projection of `hidden_size` general_features = tf.reshape( - general_features, shape=[-1, self._general_emb_size]) + general_features, shape=[-1, self._general_emb_size] + ) general_features = tf.layers.dense( - general_features, hidden_size, name='txt_projection') + general_features, hidden_size, name='txt_projection' + ) general_features = tf.reshape( - general_features, shape=[-1, self._general_feature_num, hidden_size]) + general_features, shape=[-1, self._general_feature_num, hidden_size] + ) batch_size = tf.shape(general_features)[0] general_features = multihead_cross_attention.embedding_postprocessor( - general_features, - use_token_type=self._use_token_type, - token_type_ids=tf.ones( - shape=tf.stack([batch_size, self._general_feature_num]), - dtype=tf.int32) * token_type_id, - token_type_vocab_size=self._token_type_vocab_size, - reuse_token_type=tf.AUTO_REUSE, - use_position_embeddings=False, - dropout_prob=self._model_config.hidden_dropout_prob) + general_features, + use_token_type=self._use_token_type, + token_type_ids=tf.ones( + shape=tf.stack([batch_size, self._general_feature_num]), + dtype=tf.int32 + ) * token_type_id, + token_type_vocab_size=self._token_type_vocab_size, + reuse_token_type=tf.AUTO_REUSE, + use_position_embeddings=False, + dropout_prob=self._model_config.hidden_dropout_prob + ) all_txt_features.append(general_features) mask = tf.ones( - shape=tf.stack([batch_size, self._general_feature_num]), - dtype=tf.int32) + shape=tf.stack([batch_size, self._general_feature_num]), + dtype=tf.int32 + ) input_masks.append(mask) if self._txt_seq_features is not None: @@ -171,26 +184,29 @@ def dynamic_mask(x, max_len): if emb_size != hidden_size: seq_fea = tf.reshape(seq_fea, shape=[-1, emb_size]) seq_fea = tf.layers.dense( - seq_fea, hidden_size, name='txt_seq_projection_%d' % i) + seq_fea, hidden_size, name='txt_seq_projection_%d' % i + ) seq_fea = tf.reshape(seq_fea, shape=[-1, max_seq_len, hidden_size]) seq_fea = multihead_cross_attention.embedding_postprocessor( - seq_fea, - use_token_type=self._use_token_type, - token_type_ids=tf.ones( - shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * - (i + token_type_id), - token_type_vocab_size=self._token_type_vocab_size, - reuse_token_type=tf.AUTO_REUSE, - use_position_embeddings=self._model_config.use_position_embeddings, - max_position_embeddings=self._model_config.max_position_embeddings, - position_embedding_name='txt_position_embeddings_%d' % i, - dropout_prob=self._model_config.hidden_dropout_prob) + seq_fea, + use_token_type=self._use_token_type, + token_type_ids=tf. + ones(shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * + (i + token_type_id), + token_type_vocab_size=self._token_type_vocab_size, + reuse_token_type=tf.AUTO_REUSE, + use_position_embeddings=self._model_config.use_position_embeddings, + max_position_embeddings=self._model_config.max_position_embeddings, + position_embedding_name='txt_position_embeddings_%d' % i, + dropout_prob=self._model_config.hidden_dropout_prob + ) all_txt_features.append(seq_fea) input_mask = tf.map_fn( - fn=lambda t: dynamic_mask(t, max_seq_len), - elems=tf.to_int32(seq_len)) + fn=lambda t: dynamic_mask(t, max_seq_len), + elems=tf.to_int32(seq_len) + ) input_masks.append(input_mask) return all_txt_features, input_masks @@ -203,25 +219,29 @@ def image_embeddings(self): if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` image_features = tf.reshape( - image_features, shape=[-1, self._img_emb_size]) + image_features, shape=[-1, self._img_emb_size] + ) image_features = tf.layers.dense( - image_features, hidden_size, name='img_projection') + image_features, hidden_size, name='img_projection' + ) image_features = tf.reshape( - image_features, shape=[-1, self._img_feature_num, hidden_size]) + image_features, shape=[-1, self._img_feature_num, hidden_size] + ) batch_size = tf.shape(image_features)[0] img_fea = multihead_cross_attention.embedding_postprocessor( - image_features, - use_token_type=self._use_token_type, - token_type_ids=tf.zeros( - shape=tf.stack([batch_size, self._img_feature_num]), - dtype=tf.int32), - token_type_vocab_size=self._token_type_vocab_size, - reuse_token_type=tf.AUTO_REUSE, - use_position_embeddings=self._model_config.use_position_embeddings, - max_position_embeddings=self._model_config.max_position_embeddings, - position_embedding_name='img_position_embeddings', - dropout_prob=self._model_config.hidden_dropout_prob) + image_features, + use_token_type=self._use_token_type, + token_type_ids=tf.zeros( + shape=tf.stack([batch_size, self._img_feature_num]), dtype=tf.int32 + ), + token_type_vocab_size=self._token_type_vocab_size, + reuse_token_type=tf.AUTO_REUSE, + use_position_embeddings=self._model_config.use_position_embeddings, + max_position_embeddings=self._model_config.max_position_embeddings, + position_embedding_name='img_position_embeddings', + dropout_prob=self._model_config.hidden_dropout_prob + ) return img_fea def __call__(self, is_training, *args, **kwargs): @@ -256,7 +276,8 @@ def __call__(self, is_training, *args, **kwargs): if img_fea is not None: all_features.append(img_fea) mask = tf.ones( - shape=tf.stack([batch_size, self._img_feature_num]), dtype=tf.int32) + shape=tf.stack([batch_size, self._img_feature_num]), dtype=tf.int32 + ) masks.append(mask) if txt_features: @@ -266,21 +287,23 @@ def __call__(self, is_training, *args, **kwargs): all_fea = tf.concat(all_features, axis=1) input_mask = tf.concat(masks, axis=1) attention_mask = multihead_cross_attention.create_attention_mask_from_input_mask( - from_tensor=all_fea, to_mask=input_mask) + from_tensor=all_fea, to_mask=input_mask + ) hidden_act = get_activation(self._model_config.hidden_act) attention_fea = multihead_cross_attention.transformer_encoder( - all_fea, - hidden_size=hidden_size, - num_hidden_layers=self._model_config.num_hidden_layers, - num_attention_heads=self._model_config.num_attention_heads, - attention_mask=attention_mask, - intermediate_size=self._model_config.intermediate_size, - intermediate_act_fn=hidden_act, - hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config - .attention_probs_dropout_prob, - initializer_range=self._model_config.initializer_range, - name='uniter') # shape: [batch_size, seq_length, hidden_size] + all_fea, + hidden_size=hidden_size, + num_hidden_layers=self._model_config.num_hidden_layers, + num_attention_heads=self._model_config.num_attention_heads, + attention_mask=attention_mask, + intermediate_size=self._model_config.intermediate_size, + intermediate_act_fn=hidden_act, + hidden_dropout_prob=self._model_config.hidden_dropout_prob, + attention_probs_dropout_prob=self._model_config. + attention_probs_dropout_prob, + initializer_range=self._model_config.initializer_range, + name='uniter' + ) # shape: [batch_size, seq_length, hidden_size] print('attention_fea:', attention_fea.shape) mm_fea = attention_fea[:, 0, :] # [CLS] feature sub_modules.append(mm_fea) @@ -288,8 +311,10 @@ def __call__(self, is_training, *args, **kwargs): if self._other_features is not None: if self._model_config.HasField('other_feature_dnn'): l2_reg = kwargs['l2_reg'] if 'l2_reg' in kwargs else 0 - other_dnn_layer = dnn.DNN(self._model_config.other_feature_dnn, l2_reg, - 'other_dnn', is_training) + other_dnn_layer = dnn.DNN( + self._model_config.other_feature_dnn, l2_reg, 'other_dnn', + is_training + ) other_fea = other_dnn_layer(self._other_features) else: other_fea = self._other_features diff --git a/easy_rec/python/layers/utils.py b/easy_rec/python/layers/utils.py index 7eb86b791..df7a52e5e 100644 --- a/easy_rec/python/layers/utils.py +++ b/easy_rec/python/layers/utils.py @@ -13,16 +13,12 @@ # limitations under the License. # ============================================================================== """Common util functions used by layers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import json - from google.protobuf import struct_pb2 from google.protobuf.descriptor import FieldDescriptor -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import ops, sparse_tensor from tensorflow.python.ops import variables try: @@ -35,9 +31,9 @@ def _tensor_to_map(tensor): return { - 'node_path': tensor.name, - 'shape': tensor.shape.as_list() if tensor.shape else None, - 'dtype': tensor.dtype.name + 'node_path': tensor.name, + 'shape': tensor.shape.as_list() if tensor.shape else None, + 'dtype': tensor.dtype.name } @@ -80,12 +76,14 @@ def _process_item(collection_name, name, func): idx_found = ColumnNameInCollection[key] if idx_found >= len(col): raise Exception( - 'Find column name in collection failed: index out of range') + 'Find column name in collection failed: index out of range' + ) item_found = json.loads(col[idx_found]) if item_found['name'] != name: raise Exception( - 'Find column name in collection failed: item name not match') + 'Find column name in collection failed: item name not match' + ) func(item_found) col[idx_found] = json.dumps(item_found) else: @@ -125,11 +123,13 @@ def unique_name_in_collection(collection_name, name): return unique_name -def gen_embedding_attrs(column=None, - variable=None, - bucket_size=None, - combiner=None, - is_embedding_var=None): +def gen_embedding_attrs( + column=None, + variable=None, + bucket_size=None, + combiner=None, + is_embedding_var=None +): attrs = dict() attrs['name'] = column.name attrs['bucket_size'] = bucket_size @@ -143,9 +143,11 @@ def gen_embedding_attrs(column=None, attrs['embedding_var_values'] = variable._shared_name + '-values' elif (isinstance(variable, variables.PartitionedVariable)) and \ (isinstance(variable._get_variable_list()[0], kv_variable_ops.EmbeddingVariable)): - attrs['embedding_var_keys'] = [v._shared_name + '-keys' for v in variable] + attrs['embedding_var_keys'] = [ + v._shared_name + '-keys' for v in variable + ] attrs['embedding_var_values'] = [ - v._shared_name + '-values' for v in variable + v._shared_name + '-values' for v in variable ] else: attrs['is_embedding_var'] = False @@ -155,11 +157,13 @@ def gen_embedding_attrs(column=None, def mark_input_src(name, src_desc): - ops.add_to_collection(ops.GraphKeys.RANK_SERVICE_INPUT_SRC, - json.dumps({ - 'name': name, - 'src': src_desc - })) + ops.add_to_collection( + ops.GraphKeys.RANK_SERVICE_INPUT_SRC, + json.dumps({ + 'name': name, + 'src': src_desc + }) + ) def is_proto_message(pb_obj, field): diff --git a/easy_rec/python/layers/variational_dropout_layer.py b/easy_rec/python/layers/variational_dropout_layer.py index 0eeddcf7b..04312bb7d 100644 --- a/easy_rec/python/layers/variational_dropout_layer.py +++ b/easy_rec/python/layers/variational_dropout_layer.py @@ -1,7 +1,6 @@ # -*- encoding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import json - import numpy as np import tensorflow as tf @@ -20,11 +19,13 @@ class VariationalDropoutLayer(object): arXiv: 1712.08645 """ - def __init__(self, - variational_dropout_config, - features_dimension, - is_training=False, - name=''): + def __init__( + self, + variational_dropout_config, + features_dimension, + is_training=False, + name='' + ): self._config = variational_dropout_config self.features_dimension = features_dimension self.features_total_dimension = sum(self.features_dimension.values()) @@ -39,13 +40,15 @@ def __init__(self, logit_p_name = 'logit_p' if name == 'all' else 'logit_p_%s' % name self.logit_p = tf.get_variable( - name=logit_p_name, - shape=self.drop_param_shape, - dtype=tf.float32, - initializer=None) + name=logit_p_name, + shape=self.drop_param_shape, + dtype=tf.float32, + initializer=None + ) tf.add_to_collection( - 'variational_dropout', - json.dumps([name, list(self.features_dimension.items())])) + 'variational_dropout', + json.dumps([name, list(self.features_dimension.items())]) + ) def get_lambda(self): return self._config.regularization_lambda @@ -63,7 +66,8 @@ def build_expand_index(self, batch_size): batch_size_range = tf.range(batch_size) expand_range_axis = tf.expand_dims(batch_size_range, 1) batch_size_range_expand_dim_len = tf.tile( - expand_range_axis, [1, self.features_total_dimension]) + expand_range_axis, [1, self.features_total_dimension] + ) index_i = tf.reshape(batch_size_range_expand_dim_len, [-1, 1]) expanded_index = tf.concat([index_i, expanded_index], 1) return expanded_index @@ -80,7 +84,9 @@ def sample_noisy_input(self, input): # expand dropout layer expanded_index = self.build_expand_index(batch_size) expanded_p = tf.gather_nd(p, expanded_index) - expanded_p = tf.reshape(expanded_p, [-1, self.features_total_dimension]) + expanded_p = tf.reshape( + expanded_p, [-1, self.features_total_dimension] + ) scaled_input = input * (1 - expanded_p) return scaled_input @@ -107,11 +113,13 @@ def sampled_from_logit_p(self, num_samples): def concrete_dropout_neuron(self, dropout_p, temp=1.0 / 10.0): EPSILON = np.finfo(float).eps unif_noise = tf.random_uniform( - tf.shape(dropout_p), dtype=tf.float32, seed=None, name='unif_noise') + tf.shape(dropout_p), dtype=tf.float32, seed=None, name='unif_noise' + ) approx = ( - tf.log(dropout_p + EPSILON) - tf.log(1. - dropout_p + EPSILON) + - tf.log(unif_noise + EPSILON) - tf.log(1. - unif_noise + EPSILON)) + tf.log(dropout_p + EPSILON) - tf.log(1. - dropout_p + EPSILON) + + tf.log(unif_noise + EPSILON) - tf.log(1. - unif_noise + EPSILON) + ) approx_output = tf.sigmoid(approx / temp) return 1 - approx_output @@ -121,10 +129,12 @@ def __call__(self, output_features): noisy_input = self.sample_noisy_input(output_features) dropout_p = tf.sigmoid(self.logit_p) variational_dropout_penalty = 1. - dropout_p - variational_dropout_penalty_lambda = self.get_lambda() / tf.cast( - batch_size, dtype=tf.float32) + variational_dropout_penalty_lambda = self.get_lambda( + ) / tf.cast(batch_size, dtype=tf.float32) variational_dropout_loss_sum = variational_dropout_penalty_lambda * tf.reduce_sum( - variational_dropout_penalty, axis=0) - tf.add_to_collection('variational_dropout_loss', - variational_dropout_loss_sum) + variational_dropout_penalty, axis=0 + ) + tf.add_to_collection( + 'variational_dropout_loss', variational_dropout_loss_sum + ) return noisy_input diff --git a/easy_rec/python/loss/circle_loss.py b/easy_rec/python/loss/circle_loss.py index 1b442abe1..a03dd4a6a 100644 --- a/easy_rec/python/loss/circle_loss.py +++ b/easy_rec/python/loss/circle_loss.py @@ -6,12 +6,14 @@ tf = tf.compat.v1 -def circle_loss(embeddings, - labels, - sessions=None, - margin=0.25, - gamma=32, - embed_normed=False): +def circle_loss( + embeddings, + labels, + sessions=None, + margin=0.25, + gamma=32, + embed_normed=False +): """Paper: Circle Loss: A Unified Perspective of Pair Similarity Optimization. Link: http://arxiv.org/pdf/2002.10857.pdf @@ -25,9 +27,11 @@ def circle_loss(embeddings, embed_normed: bool, whether input embeddings l2 normalized """ norm_embeddings = embeddings if embed_normed else tf.nn.l2_normalize( - embeddings, axis=-1) + embeddings, axis=-1 + ) pair_wise_cosine_matrix = tf.matmul( - norm_embeddings, norm_embeddings, transpose_b=True) + norm_embeddings, norm_embeddings, transpose_b=True + ) positive_mask = get_anchor_positive_triplet_mask(labels, sessions) negative_mask = 1 - positive_mask - tf.eye(tf.shape(labels)[0]) @@ -35,15 +39,17 @@ def circle_loss(embeddings, delta_p = 1 - margin delta_n = margin - ap = tf.nn.relu(-tf.stop_gradient(pair_wise_cosine_matrix * positive_mask) + - 1 + margin) + ap = tf.nn.relu( + -tf.stop_gradient(pair_wise_cosine_matrix * positive_mask) + 1 + margin + ) an = tf.nn.relu( - tf.stop_gradient(pair_wise_cosine_matrix * negative_mask) + margin) + tf.stop_gradient(pair_wise_cosine_matrix * negative_mask) + margin + ) - logit_p = -ap * (pair_wise_cosine_matrix - - delta_p) * gamma * positive_mask - (1 - positive_mask) * 1e12 - logit_n = an * (pair_wise_cosine_matrix - - delta_n) * gamma * negative_mask - (1 - negative_mask) * 1e12 + logit_p = -ap * (pair_wise_cosine_matrix - delta_p + ) * gamma * positive_mask - (1 - positive_mask) * 1e12 + logit_n = an * (pair_wise_cosine_matrix - delta_n + ) * gamma * negative_mask - (1 - negative_mask) * 1e12 joint_neg_loss = tf.reduce_logsumexp(logit_n, axis=-1) joint_pos_loss = tf.reduce_logsumexp(logit_p, axis=-1) @@ -74,7 +80,8 @@ def get_anchor_positive_triplet_mask(labels, sessions=None): class_equal = labels_equal else: sessions_equal = tf.equal( - tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1)) + tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1) + ) class_equal = tf.logical_and(sessions_equal, labels_equal) # Combine the three masks diff --git a/easy_rec/python/loss/contrastive_loss.py b/easy_rec/python/loss/contrastive_loss.py index 3fd2be645..cadc36e3f 100644 --- a/easy_rec/python/loss/contrastive_loss.py +++ b/easy_rec/python/loss/contrastive_loss.py @@ -31,7 +31,7 @@ def info_nce_loss(query, positive, temperature=0.1): # Embedding vectors should have same number of components. if query.shape[-1] != positive.shape[-1]: raise ValueError( - 'Vectors of and should have the same number of components.' + 'Vectors of and should have the same number of components.' ) # Negative keys are implicitly off-diagonal positive keys. @@ -65,9 +65,11 @@ def nce_loss(z_i, z_j, temperature=1.0): z = tf.concat((z_i, z_j), axis=0) sim = tf.matmul(z, tf.transpose(z)) / temperature sim_i_j = tf.matrix_diag_part( - tf.slice(sim, [batch_size, 0], [batch_size, batch_size])) + tf.slice(sim, [batch_size, 0], [batch_size, batch_size]) + ) sim_j_i = tf.matrix_diag_part( - tf.slice(sim, [0, batch_size], [batch_size, batch_size])) + tf.slice(sim, [0, batch_size], [batch_size, batch_size]) + ) positive_samples = tf.reshape(tf.concat((sim_i_j, sim_j_i), axis=0), (N, 1)) mask = get_mask_matrix(batch_size) negative_samples = tf.reshape(tf.boolean_mask(sim, mask), (N, -1)) diff --git a/easy_rec/python/loss/f1_reweight_loss.py b/easy_rec/python/loss/f1_reweight_loss.py index 3f9689f4d..ffe83a591 100644 --- a/easy_rec/python/loss/f1_reweight_loss.py +++ b/easy_rec/python/loss/f1_reweight_loss.py @@ -7,11 +7,9 @@ tf = tf.compat.v1 -def f1_reweight_sigmoid_cross_entropy(labels, - logits, - beta_square, - label_smoothing=0, - weights=None): +def f1_reweight_sigmoid_cross_entropy( + labels, logits, beta_square, label_smoothing=0, weights=None +): """Refer paper: Adaptive Scaling for Sparse Detection in Information Extraction.""" probs = tf.nn.sigmoid(logits) if len(logits.shape.as_list()) == 1: @@ -28,11 +26,13 @@ def f1_reweight_sigmoid_cross_entropy(labels, neg_weight = tp / (beta_square * num_pos + num_neg - tn + 1e-8) neg_weight_tile = tf.tile(tf.expand_dims(neg_weight, 0), [batch_size, 1]) final_weights = tf.where( - tf.equal(labels, 1.0), tf.ones_like(labels), neg_weight_tile) + tf.equal(labels, 1.0), tf.ones_like(labels), neg_weight_tile + ) if weights is not None: weights = tf.cast(weights, tf.float32) if len(weights.shape.as_list()) == 1: weights = tf.expand_dims(weights, -1) final_weights *= weights return tf.losses.sigmoid_cross_entropy( - labels, logits, final_weights, label_smoothing=label_smoothing) + labels, logits, final_weights, label_smoothing=label_smoothing + ) diff --git a/easy_rec/python/loss/focal_loss.py b/easy_rec/python/loss/focal_loss.py index 9ef6a94a7..29d15ec91 100644 --- a/easy_rec/python/loss/focal_loss.py +++ b/easy_rec/python/loss/focal_loss.py @@ -1,21 +1,22 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf if tf.__version__ >= '2.0': tf = tf.compat.v1 -def sigmoid_focal_loss_with_logits(labels, - logits, - gamma=2.0, - alpha=None, - ohem_ratio=1.0, - sample_weights=None, - label_smoothing=0, - name=''): +def sigmoid_focal_loss_with_logits( + labels, + logits, + gamma=2.0, + alpha=None, + ohem_ratio=1.0, + sample_weights=None, + label_smoothing=0, + name='' +): """Implements the focal loss function. Focal loss was first introduced in the RetinaNet paper @@ -52,8 +53,10 @@ def sigmoid_focal_loss_with_logits(labels, if gamma and gamma < 0: raise ValueError('Value of gamma should be greater than or equal to zero') logging.info( - '[{}] gamma: {}, alpha: {}, ohem_ratho: {}, label smoothing: {}'.format( - loss_name, gamma, alpha, ohem_ratio, label_smoothing)) + '[{}] gamma: {}, alpha: {}, ohem_ratho: {}, label smoothing: {}'.format( + loss_name, gamma, alpha, ohem_ratio, label_smoothing + ) + ) y_true = tf.cast(labels, logits.dtype) @@ -78,14 +81,16 @@ def sigmoid_focal_loss_with_logits(labels, if ohem_ratio == 1.0: return tf.losses.sigmoid_cross_entropy( - y_true, logits, weights=weights, label_smoothing=label_smoothing) + y_true, logits, weights=weights, label_smoothing=label_smoothing + ) losses = tf.losses.sigmoid_cross_entropy( - y_true, - logits, - weights=weights, - label_smoothing=label_smoothing, - reduction=tf.losses.Reduction.NONE) + y_true, + logits, + weights=weights, + label_smoothing=label_smoothing, + reduction=tf.losses.Reduction.NONE + ) k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio) k = tf.to_int32(tf.math.rint(k)) topk = tf.nn.top_k(losses, k) diff --git a/easy_rec/python/loss/jrc_loss.py b/easy_rec/python/loss/jrc_loss.py index b5165d3c2..69c335ef0 100644 --- a/easy_rec/python/loss/jrc_loss.py +++ b/easy_rec/python/loss/jrc_loss.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import numpy as np import tensorflow as tf @@ -9,14 +8,16 @@ tf = tf.compat.v1 -def jrc_loss(labels, - logits, - session_ids, - alpha=0.5, - loss_weight_strategy='fixed', - sample_weights=1.0, - same_label_loss=True, - name=''): +def jrc_loss( + labels, + logits, + session_ids, + alpha=0.5, + loss_weight_strategy='fixed', + sample_weights=1.0, + same_label_loss=True, + name='' +): """Joint Optimization of Ranking and Calibration with Contextualized Hybrid Model. https://arxiv.org/abs/2208.06164 @@ -33,11 +34,15 @@ def jrc_loss(labels, name: the name of loss """ loss_name = name if name else 'jrc_loss' - logging.info('[{}] alpha: {}, loss_weight_strategy: {}'.format( - loss_name, alpha, loss_weight_strategy)) + logging.info( + '[{}] alpha: {}, loss_weight_strategy: {}'.format( + loss_name, alpha, loss_weight_strategy + ) + ) ce_loss = tf.losses.sparse_softmax_cross_entropy( - labels, logits, weights=sample_weights) + labels, logits, weights=sample_weights + ) labels = tf.expand_dims(labels, 1) # [B, 1] labels = tf.concat([1 - labels, labels], axis=1) # [B, 2] @@ -47,7 +52,8 @@ def jrc_loss(labels, # Mask: shape [B, B], mask[i,j]=1 indicates the i-th sample # and j-th sample are in the same context mask = tf.equal( - tf.expand_dims(session_ids, 1), tf.expand_dims(session_ids, 0)) + tf.expand_dims(session_ids, 1), tf.expand_dims(session_ids, 0) + ) mask = tf.to_float(mask) # Tile logits and label: [B, 2]->[B, B, 2] @@ -74,7 +80,8 @@ def jrc_loss(labels, loss_pos = -tf.reduce_sum(y_pos * tf.nn.log_softmax(l_pos, axis=0), axis=0) loss_neg = -tf.reduce_sum(y_neg * tf.nn.log_softmax(l_neg, axis=0), axis=0) ge_loss = tf.reduce_mean( - (loss_pos + loss_neg) / tf.reduce_sum(mask, axis=0)) + (loss_pos + loss_neg) / tf.reduce_sum(mask, axis=0) + ) else: logging.info('[%s] disable same_label_loss' % loss_name) diag = tf.one_hot(tf.range(batch_size), batch_size) @@ -105,24 +112,30 @@ def jrc_loss(labels, bern = tf.distributions.Bernoulli(probs=0.5, dtype=tf.float32) weights = bern.sample(2) loss_weight = tf.cond( - tf.equal(tf.reduce_sum(weights), 1), lambda: weights, - lambda: tf.convert_to_tensor([0.5, 0.5])) + tf.equal(tf.reduce_sum(weights), 1), lambda: weights, + lambda: tf.convert_to_tensor([0.5, 0.5]) + ) loss = loss_weight[0] * ce_loss + loss_weight[1] * ge_loss tf.summary.scalar('loss/%s_ce_weight' % loss_name, loss_weight[0]) tf.summary.scalar('loss/%s_ge_weight' % loss_name, loss_weight[1]) elif loss_weight_strategy == 'uncertainty': uncertainty1 = tf.Variable( - 0, name='%s_ranking_loss_weight' % loss_name, dtype=tf.float32) + 0, name='%s_ranking_loss_weight' % loss_name, dtype=tf.float32 + ) tf.summary.scalar('loss/%s_ranking_uncertainty' % loss_name, uncertainty1) uncertainty2 = tf.Variable( - 0, name='%s_calibration_loss_weight' % loss_name, dtype=tf.float32) - tf.summary.scalar('loss/%s_calibration_uncertainty' % loss_name, - uncertainty2) + 0, name='%s_calibration_loss_weight' % loss_name, dtype=tf.float32 + ) + tf.summary.scalar( + 'loss/%s_calibration_uncertainty' % loss_name, uncertainty2 + ) loss = tf.exp(-uncertainty1) * ce_loss + 0.5 * uncertainty1 loss += tf.exp(-uncertainty2) * ge_loss + 0.5 * uncertainty2 else: - raise ValueError('Unsupported loss weight strategy `%s` for jrc loss' % - loss_weight_strategy) + raise ValueError( + 'Unsupported loss weight strategy `%s` for jrc loss' % + loss_weight_strategy + ) if np.isscalar(sample_weights) and sample_weights != 1.0: return loss * sample_weights return loss diff --git a/easy_rec/python/loss/listwise_loss.py b/easy_rec/python/loss/listwise_loss.py index 24bd5864f..5f70a6051 100644 --- a/easy_rec/python/loss/listwise_loss.py +++ b/easy_rec/python/loss/listwise_loss.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.utils.load_class import load_by_path @@ -25,15 +24,17 @@ def _list_prob_loss(x, labels, logits, session_ids): return -tf.reduce_sum(y * y_hat) -def listwise_rank_loss(labels, - logits, - session_ids, - transform_fn=None, - temperature=1.0, - label_is_logits=False, - scale_logits=False, - weights=1.0, - name='listwise_loss'): +def listwise_rank_loss( + labels, + logits, + session_ids, + transform_fn=None, + temperature=1.0, + label_is_logits=False, + scale_logits=False, + weights=1.0, + name='listwise_loss' +): r"""Computes listwise softmax cross entropy loss between `labels` and `logits`. Definition: @@ -56,21 +57,26 @@ def listwise_rank_loss(labels, name: the name of loss """ loss_name = name if name else 'listwise_rank_loss' - logging.info('[{}] temperature: {}, scale logits: {}'.format( - loss_name, temperature, scale_logits)) + logging.info( + '[{}] temperature: {}, scale logits: {}'.format( + loss_name, temperature, scale_logits + ) + ) labels = tf.to_float(labels) if scale_logits: with tf.variable_scope(loss_name): w = tf.get_variable( - 'scale_w', - dtype=tf.float32, - shape=(1,), - initializer=tf.ones_initializer()) + 'scale_w', + dtype=tf.float32, + shape=(1, ), + initializer=tf.ones_initializer() + ) b = tf.get_variable( - 'scale_b', - dtype=tf.float32, - shape=(1,), - initializer=tf.zeros_initializer()) + 'scale_b', + dtype=tf.float32, + shape=(1, ), + initializer=tf.zeros_initializer() + ) logits = logits * tf.abs(w) + b if temperature != 1.0: logits /= temperature @@ -83,10 +89,10 @@ def listwise_rank_loss(labels, sessions, _ = tf.unique(tf.squeeze(session_ids)) tf.summary.scalar('loss/%s_num_of_group' % loss_name, tf.size(sessions)) losses = tf.map_fn( - lambda x: _list_wise_loss(x, labels, logits, session_ids, label_is_logits - ), - sessions, - dtype=tf.float32) + lambda x: _list_wise_loss(x, labels, logits, session_ids, label_is_logits), + sessions, + dtype=tf.float32 + ) if tf.is_numeric_tensor(weights): logging.error('[%s] use unsupported sample weight' % loss_name) return tf.reduce_mean(losses) @@ -94,15 +100,17 @@ def listwise_rank_loss(labels, return tf.reduce_mean(losses) * weights -def listwise_distill_loss(labels, - logits, - session_ids, - transform_fn=None, - temperature=1.0, - label_clip_max_value=512, - scale_logits=False, - weights=1.0, - name='listwise_distill_loss'): +def listwise_distill_loss( + labels, + logits, + session_ids, + transform_fn=None, + temperature=1.0, + label_clip_max_value=512, + scale_logits=False, + weights=1.0, + name='listwise_distill_loss' +): r"""Computes listwise softmax cross entropy loss between `labels` and `logits`. Definition: @@ -135,15 +143,17 @@ def listwise_distill_loss(labels, if scale_logits: with tf.variable_scope(loss_name): w = tf.get_variable( - 'scale_w', - dtype=tf.float32, - shape=(1,), - initializer=tf.ones_initializer()) + 'scale_w', + dtype=tf.float32, + shape=(1, ), + initializer=tf.ones_initializer() + ) b = tf.get_variable( - 'scale_b', - dtype=tf.float32, - shape=(1,), - initializer=tf.zeros_initializer()) + 'scale_b', + dtype=tf.float32, + shape=(1, ), + initializer=tf.zeros_initializer() + ) logits = logits * tf.abs(w) + b if temperature != 1.0: logits /= temperature @@ -151,9 +161,10 @@ def listwise_distill_loss(labels, sessions, _ = tf.unique(tf.squeeze(session_ids)) tf.summary.scalar('loss/%s_num_of_group' % loss_name, tf.size(sessions)) losses = tf.map_fn( - lambda x: _list_prob_loss(x, labels, logits, session_ids), - sessions, - dtype=tf.float32) + lambda x: _list_prob_loss(x, labels, logits, session_ids), + sessions, + dtype=tf.float32 + ) if tf.is_numeric_tensor(weights): logging.error('[%s] use unsupported sample weight' % loss_name) return tf.reduce_mean(losses) diff --git a/easy_rec/python/loss/multi_similarity.py b/easy_rec/python/loss/multi_similarity.py index 2b7d4e0e9..a625a662a 100644 --- a/easy_rec/python/loss/multi_similarity.py +++ b/easy_rec/python/loss/multi_similarity.py @@ -8,15 +8,17 @@ tf = tf.compat.v1 -def ms_loss(embeddings, - labels, - session_ids=None, - alpha=2.0, - beta=50.0, - lamb=1.0, - eps=0.1, - ms_mining=False, - embed_normed=False): +def ms_loss( + embeddings, + labels, + session_ids=None, + alpha=2.0, + beta=50.0, + lamb=1.0, + eps=0.1, + ms_mining=False, + embed_normed=False +): """Refer paper: Multi-Similarity Loss with General Pair Weighting for Deep Metric Learning. ref: http://openaccess.thecvf.com/content_CVPR_2019/papers/ @@ -34,7 +36,8 @@ def ms_loss(embeddings, mask_neg = 1 - mask_pos - tf.eye(batch_size) sim_mat = tf.matmul( - embeddings, embeddings, transpose_a=False, transpose_b=True) + embeddings, embeddings, transpose_a=False, transpose_b=True + ) sim_mat = tf.maximum(sim_mat, 0.0) pos_mat = tf.multiply(sim_mat, mask_pos) @@ -44,16 +47,18 @@ def ms_loss(embeddings, max_val = tf.reduce_max(neg_mat, axis=1, keepdims=True) tmp_max_val = tf.reduce_max(pos_mat, axis=1, keepdims=True) min_val = tf.reduce_min( - tf.multiply(sim_mat - tmp_max_val, mask_pos), axis=1, - keepdims=True) + tmp_max_val + tf.multiply(sim_mat - tmp_max_val, mask_pos), axis=1, keepdims=True + ) + tmp_max_val max_val = tf.tile(max_val, [1, batch_size]) min_val = tf.tile(min_val, [1, batch_size]) - mask_pos = tf.where(pos_mat < max_val + eps, mask_pos, - tf.zeros_like(mask_pos)) - mask_neg = tf.where(neg_mat > min_val - eps, mask_neg, - tf.zeros_like(mask_neg)) + mask_pos = tf.where( + pos_mat < max_val + eps, mask_pos, tf.zeros_like(mask_pos) + ) + mask_neg = tf.where( + neg_mat > min_val - eps, mask_neg, tf.zeros_like(mask_neg) + ) pos_exp = tf.exp(-alpha * (pos_mat - lamb)) pos_exp = tf.where(mask_pos > 0.0, pos_exp, tf.zeros_like(pos_exp)) diff --git a/easy_rec/python/loss/pairwise_loss.py b/easy_rec/python/loss/pairwise_loss.py index 604f1ce2e..f9a674f19 100644 --- a/easy_rec/python/loss/pairwise_loss.py +++ b/easy_rec/python/loss/pairwise_loss.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from tensorflow.python.ops.losses.losses_impl import compute_weighted_loss @@ -12,13 +11,15 @@ tf = tf.compat.v1 -def pairwise_loss(labels, - logits, - session_ids=None, - margin=0, - temperature=1.0, - weights=1.0, - name=''): +def pairwise_loss( + labels, + logits, + session_ids=None, + margin=0, + temperature=1.0, + weights=1.0, + name='' +): """Deprecated Pairwise loss. Also see `pairwise_logistic_loss` below. Args: @@ -31,22 +32,27 @@ def pairwise_loss(labels, name: the name of loss """ logging.warning( - 'The old `pairwise_loss` is being deprecated. ' - 'Please use the new `pairwise_logistic_loss` or `pairwise_focal_loss`') + 'The old `pairwise_loss` is being deprecated. ' + 'Please use the new `pairwise_logistic_loss` or `pairwise_focal_loss`' + ) loss_name = name if name else 'pairwise_loss' - logging.info('[{}] margin: {}, temperature: {}'.format( - loss_name, margin, temperature)) + logging.info( + '[{}] margin: {}, temperature: {}'.format(loss_name, margin, temperature) + ) if temperature != 1.0: logits /= temperature pairwise_logits = tf.math.subtract( - tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) - margin + tf.expand_dims(logits, -1), tf.expand_dims(logits, 0) + ) - margin pairwise_mask = tf.greater( - tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) + tf.expand_dims(labels, -1), tf.expand_dims(labels, 0) + ) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) group_equal = tf.equal( - tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) + tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0) + ) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -64,41 +70,47 @@ def pairwise_loss(labels, pairwise_pseudo_labels = tf.ones_like(pairwise_logits) loss = tf.losses.sigmoid_cross_entropy( - pairwise_pseudo_labels, pairwise_logits, weights=pairwise_weights) + pairwise_pseudo_labels, pairwise_logits, weights=pairwise_weights + ) # set rank loss to zero if a batch has no positive sample. # loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss) return loss -def pairwise_focal_loss(labels, - logits, - session_ids=None, - hinge_margin=None, - gamma=2, - alpha=None, - ohem_ratio=1.0, - temperature=1.0, - weights=1.0, - name=''): +def pairwise_focal_loss( + labels, + logits, + session_ids=None, + hinge_margin=None, + gamma=2, + alpha=None, + ohem_ratio=1.0, + temperature=1.0, + weights=1.0, + name='' +): loss_name = name if name else 'pairwise_focal_loss' assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]' logging.info( - '[{}] hinge margin: {}, gamma: {}, alpha: {}, ohem_ratio: {}, temperature: {}' - .format(loss_name, hinge_margin, gamma, alpha, ohem_ratio, temperature)) + '[{}] hinge margin: {}, gamma: {}, alpha: {}, ohem_ratio: {}, temperature: {}' + .format(loss_name, hinge_margin, gamma, alpha, ohem_ratio, temperature) + ) if temperature != 1.0: logits /= temperature pairwise_logits = tf.expand_dims(logits, -1) - tf.expand_dims(logits, 0) pairwise_mask = tf.greater( - tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) + tf.expand_dims(labels, -1), tf.expand_dims(labels, 0) + ) if hinge_margin is not None: hinge_mask = tf.less(pairwise_logits, hinge_margin) pairwise_mask = tf.logical_and(pairwise_mask, hinge_mask) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) group_equal = tf.equal( - tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) + tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0) + ) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -116,24 +128,27 @@ def pairwise_focal_loss(labels, pairwise_pseudo_labels = tf.ones_like(pairwise_logits) loss = sigmoid_focal_loss_with_logits( - pairwise_pseudo_labels, - pairwise_logits, - gamma=gamma, - alpha=alpha, - ohem_ratio=ohem_ratio, - sample_weights=pairwise_weights) + pairwise_pseudo_labels, + pairwise_logits, + gamma=gamma, + alpha=alpha, + ohem_ratio=ohem_ratio, + sample_weights=pairwise_weights + ) return loss -def pairwise_logistic_loss(labels, - logits, - session_ids=None, - temperature=1.0, - hinge_margin=None, - weights=1.0, - ohem_ratio=1.0, - use_label_margin=False, - name=''): +def pairwise_logistic_loss( + labels, + logits, + session_ids=None, + temperature=1.0, + hinge_margin=None, + weights=1.0, + ohem_ratio=1.0, + use_label_margin=False, + name='' +): r"""Computes pairwise logistic loss between `labels` and `logits`, equivalent to RankNet loss. Definition: @@ -157,8 +172,11 @@ def pairwise_logistic_loss(labels, """ loss_name = name if name else 'pairwise_logistic_loss' assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]' - logging.info('[{}] hinge margin: {}, ohem_ratio: {}, temperature: {}'.format( - loss_name, hinge_margin, ohem_ratio, temperature)) + logging.info( + '[{}] hinge margin: {}, ohem_ratio: {}, temperature: {}'.format( + loss_name, hinge_margin, ohem_ratio, temperature + ) + ) if temperature != 1.0: logits /= temperature @@ -166,19 +184,23 @@ def pairwise_logistic_loss(labels, labels /= temperature pairwise_logits = tf.math.subtract( - tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) + tf.expand_dims(logits, -1), tf.expand_dims(logits, 0) + ) if use_label_margin: pairwise_logits -= tf.math.subtract( - tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) + tf.expand_dims(labels, -1), tf.expand_dims(labels, 0) + ) elif hinge_margin is not None: pairwise_logits -= hinge_margin pairwise_mask = tf.greater( - tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) + tf.expand_dims(labels, -1), tf.expand_dims(labels, 0) + ) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) group_equal = tf.equal( - tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) + tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0) + ) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -187,7 +209,8 @@ def pairwise_logistic_loss(labels, # The following is the same as log(1 + exp(-pairwise_logits)). losses = tf.nn.relu(-pairwise_logits) + tf.math.log1p( - tf.exp(-tf.abs(pairwise_logits))) + tf.exp(-tf.abs(pairwise_logits)) + ) if tf.is_numeric_tensor(weights): logging.info('[%s] use sample weight' % loss_name) @@ -202,7 +225,8 @@ def pairwise_logistic_loss(labels, return compute_weighted_loss(losses, pairwise_weights) losses = compute_weighted_loss( - losses, pairwise_weights, reduction=tf.losses.Reduction.NONE) + losses, pairwise_weights, reduction=tf.losses.Reduction.NONE + ) k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio) k = tf.to_int32(tf.math.rint(k)) topk = tf.nn.top_k(losses, k) @@ -210,17 +234,19 @@ def pairwise_logistic_loss(labels, return tf.reduce_mean(losses) -def pairwise_hinge_loss(labels, - logits, - session_ids=None, - temperature=1.0, - margin=1.0, - weights=1.0, - ohem_ratio=1.0, - label_is_logits=True, - use_label_margin=True, - use_exponent=False, - name=''): +def pairwise_hinge_loss( + labels, + logits, + session_ids=None, + temperature=1.0, + margin=1.0, + weights=1.0, + ohem_ratio=1.0, + label_is_logits=True, + use_label_margin=True, + use_exponent=False, + name='' +): r"""Computes pairwise hinge loss between `labels` and `logits`. Definition: @@ -246,9 +272,12 @@ def pairwise_hinge_loss(labels, loss_name = name if name else 'pairwise_hinge_loss' assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]' logging.info( - '[{}] margin: {}, ohem_ratio: {}, temperature: {}, use_exponent: {}, label_is_logits: {}, use_label_margin: {}' - .format(loss_name, margin, ohem_ratio, temperature, use_exponent, - label_is_logits, use_label_margin)) + '[{}] margin: {}, ohem_ratio: {}, temperature: {}, use_exponent: {}, label_is_logits: {}, use_label_margin: {}' + .format( + loss_name, margin, ohem_ratio, temperature, use_exponent, + label_is_logits, use_label_margin + ) + ) if temperature != 1.0: logits /= temperature @@ -259,15 +288,18 @@ def pairwise_hinge_loss(labels, logits = tf.nn.sigmoid(labels) pairwise_logits = tf.math.subtract( - tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) + tf.expand_dims(logits, -1), tf.expand_dims(logits, 0) + ) pairwise_labels = tf.math.subtract( - tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) + tf.expand_dims(labels, -1), tf.expand_dims(labels, 0) + ) pairwise_mask = tf.greater(pairwise_labels, 0) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) group_equal = tf.equal( - tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) + tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0) + ) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -299,7 +331,8 @@ def pairwise_hinge_loss(labels, return compute_weighted_loss(losses, pairwise_weights) losses = compute_weighted_loss( - losses, pairwise_weights, reduction=tf.losses.Reduction.NONE) + losses, pairwise_weights, reduction=tf.losses.Reduction.NONE + ) k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio) k = tf.to_int32(tf.math.rint(k)) topk = tf.nn.top_k(losses, k) diff --git a/easy_rec/python/loss/softmax_loss_with_negative_mining.py b/easy_rec/python/loss/softmax_loss_with_negative_mining.py index 99f92d4af..b9ac28c99 100644 --- a/easy_rec/python/loss/softmax_loss_with_negative_mining.py +++ b/easy_rec/python/loss/softmax_loss_with_negative_mining.py @@ -6,40 +6,40 @@ tf = tf.compat.v1 -def support_vector_guided_softmax_loss(pos_score, - neg_scores, - margin=0, - t=1, - smooth=1.0, - threshold=0, - weights=1.0): +def support_vector_guided_softmax_loss( + pos_score, neg_scores, margin=0, t=1, smooth=1.0, threshold=0, weights=1.0 +): """Refer paper: Support Vector Guided Softmax Loss for Face Recognition (https://128.84.21.199/abs/1812.11317).""" new_pos_score = pos_score - margin cond = tf.greater_equal(new_pos_score - neg_scores, threshold) - mask = tf.where(cond, tf.zeros_like(cond, tf.float32), - tf.ones_like(cond, tf.float32)) # I_k + mask = tf.where( + cond, tf.zeros_like(cond, tf.float32), tf.ones_like(cond, tf.float32) + ) # I_k new_neg_scores = mask * (neg_scores * t + t - 1) + (1 - mask) * neg_scores logits = tf.concat([new_pos_score, new_neg_scores], axis=1) if 1.0 != smooth: logits *= smooth loss = tf.losses.sparse_softmax_cross_entropy( - tf.zeros_like(pos_score, dtype=tf.int32), logits, weights=weights) + tf.zeros_like(pos_score, dtype=tf.int32), logits, weights=weights + ) # set rank loss to zero if a batch has no positive sample. loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss) return loss -def softmax_loss_with_negative_mining(user_emb, - item_emb, - labels, - num_negative_samples=4, - embed_normed=False, - weights=1.0, - gamma=1.0, - margin=0, - t=1, - seed=None): +def softmax_loss_with_negative_mining( + user_emb, + item_emb, + labels, + num_negative_samples=4, + embed_normed=False, + weights=1.0, + gamma=1.0, + margin=0, + t=1, + seed=None +): """Compute the softmax loss based on the cosine distance explained below. Given mini batches for `user_emb` and `item_emb`, this function computes for each element in `user_emb` @@ -72,9 +72,10 @@ def softmax_loss_with_negative_mining(user_emb, batch_size = tf.shape(item_emb)[0] is_valid = tf.assert_less( - num_negative_samples, - batch_size, - message='`num_negative_samples` should be less than batch_size') + num_negative_samples, + batch_size, + message='`num_negative_samples` should be less than batch_size' + ) with tf.control_dependencies([is_valid]): if not embed_normed: user_emb = tf.nn.l2_normalize(user_emb, axis=-1) @@ -96,15 +97,12 @@ def softmax_loss_with_negative_mining(user_emb, # sim_scores's shape: (num_of_pos_label_in_batch_size, num_negative_samples + 1) sim_scores = tf.keras.backend.batch_dot( - mask_user_emb, mask_item_emb, axes=(1, 2)) + mask_user_emb, mask_item_emb, axes=(1, 2) + ) pos_score = tf.slice(sim_scores, [0, 0], [-1, 1]) neg_scores = tf.slice(sim_scores, [0, 1], [-1, -1]) loss = support_vector_guided_softmax_loss( - pos_score, - neg_scores, - margin=margin, - t=t, - smooth=gamma, - weights=weights) + pos_score, neg_scores, margin=margin, t=t, smooth=gamma, weights=weights + ) return loss diff --git a/easy_rec/python/loss/zero_inflated_lognormal.py b/easy_rec/python/loss/zero_inflated_lognormal.py index e5b916794..8a4da34e1 100644 --- a/easy_rec/python/loss/zero_inflated_lognormal.py +++ b/easy_rec/python/loss/zero_inflated_lognormal.py @@ -2,7 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Zero-inflated lognormal loss for lifetime value prediction.""" import logging - import tensorflow as tf import tensorflow_probability as tfp @@ -16,10 +15,9 @@ def log_sigmoid(x): return -tf.nn.softplus(-x) # 兼容 TF 1.12 -def zero_inflated_lognormal_pred(logits, - max_sigma=5.0, - max_log_clip=20.0, - return_log=False): +def zero_inflated_lognormal_pred( + logits, max_sigma=5.0, max_log_clip=20.0, return_log=False +): """Calculates predicted mean of zero inflated lognormal logits. Arguments: @@ -38,8 +36,9 @@ def zero_inflated_lognormal_pred(logits, log_positive_probs = log_sigmoid(logits[..., :1]) mu = logits[..., 1:2] sigma = tf.keras.backend.softplus(logits[..., 2:]) - sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), - max_sigma) + sigma = tf.clip_by_value( + sigma, tf.math.sqrt(tf.keras.backend.epsilon()), max_sigma + ) log_mean_pos = mu + 0.5 * tf.keras.backend.square(sigma) log_preds = log_positive_probs + log_mean_pos if return_log: @@ -51,14 +50,16 @@ def zero_inflated_lognormal_pred(logits, return positive_probs, preds -def zero_inflated_lognormal_loss(labels, - logits, - max_sigma=5.0, - mu_reg=0.01, - sigma_reg=0.01, - class_weight=1.0, - reg_weight=1.0, - name=''): +def zero_inflated_lognormal_loss( + labels, + logits, + max_sigma=5.0, + mu_reg=0.01, + sigma_reg=0.01, + class_weight=1.0, + reg_weight=1.0, + name='' +): """Computes the zero inflated lognormal loss. Usage with tf.keras API: @@ -83,8 +84,9 @@ def zero_inflated_lognormal_loss(labels, """ loss_name = name if name else 'ziln_loss' logging.info( - '%s max_sigma=%f, mu_reg=%f, sigma_reg=%f, classify weight:%f, regression weight %f' - % (loss_name, max_sigma, mu_reg, sigma_reg, class_weight, reg_weight)) + '%s max_sigma=%f, mu_reg=%f, sigma_reg=%f, classify weight:%f, regression weight %f' + % (loss_name, max_sigma, mu_reg, sigma_reg, class_weight, reg_weight) + ) labels = tf.cast(labels, dtype=tf.float32) if labels.shape.ndims == 1: labels = tf.expand_dims(labels, 1) # [B, 1] @@ -92,21 +94,24 @@ def zero_inflated_lognormal_loss(labels, logits = tf.convert_to_tensor(logits, dtype=tf.float32) logits.shape.assert_is_compatible_with( - tf.TensorShape(labels.shape[:-1].as_list() + [3])) + tf.TensorShape(labels.shape[:-1].as_list() + [3]) + ) positive_logits = logits[..., :1] classification_loss = tf.keras.backend.binary_crossentropy( - positive, positive_logits, from_logits=True) + positive, positive_logits, from_logits=True + ) classification_loss = tf.keras.backend.mean(classification_loss) tf.summary.scalar('loss/%s_classify' % loss_name, classification_loss) mu = logits[..., 1:2] sigma = tf.keras.backend.softplus(logits[..., 2:]) - sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), - max_sigma) + sigma = tf.clip_by_value( + sigma, tf.math.sqrt(tf.keras.backend.epsilon()), max_sigma + ) - safe_labels = positive * labels + ( - 1 - positive) * tf.keras.backend.ones_like(labels) + safe_labels = positive * labels + (1 - positive + ) * tf.keras.backend.ones_like(labels) logprob = tfd.LogNormal(loc=mu, scale=sigma).log_prob(safe_labels) num_pos = tf.reduce_sum(positive) + 1e-8 regression_loss = -(tf.reduce_sum(positive * logprob) / num_pos) diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index abce86237..45119bb8e 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -1,39 +1,27 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import json import logging import math import os -import time - import six import tensorflow as tf +import time from tensorflow.core.protobuf import saved_model_pb2 import easy_rec from easy_rec.python.builders import strategy_builder -from easy_rec.python.compat import estimator_train -from easy_rec.python.compat import exporter +from easy_rec.python.compat import estimator_train, exporter from easy_rec.python.input.input import Input from easy_rec.python.model.easy_rec_estimator import EasyRecEstimator from easy_rec.python.model.easy_rec_model import EasyRecModel from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.utils import config_util -from easy_rec.python.utils import constant -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import fg_util -from easy_rec.python.utils import load_class -from easy_rec.python.utils.config_util import get_eval_input_path -from easy_rec.python.utils.config_util import get_model_dir_path -from easy_rec.python.utils.config_util import get_train_input_path -from easy_rec.python.utils.config_util import set_eval_input_path -from easy_rec.python.utils.export_big_model import export_big_model -from easy_rec.python.utils.export_big_model import export_big_model_to_oss +from easy_rec.python.utils import config_util, constant, estimator_utils, fg_util, load_class # NOQA +from easy_rec.python.utils.config_util import get_eval_input_path, get_model_dir_path, get_train_input_path, set_eval_input_path # NOQA +from easy_rec.python.utils.export_big_model import export_big_model, export_big_model_to_oss # NOQA try: import horovod.tensorflow as hvd @@ -64,12 +52,14 @@ BestExporter = exporter.BestExporter -def _get_input_fn(data_config, - feature_configs, - data_path=None, - export_config=None, - check_mode=False, - **kwargs): +def _get_input_fn( + data_config, + feature_configs, + data_path=None, + export_config=None, + check_mode=False, + **kwargs +): """Build estimator input function. Args: @@ -88,13 +78,14 @@ def _get_input_fn(data_config, task_id, task_num = estimator_utils.get_task_index_and_num() input_obj = input_class( - data_config, - feature_configs, - data_path, - task_index=task_id, - task_num=task_num, - check_mode=check_mode, - **kwargs) + data_config, + feature_configs, + data_path, + task_index=task_id, + task_num=task_num, + check_mode=check_mode, + **kwargs + ) input_fn = input_obj.create_input(export_config) return input_fn @@ -105,14 +96,16 @@ def _create_estimator(pipeline_config, distribution=None, params={}): gpu_options = GPUOptions(allow_growth=True) # False) logging.info( - 'train_config.train_distribute=%s[value=%d]' % - (DistributionStrategy.Name(pipeline_config.train_config.train_distribute), - pipeline_config.train_config.train_distribute)) + 'train_config.train_distribute=%s[value=%d]' % ( + DistributionStrategy.Name(pipeline_config.train_config.train_distribute + ), pipeline_config.train_config.train_distribute + ) + ) # set gpu options only under hvd scenes if hvd is not None and pipeline_config.train_config.train_distribute in [ - DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy, DistributionStrategy.HorovodStrategy + DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy, DistributionStrategy.HorovodStrategy ]: local_rnk = hvd.local_rank() gpus = tf.config.experimental.list_physical_devices('GPU') @@ -122,11 +115,12 @@ def _create_estimator(pipeline_config, distribution=None, params={}): gpu_options.visible_device_list = str(local_rnk) session_config = ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=params.get('log_device_placement', False), - inter_op_parallelism_threads=train_config.inter_op_parallelism_threads, - intra_op_parallelism_threads=train_config.intra_op_parallelism_threads) + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=params.get('log_device_placement', False), + inter_op_parallelism_threads=train_config.inter_op_parallelism_threads, + intra_op_parallelism_threads=train_config.intra_op_parallelism_threads + ) if constant.NO_ARITHMETRIC_OPTI in os.environ: logging.info('arithmetic_optimization is closed to improve performance') @@ -148,18 +142,20 @@ def _create_estimator(pipeline_config, distribution=None, params={}): save_checkpoints_steps = train_config.save_checkpoints_steps run_config = tf.estimator.RunConfig( - model_dir=pipeline_config.model_dir, - log_step_count_steps=None, # train_config.log_step_count_steps, - save_summary_steps=train_config.save_summary_steps, - save_checkpoints_steps=save_checkpoints_steps, - save_checkpoints_secs=save_checkpoints_secs, - keep_checkpoint_max=train_config.keep_checkpoint_max, - train_distribute=distribution, - eval_distribute=distribution, - session_config=session_config) + model_dir=pipeline_config.model_dir, + log_step_count_steps=None, # train_config.log_step_count_steps, + save_summary_steps=train_config.save_summary_steps, + save_checkpoints_steps=save_checkpoints_steps, + save_checkpoints_secs=save_checkpoints_secs, + keep_checkpoint_max=train_config.keep_checkpoint_max, + train_distribute=distribution, + eval_distribute=distribution, + session_config=session_config + ) estimator = EasyRecEstimator( - pipeline_config, model_cls, run_config=run_config, params=params) + pipeline_config, model_cls, run_config=run_config, params=params + ) return estimator, run_config @@ -171,7 +167,8 @@ def _create_eval_export_spec(pipeline_config, eval_data, check_mode=False): export_config = pipeline_config.export_config if eval_config.num_examples > 0: eval_steps = int( - math.ceil(float(eval_config.num_examples) / data_config.batch_size)) + math.ceil(float(eval_config.num_examples) / data_config.batch_size) + ) logging.info('eval_steps = %d' % eval_steps) else: eval_steps = None @@ -180,44 +177,54 @@ def _create_eval_export_spec(pipeline_config, eval_data, check_mode=False): input_fn_kwargs['fg_json_path'] = pipeline_config.fg_json_path # create eval input export_input_fn = _get_input_fn( - data_config, - feature_configs, - None, - export_config, - check_mode=check_mode, - **input_fn_kwargs) + data_config, + feature_configs, + None, + export_config, + check_mode=check_mode, + **input_fn_kwargs + ) if export_config.exporter_type == 'final': exporters = [ - FinalExporter(name='final', serving_input_receiver_fn=export_input_fn) + FinalExporter(name='final', serving_input_receiver_fn=export_input_fn) ] elif export_config.exporter_type == 'latest': exporters = [ - LatestExporter( - name='latest', - serving_input_receiver_fn=export_input_fn, - exports_to_keep=export_config.exports_to_keep) + LatestExporter( + name='latest', + serving_input_receiver_fn=export_input_fn, + exports_to_keep=export_config.exports_to_keep + ) ] elif export_config.exporter_type == 'best': logging.info( - 'will use BestExporter, metric is %s, the bigger the better: %d' % - (export_config.best_exporter_metric, export_config.metric_bigger)) + 'will use BestExporter, metric is %s, the bigger the better: %d' % + (export_config.best_exporter_metric, export_config.metric_bigger) + ) def _metric_cmp_fn(best_eval_result, current_eval_result): - logging.info('metric: best = %s current = %s' % - (str(best_eval_result), str(current_eval_result))) + logging.info( + 'metric: best = %s current = %s' % + (str(best_eval_result), str(current_eval_result)) + ) if export_config.metric_bigger: - return (best_eval_result[export_config.best_exporter_metric] < - current_eval_result[export_config.best_exporter_metric]) + return ( + best_eval_result[export_config.best_exporter_metric] + < current_eval_result[export_config.best_exporter_metric] + ) else: - return (best_eval_result[export_config.best_exporter_metric] > - current_eval_result[export_config.best_exporter_metric]) + return ( + best_eval_result[export_config.best_exporter_metric] + > current_eval_result[export_config.best_exporter_metric] + ) exporters = [ - BestExporter( - name='best', - serving_input_receiver_fn=export_input_fn, - compare_fn=_metric_cmp_fn, - exports_to_keep=export_config.exports_to_keep) + BestExporter( + name='best', + serving_input_receiver_fn=export_input_fn, + compare_fn=_metric_cmp_fn, + exports_to_keep=export_config.exports_to_keep + ) ] elif export_config.exporter_type == 'none': exporters = [] @@ -226,14 +233,16 @@ def _metric_cmp_fn(best_eval_result, current_eval_result): # set throttle_secs to a small number, so that we can control evaluation # interval steps by checkpoint saving steps - eval_input_fn = _get_input_fn(data_config, feature_configs, eval_data, - **input_fn_kwargs) + eval_input_fn = _get_input_fn( + data_config, feature_configs, eval_data, **input_fn_kwargs + ) eval_spec = tf.estimator.EvalSpec( - name='val', - input_fn=eval_input_fn, - steps=eval_steps, - throttle_secs=10, - exporters=exporters) + name='val', + input_fn=eval_input_fn, + steps=eval_steps, + throttle_secs=10, + exporters=exporters + ) return eval_spec @@ -261,9 +270,11 @@ def _get_ckpt_path(pipeline_config, checkpoint_path): ckpt_path = checkpoint_path elif tf.gfile.IsDirectory(pipeline_config.model_dir): ckpt_path = estimator_utils.latest_checkpoint(pipeline_config.model_dir) - logging.info('checkpoint_path is not specified, ' - 'will use latest checkpoint %s from %s' % - (ckpt_path, pipeline_config.model_dir)) + logging.info( + 'checkpoint_path is not specified, ' + 'will use latest checkpoint %s from %s' % + (ckpt_path, pipeline_config.model_dir) + ) else: assert False, 'pipeline_config.model_dir(%s) does not exist' \ % pipeline_config.model_dir @@ -284,20 +295,24 @@ def train_and_evaluate(pipeline_config_path, continue_train=False): None, the model will be saved into pipeline_config.model_dir """ assert tf.gfile.Exists( - pipeline_config_path), 'pipeline_config_path not exists' + pipeline_config_path + ), 'pipeline_config_path not exists' pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path) + pipeline_config_path + ) _train_and_evaluate_impl(pipeline_config, continue_train) return pipeline_config -def _train_and_evaluate_impl(pipeline_config, - continue_train=False, - check_mode=False, - fit_on_eval=False, - fit_on_eval_steps=None): +def _train_and_evaluate_impl( + pipeline_config, + continue_train=False, + check_mode=False, + fit_on_eval=False, + fit_on_eval_steps=None +): train_config = pipeline_config.train_config data_config = pipeline_config.data_config feature_configs = config_util.get_compatible_feature_configs(pipeline_config) @@ -305,8 +320,9 @@ def _train_and_evaluate_impl(pipeline_config, if train_config.train_distribute != DistributionStrategy.NoStrategy\ and train_config.sync_replicas: logging.warning( - 'will set sync_replicas to False, because train_distribute[%s] != NoStrategy' - % pipeline_config.train_config.train_distribute) + 'will set sync_replicas to False, because train_distribute[%s] != NoStrategy' + % pipeline_config.train_config.train_distribute + ) pipeline_config.train_config.sync_replicas = False train_data = get_train_input_path(pipeline_config) @@ -317,12 +333,15 @@ def _train_and_evaluate_impl(pipeline_config, if train_config.is_profiling: params['log_device_placement'] = True estimator, run_config = _create_estimator( - pipeline_config, distribution=distribution, params=params) + pipeline_config, distribution=distribution, params=params + ) version_file = os.path.join(pipeline_config.model_dir, 'version') if estimator_utils.is_chief(): _check_model_dir(pipeline_config.model_dir, continue_train) - config_util.save_pipeline_config(pipeline_config, pipeline_config.model_dir) + config_util.save_pipeline_config( + pipeline_config, pipeline_config.model_dir + ) with tf.gfile.GFile(version_file, 'w') as f: f.write(easy_rec.__version__ + '\n') @@ -330,7 +349,8 @@ def _train_and_evaluate_impl(pipeline_config, if train_config.HasField('num_steps') and train_config.num_steps > 0: train_steps = train_config.num_steps assert train_steps is not None or data_config.num_epochs > 0, ( - 'either num_steps and num_epochs must be set to an integer > 0.') + 'either num_steps and num_epochs must be set to an integer > 0.' + ) if train_steps and data_config.num_epochs: logging.info('Both num_steps and num_epochs are set.') @@ -348,62 +368,73 @@ def _train_and_evaluate_impl(pipeline_config, # create train input train_input_fn = _get_input_fn( - data_config, - feature_configs, - train_data, - check_mode=check_mode, - **input_fn_kwargs) + data_config, + feature_configs, + train_data, + check_mode=check_mode, + **input_fn_kwargs + ) # Currently only a single Eval Spec is allowed. train_spec = tf.estimator.TrainSpec( - input_fn=train_input_fn, max_steps=train_steps) + input_fn=train_input_fn, max_steps=train_steps + ) embedding_parallel = train_config.train_distribute in ( - DistributionStrategy.SokStrategy, - DistributionStrategy.EmbeddingParallelStrategy) + DistributionStrategy.SokStrategy, + DistributionStrategy.EmbeddingParallelStrategy + ) if embedding_parallel: estimator.train( - input_fn=train_input_fn, - max_steps=train_spec.max_steps, - hooks=list(train_spec.hooks), - saving_listeners=train_spec.saving_listeners) + input_fn=train_input_fn, + max_steps=train_spec.max_steps, + hooks=list(train_spec.hooks), + saving_listeners=train_spec.saving_listeners + ) train_input_fn.input_creator.stop() else: # create eval spec eval_spec = _create_eval_export_spec( - pipeline_config, eval_data, check_mode=check_mode) + pipeline_config, eval_data, check_mode=check_mode + ) estimator_train.train_and_evaluate(estimator, train_spec, eval_spec) logging.info('Train and evaluate finish') if fit_on_eval and (not estimator_utils.is_evaluator()): tf.reset_default_graph() logging.info('Start continue training on eval data') - eval_input_fn = _get_input_fn(data_config, feature_configs, eval_data, - **input_fn_kwargs) + eval_input_fn = _get_input_fn( + data_config, feature_configs, eval_data, **input_fn_kwargs + ) if fit_on_eval_steps is not None: # wait estimator train done to get the correct train_steps while not estimator_train.estimator_train_done(estimator): time.sleep(1) train_steps = estimator_utils.get_trained_steps(estimator.model_dir) - logging.info('\ttrain_steps=%d fit_on_eval_steps=%d' % - (train_steps, fit_on_eval_steps)) + logging.info( + '\ttrain_steps=%d fit_on_eval_steps=%d' % + (train_steps, fit_on_eval_steps) + ) fit_on_eval_steps += train_steps # Do not use estimator_train.train_and_evaluate as it starts tf.Server, # which is redundant and reports port not available error. estimator.train( - input_fn=eval_input_fn, - max_steps=fit_on_eval_steps, - hooks=list(train_spec.hooks), - saving_listeners=train_spec.saving_listeners if hasattr( - train_spec, 'saving_listeners') else None) + input_fn=eval_input_fn, + max_steps=fit_on_eval_steps, + hooks=list(train_spec.hooks), + saving_listeners=train_spec.saving_listeners + if hasattr(train_spec, 'saving_listeners') else None + ) logging.info('Finished training on eval data') # return estimator for custom training using estimator.train return estimator -def evaluate(pipeline_config, - eval_checkpoint_path='', - eval_data_path=None, - eval_result_filename='eval_result.txt'): +def evaluate( + pipeline_config, + eval_checkpoint_path='', + eval_data_path=None, + eval_result_filename='eval_result.txt' +): """Evaluate a EasyRec model defined in pipeline_config_path. Evaluate the model defined in pipeline_config_path on the eval data, @@ -443,7 +474,8 @@ def evaluate(pipeline_config, if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) server = server_lib.Server( - cluster, job_name='ps', task_index=tf_config['task']['index']) + cluster, job_name='ps', task_index=tf_config['task']['index'] + ) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: @@ -459,33 +491,39 @@ def evaluate(pipeline_config, if server_target: # evaluate with parameter server - input_iter = eval_spec.input_fn( - mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() + input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL + ).make_one_shot_iterator() input_feas, input_lbls = input_iter.get_next() - from tensorflow.python.training.device_setter import replica_device_setter from tensorflow.python.framework.ops import device + from tensorflow.python.training.device_setter import replica_device_setter + from tensorflow.python.training.monitored_session import ChiefSessionCreator # NOQA from tensorflow.python.training.monitored_session import MonitoredSession - from tensorflow.python.training.monitored_session import ChiefSessionCreator with device( - replica_device_setter( - worker_device='/job:master/task:0', cluster=cluster)): - estimator_spec = estimator._eval_model_fn(input_feas, input_lbls, - run_config) + replica_device_setter( + worker_device='/job:master/task:0', cluster=cluster + ) + ): + estimator_spec = estimator._eval_model_fn( + input_feas, input_lbls, run_config + ) session_config = ConfigProto( - allow_soft_placement=True, log_device_placement=True) + allow_soft_placement=True, log_device_placement=True + ) chief_sess_creator = ChiefSessionCreator( - master=server_target, - checkpoint_filename_with_path=ckpt_path, - config=session_config) + master=server_target, + checkpoint_filename_with_path=ckpt_path, + config=session_config + ) eval_metric_ops = estimator_spec.eval_metric_ops update_ops = [eval_metric_ops[x][1] for x in eval_metric_ops.keys()] metric_ops = {x: eval_metric_ops[x][0] for x in eval_metric_ops.keys()} update_op = tf.group(update_ops) with MonitoredSession( - session_creator=chief_sess_creator, - hooks=None, - stop_grace_period_secs=120) as sess: + session_creator=chief_sess_creator, + hooks=None, + stop_grace_period_secs=120 + ) as sess: while True: try: sess.run(update_op) @@ -499,7 +537,8 @@ def evaluate(pipeline_config, # replica_device_setter( # worker_device='/job:master/task:0', cluster=cluster)): eval_result = estimator.evaluate( - eval_spec.input_fn, eval_spec.steps, checkpoint_path=ckpt_path) + eval_spec.input_fn, eval_spec.steps, checkpoint_path=ckpt_path + ) eval_spec.input_fn.input_creator.stop() logging.info('Evaluate finish') @@ -521,10 +560,12 @@ def evaluate(pipeline_config, return eval_result -def distribute_evaluate(pipeline_config, - eval_checkpoint_path='', - eval_data_path=None, - eval_result_filename='distribute_eval_result.txt'): +def distribute_evaluate( + pipeline_config, + eval_checkpoint_path='', + eval_data_path=None, + eval_result_filename='distribute_eval_result.txt' +): """Evaluate a EasyRec model defined in pipeline_config_path. Evaluate the model defined in pipeline_config_path on the eval data, @@ -556,7 +597,7 @@ def distribute_evaluate(pipeline_config, data_config = pipeline_config.data_config if data_config.HasField('sampler'): logging.warning( - 'It is not accuracy to use eval with negative sampler, recommand to use hitrate.py!' + 'It is not accuracy to use eval with negative sampler, recommand to use hitrate.py!' ) eval_result = {} return eval_result @@ -566,7 +607,8 @@ def distribute_evaluate(pipeline_config, logging.info('create eval tmp results dir {}'.format(eval_tmp_results_dir)) tf.gfile.MakeDirs(eval_tmp_results_dir) assert tf.gfile.IsDirectory( - eval_tmp_results_dir), 'tmp results dir not create success.' + eval_tmp_results_dir + ), 'tmp results dir not create success.' os.environ['eval_tmp_results_dir'] = eval_tmp_results_dir server_target = None @@ -578,7 +620,8 @@ def distribute_evaluate(pipeline_config, if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) server = server_lib.Server( - cluster, job_name='ps', task_index=tf_config['task']['index']) + cluster, job_name='ps', task_index=tf_config['task']['index'] + ) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: @@ -586,7 +629,8 @@ def distribute_evaluate(pipeline_config, cur_task_index = tf_config['task']['index'] cluster = tf.train.ClusterSpec(tf_config['cluster']) server = server_lib.Server( - cluster, job_name=cur_job_name, task_index=cur_task_index) + cluster, job_name=cur_job_name, task_index=cur_task_index + ) server_target = server.target print('server_target = %s' % server_target) elif tf_config['task']['type'] == 'worker': @@ -595,57 +639,67 @@ def distribute_evaluate(pipeline_config, cur_task_index = tf_config['task']['index'] cluster = tf.train.ClusterSpec(tf_config['cluster']) server = server_lib.Server( - cluster, job_name=cur_job_name, task_index=cur_task_index) + cluster, job_name=cur_job_name, task_index=cur_task_index + ) server_target = server.target print('server_target = %s' % server_target) if server_target: - from tensorflow.python.training.device_setter import replica_device_setter from tensorflow.python.framework.ops import device + from tensorflow.python.training.device_setter import replica_device_setter + from tensorflow.python.training.monitored_session import ChiefSessionCreator # NOQA + from tensorflow.python.training.monitored_session import WorkerSessionCreator # NOQA from tensorflow.python.training.monitored_session import MonitoredSession - from tensorflow.python.training.monitored_session import ChiefSessionCreator - from tensorflow.python.training.monitored_session import WorkerSessionCreator + from easy_rec.python.utils.estimator_utils import EvaluateExitBarrierHook cur_work_device = '/job:' + cur_job_name + '/task:' + str(cur_task_index) cur_ps_num = len(tf_config['cluster']['ps']) with device( - replica_device_setter( - ps_tasks=cur_ps_num, worker_device=cur_work_device, - cluster=cluster)): + replica_device_setter( + ps_tasks=cur_ps_num, worker_device=cur_work_device, cluster=cluster + ) + ): distribution = strategy_builder.build(train_config) estimator, run_config = _create_estimator(pipeline_config, distribution) eval_spec = _create_eval_export_spec(pipeline_config, eval_data) ckpt_path = _get_ckpt_path(pipeline_config, eval_checkpoint_path) ckpt_dir = os.path.dirname(ckpt_path) - input_iter = eval_spec.input_fn( - mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() + input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL + ).make_one_shot_iterator() input_feas, input_lbls = input_iter.get_next() estimator_spec = estimator._distribute_eval_model_fn( - input_feas, input_lbls, run_config) + input_feas, input_lbls, run_config + ) session_config = ConfigProto( - allow_soft_placement=True, - log_device_placement=True, - device_filters=['/job:ps', - '/job:worker/task:%d' % cur_task_index]) + allow_soft_placement=True, + log_device_placement=True, + device_filters=['/job:ps', + '/job:worker/task:%d' % cur_task_index] + ) if cur_job_name == 'master': metric_variables = tf.get_collection(tf.GraphKeys.METRIC_VARIABLES) - model_ready_for_local_init_op = tf.variables_initializer(metric_variables) + model_ready_for_local_init_op = tf.variables_initializer( + metric_variables + ) global_variables = tf.global_variables() remain_variables = list( - set(global_variables).difference(set(metric_variables))) + set(global_variables).difference(set(metric_variables)) + ) cur_saver = tf.train.Saver(var_list=remain_variables, sharded=True) cur_scaffold = tf.train.Scaffold( - saver=cur_saver, - ready_for_local_init_op=model_ready_for_local_init_op) + saver=cur_saver, ready_for_local_init_op=model_ready_for_local_init_op + ) cur_sess_creator = ChiefSessionCreator( - scaffold=cur_scaffold, - master=server_target, - checkpoint_filename_with_path=ckpt_path, - config=session_config) + scaffold=cur_scaffold, + master=server_target, + checkpoint_filename_with_path=ckpt_path, + config=session_config + ) else: cur_sess_creator = WorkerSessionCreator( - master=server_target, config=session_config) + master=server_target, config=session_config + ) eval_metric_ops = estimator_spec.eval_metric_ops update_ops = [eval_metric_ops[x][1] for x in eval_metric_ops.keys()] metric_ops = {x: eval_metric_ops[x][0] for x in eval_metric_ops.keys()} @@ -653,16 +707,19 @@ def distribute_evaluate(pipeline_config, cur_worker_num = len(tf_config['cluster']['worker']) + 1 if cur_job_name == 'master': cur_stop_grace_period_sesc = 120 - cur_hooks = EvaluateExitBarrierHook(cur_worker_num, True, ckpt_dir, - metric_ops) + cur_hooks = EvaluateExitBarrierHook( + cur_worker_num, True, ckpt_dir, metric_ops + ) else: cur_stop_grace_period_sesc = 10 - cur_hooks = EvaluateExitBarrierHook(cur_worker_num, False, ckpt_dir, - metric_ops) + cur_hooks = EvaluateExitBarrierHook( + cur_worker_num, False, ckpt_dir, metric_ops + ) with MonitoredSession( - session_creator=cur_sess_creator, - hooks=[cur_hooks], - stop_grace_period_secs=cur_stop_grace_period_sesc) as sess: + session_creator=cur_sess_creator, + hooks=[cur_hooks], + stop_grace_period_secs=cur_stop_grace_period_sesc + ) as sess: while True: try: sess.run(update_op) @@ -726,17 +783,21 @@ def predict(pipeline_config, checkpoint_path='', data_path=None): ckpt_path = _get_ckpt_path(pipeline_config, checkpoint_path) - pred_result = estimator.predict(eval_spec.input_fn, checkpoint_path=ckpt_path) + pred_result = estimator.predict( + eval_spec.input_fn, checkpoint_path=ckpt_path + ) logging.info('Predict finish') return pred_result -def export(export_dir, - pipeline_config, - checkpoint_path='', - asset_files=None, - verbose=False, - **extra_params): +def export( + export_dir, + pipeline_config, + checkpoint_path='', + asset_files=None, + verbose=False, + **extra_params +): """Export model defined in pipeline_config_path. Args: @@ -782,7 +843,8 @@ def export(export_dir, for asset_file in asset_files.split(','): asset_file = asset_file.strip() if ':' not in asset_file or asset_file.startswith( - 'oss:') or asset_file.startswith('hdfs:'): + 'oss:' + ) or asset_file.startswith('hdfs:'): _, asset_name = os.path.split(asset_file) else: asset_name, asset_file = asset_file.split(':', 1) @@ -795,8 +857,9 @@ def export(export_dir, input_fn_kwargs = {'pipeline_config': pipeline_config} if data_config.input_type == data_config.InputType.OdpsRTPInputV2: input_fn_kwargs['fg_json_path'] = pipeline_config.fg_json_path - serving_input_fn = _get_input_fn(data_config, feature_configs, None, - export_config, **input_fn_kwargs) + serving_input_fn = _get_input_fn( + data_config, feature_configs, None, export_config, **input_fn_kwargs + ) ckpt_path = _get_ckpt_path(pipeline_config, checkpoint_path) if 'oss_path' in extra_params: if pipeline_config.train_config.HasField('incr_save_config'): @@ -806,27 +869,32 @@ def export(export_dir, logging.info('incr_save_type=%s' % incr_save_type) if incr_save_type: extra_params['incr_update'][incr_save_type] = getattr( - incr_save_config, incr_save_type) - return export_big_model_to_oss(export_dir, pipeline_config, extra_params, - serving_input_fn, estimator, ckpt_path, - verbose) + incr_save_config, incr_save_type + ) + return export_big_model_to_oss( + export_dir, pipeline_config, extra_params, serving_input_fn, estimator, + ckpt_path, verbose + ) if 'redis_url' in extra_params: - return export_big_model(export_dir, pipeline_config, extra_params, - serving_input_fn, estimator, ckpt_path, verbose) + return export_big_model( + export_dir, pipeline_config, extra_params, serving_input_fn, estimator, + ckpt_path, verbose + ) final_export_dir = estimator.export_savedmodel( - export_dir_base=export_dir, - serving_input_receiver_fn=serving_input_fn, - checkpoint_path=ckpt_path, - strip_default_attrs=True) + export_dir_base=export_dir, + serving_input_receiver_fn=serving_input_fn, + checkpoint_path=ckpt_path, + strip_default_attrs=True + ) # add export ts as version info saved_model = saved_model_pb2.SavedModel() if type(final_export_dir) not in [type(''), type(u'')]: final_export_dir = final_export_dir.decode('utf-8') export_ts = [ - x for x in final_export_dir.split('/') if x != '' and x is not None + x for x in final_export_dir.split('/') if x != '' and x is not None ] export_ts = export_ts[-1] saved_pb_path = os.path.join(final_export_dir, 'saved_model.pb') @@ -856,12 +924,14 @@ def export(export_dir, return final_export_dir -def export_checkpoint(pipeline_config=None, - export_path='', - checkpoint_path='', - asset_files=None, - verbose=False, - mode=tf.estimator.ModeKeys.PREDICT): +def export_checkpoint( + pipeline_config=None, + export_path='', + checkpoint_path='', + asset_files=None, + verbose=False, + mode=tf.estimator.ModeKeys.PREDICT +): """Export the EasyRec model as checkpoint.""" pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config) if pipeline_config.fg_json_path: @@ -882,13 +952,15 @@ def export_checkpoint(pipeline_config=None, # construct serving input fn export_config = pipeline_config.export_config - serving_input_fn = _get_input_fn(data_config, feature_configs, None, - export_config, **input_fn_kwargs) + serving_input_fn = _get_input_fn( + data_config, feature_configs, None, export_config, **input_fn_kwargs + ) ckpt_path = _get_ckpt_path(pipeline_config, checkpoint_path) estimator.export_checkpoint( - export_path=export_path, - serving_input_receiver_fn=serving_input_fn, - checkpoint_path=ckpt_path, - mode=mode) + export_path=export_path, + serving_input_receiver_fn=serving_input_fn, + checkpoint_path=ckpt_path, + mode=mode + ) logging.info('model checkpoint has been exported successfully') diff --git a/easy_rec/python/model/autoint.py b/easy_rec/python/model/autoint.py index b7013486e..bab8565d5 100644 --- a/easy_rec/python/model/autoint.py +++ b/easy_rec/python/model/autoint.py @@ -1,12 +1,10 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.layers import multihead_attention from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.autoint_pb2 import AutoInt as AutoIntConfig # NOQA if tf.__version__ >= '2.0': @@ -15,14 +13,17 @@ class AutoInt(RankModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(AutoInt, self).__init__(model_config, feature_configs, features, - labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(AutoInt, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'autoint', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._features, _ = self._input_layer(self._feature_dict, 'all') @@ -50,21 +51,24 @@ def build_predict_graph(self): logging.info('feature_num: {0}'.format(self._feature_num)) attention_fea = tf.reshape( - self._features, - shape=[-1, self._feature_num + self._seq_key_num, self._d_model]) + self._features, + shape=[-1, self._feature_num + self._seq_key_num, self._d_model] + ) for i in range(self._model_config.interacting_layer_num): attention_layer = multihead_attention.MultiHeadAttention( - head_num=self._head_num, - head_size=self._head_size, - l2_reg=self._l2_reg, - use_res=True, - name='multi_head_self_attention_layer_%d' % i) + head_num=self._head_num, + head_size=self._head_size, + l2_reg=self._l2_reg, + use_res=True, + name='multi_head_self_attention_layer_%d' % i + ) attention_fea = attention_layer(attention_fea) attention_fea = tf.reshape( - attention_fea, - shape=[-1, attention_fea.shape[1] * attention_fea.shape[2]]) + attention_fea, + shape=[-1, attention_fea.shape[1] * attention_fea.shape[2]] + ) final = tf.layers.dense(attention_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/cmbf.py b/easy_rec/python/model/cmbf.py index 0f0a8f3aa..32bf596e5 100644 --- a/easy_rec/python/model/cmbf.py +++ b/easy_rec/python/model/cmbf.py @@ -2,10 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import cmbf -from easy_rec.python.layers import dnn +from easy_rec.python.layers import cmbf, dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.cmbf_pb2 import CMBF as CMBFConfig # NOQA if tf.__version__ >= '2.0': @@ -20,26 +18,33 @@ class CMBF(RankModel): https://www.mdpi.com/1424-8220/21/16/5275 """ - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(CMBF, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(CMBF, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'cmbf', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model')) + 'invalid model config: %s' % self._model_config.WhichOneof('model') + ) - self._cmbf_layer = cmbf.CMBF(model_config, feature_configs, features, - self._model_config.cmbf.config, - self._input_layer) + self._cmbf_layer = cmbf.CMBF( + model_config, feature_configs, features, self._model_config.cmbf.config, + self._input_layer + ) self._model_config = self._model_config.cmbf def build_predict_graph(self): hidden = self._cmbf_layer(self._is_training, l2_reg=self._l2_reg) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, - 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN( + self._model_config.final_dnn, self._l2_reg, 'final_dnn', + self._is_training + ) all_fea = final_dnn_layer(hidden) final = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/collaborative_metric_learning.py b/easy_rec/python/model/collaborative_metric_learning.py index b19537239..dde65a615 100644 --- a/easy_rec/python/model/collaborative_metric_learning.py +++ b/easy_rec/python/model/collaborative_metric_learning.py @@ -1,18 +1,16 @@ import tensorflow as tf -from easy_rec.python.core.metrics import metric_learning_average_precision_at_k -from easy_rec.python.core.metrics import metric_learning_recall_at_k +from easy_rec.python.core.metrics import metric_learning_average_precision_at_k, metric_learning_recall_at_k # NOQA from easy_rec.python.layers import dnn from easy_rec.python.layers.common_layers import highway from easy_rec.python.loss.circle_loss import circle_loss from easy_rec.python.loss.multi_similarity import ms_loss from easy_rec.python.model.easy_rec_model import EasyRecModel +from easy_rec.python.protos.collaborative_metric_learning_pb2 import CoMetricLearningI2I as MetricLearningI2IConfig # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.utils.activation import gelu from easy_rec.python.utils.proto_util import copy_obj -from easy_rec.python.protos.collaborative_metric_learning_pb2 import CoMetricLearningI2I as MetricLearningI2IConfig # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -20,14 +18,16 @@ class CoMetricLearningI2I(EasyRecModel): def __init__( - self, - model_config, # pipeline.model_config - feature_configs, # pipeline.feature_configs - features, # same as model_fn input - labels=None, - is_training=False): - super(CoMetricLearningI2I, self).__init__(model_config, feature_configs, - features, labels, is_training) + self, + model_config, # pipeline.model_config + feature_configs, # pipeline.feature_configs + features, # same as model_fn input + labels=None, + is_training=False + ): + super(CoMetricLearningI2I, self).__init__( + model_config, feature_configs, features, labels, is_training + ) model = self._model_config.WhichOneof('model') assert model == 'metric_learning', 'invalid model config: %s' % model @@ -45,22 +45,25 @@ def __init__( elif self._loss_type == LossType.MULTI_SIMILARITY_LOSS: self.loss = self._model_config.multi_similarity_loss else: - raise ValueError('unsupported loss type: %s' % - LossType.Name(self._loss_type)) + raise ValueError( + 'unsupported loss type: %s' % LossType.Name(self._loss_type) + ) if not self.has_backbone: self._highway_features = {} self._highway_num = len(self._model_config.highway) for _id in range(self._highway_num): highway_cfg = self._model_config.highway[_id] - highway_feature, _ = self._input_layer(self._feature_dict, - highway_cfg.input) + highway_feature, _ = self._input_layer( + self._feature_dict, highway_cfg.input + ) self._highway_features[highway_cfg.input] = highway_feature self.input_features = [] if self._model_config.HasField('input'): - input_feature, _ = self._input_layer(self._feature_dict, - self._model_config.input) + input_feature, _ = self._input_layer( + self._feature_dict, self._model_config.input + ) self.input_features.append(input_feature) self.dnn = copy_obj(self._model_config.dnn) @@ -86,15 +89,17 @@ def build_predict_graph(self): for _id in range(self._highway_num): highway_cfg = self._model_config.highway[_id] highway_fea = tf.layers.batch_normalization( - self._highway_features[highway_cfg.input], - training=self._is_training, - trainable=True, - name='highway_%s_bn' % highway_cfg.input) + self._highway_features[highway_cfg.input], + training=self._is_training, + trainable=True, + name='highway_%s_bn' % highway_cfg.input + ) highway_fea = highway( - highway_fea, - highway_cfg.emb_size, - activation=gelu, - scope='highway_%s' % _id) + highway_fea, + highway_cfg.emb_size, + activation=gelu, + scope='highway_%s' % _id + ) print('highway_fea: ', highway_fea) self.input_features.append(highway_fea) @@ -105,23 +110,27 @@ def build_predict_graph(self): dnn_net = dnn.DNN(self.dnn, self._l2_reg, 'dnn', self._is_training) net_output = dnn_net(feature) tower_emb = tf.layers.dense( - inputs=net_output, - units=last_hidden, - kernel_regularizer=self._l2_reg, - name='dnn/dnn_%d' % (num_dnn_layer - 1)) + inputs=net_output, + units=last_hidden, + kernel_regularizer=self._l2_reg, + name='dnn/dnn_%d' % (num_dnn_layer - 1) + ) if self._model_config.output_l2_normalized_emb: norm_emb = tf.nn.l2_normalize(tower_emb, axis=-1) self._prediction_dict['norm_emb'] = norm_emb self._prediction_dict['norm_embedding'] = tf.reduce_join( - tf.as_string(norm_emb), axis=-1, separator=',') + tf.as_string(norm_emb), axis=-1, separator=',' + ) self._prediction_dict['float_emb'] = tower_emb self._prediction_dict['embedding'] = tf.reduce_join( - tf.as_string(tower_emb), axis=-1, separator=',') + tf.as_string(tower_emb), axis=-1, separator=',' + ) if self.sample_id is not None and self.sample_id in self._feature_dict: self._prediction_dict['sample_id'] = tf.identity( - self._feature_dict[self.sample_id]) + self._feature_dict[self.sample_id] + ) return self._prediction_dict def build_loss_graph(self): @@ -130,24 +139,28 @@ def build_loss_graph(self): norm_emb = self._prediction_dict['norm_emb'] if emb_normed else emb if self._loss_type == LossType.CIRCLE_LOSS: self._loss_dict['circle_loss'] = circle_loss( - norm_emb, - self.labels, - self.session_ids, - self.loss.margin, - self.loss.gamma, - embed_normed=emb_normed) + norm_emb, + self.labels, + self.session_ids, + self.loss.margin, + self.loss.gamma, + embed_normed=emb_normed + ) elif self._loss_type == LossType.MULTI_SIMILARITY_LOSS: self._loss_dict['ms_loss'] = ms_loss( - norm_emb, - self.labels, - self.session_ids, - self.loss.alpha, - self.loss.beta, - self.loss.lamb, - self.loss.eps, - embed_normed=emb_normed) + norm_emb, + self.labels, + self.session_ids, + self.loss.alpha, + self.loss.beta, + self.loss.lamb, + self.loss.eps, + embed_normed=emb_normed + ) else: - raise ValueError('invalid loss type: %s' % LossType.Name(self._loss_type)) + raise ValueError( + 'invalid loss type: %s' % LossType.Name(self._loss_type) + ) return self._loss_dict @@ -173,10 +186,14 @@ def build_metric_graph(self, eval_config): emb = self._prediction_dict['float_emb'] if len(recall_at_k) > 0: metric_dict.update( - metric_learning_recall_at_k(recall_at_k, emb, self.labels, - self.session_ids)) + metric_learning_recall_at_k( + recall_at_k, emb, self.labels, self.session_ids + ) + ) if len(precision_at_k) > 0: metric_dict.update( - metric_learning_average_precision_at_k(precision_at_k, emb, - self.labels, self.session_ids)) + metric_learning_average_precision_at_k( + precision_at_k, emb, self.labels, self.session_ids + ) + ) return metric_dict diff --git a/easy_rec/python/model/dat.py b/easy_rec/python/model/dat.py index 5c312299c..9b86387e1 100644 --- a/easy_rec/python/model/dat.py +++ b/easy_rec/python/model/dat.py @@ -15,19 +15,22 @@ class DAT(MatchModel): """Dual Augmented Two-tower Model.""" - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(DAT, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(DAT, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'dat', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') feature_group_names = [ - fg.group_name for fg in self._model_config.feature_groups + fg.group_name for fg in self._model_config.feature_groups ] assert 'user' in feature_group_names, 'user feature group not found' assert 'item' in feature_group_names, 'item feature group not found' @@ -39,13 +42,15 @@ def __init__(self, self.user_tower = copy_obj(self._model_config.user_tower) self.user_deep_feature, _ = self._input_layer(self._feature_dict, 'user') - self.user_augmented_vec, _ = self._input_layer(self._feature_dict, - 'user_id_augment') + self.user_augmented_vec, _ = self._input_layer( + self._feature_dict, 'user_id_augment' + ) self.item_tower = copy_obj(self._model_config.item_tower) self.item_deep_feature, _ = self._input_layer(self._feature_dict, 'item') - self.item_augmented_vec, _ = self._input_layer(self._feature_dict, - 'item_id_augment') + self.item_augmented_vec, _ = self._input_layer( + self._feature_dict, 'item_id_augment' + ) self._user_tower_emb = None self._item_tower_emb = None @@ -53,31 +58,37 @@ def __init__(self, def build_predict_graph(self): num_user_dnn_layer = len(self.user_tower.dnn.hidden_units) last_user_hidden = self.user_tower.dnn.hidden_units.pop() - user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', - self._is_training) + user_dnn = dnn.DNN( + self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training + ) user_tower_feature = tf.concat( - [self.user_deep_feature, self.user_augmented_vec], axis=-1) + [self.user_deep_feature, self.user_augmented_vec], axis=-1 + ) user_tower_emb = user_dnn(user_tower_feature) user_tower_emb = tf.layers.dense( - inputs=user_tower_emb, - units=last_user_hidden, - kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1)) + inputs=user_tower_emb, + units=last_user_hidden, + kernel_regularizer=self._l2_reg, + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1) + ) num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() - item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', - self._is_training) + item_dnn = dnn.DNN( + self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training + ) item_tower_feature = tf.concat( - [self.item_deep_feature, self.item_augmented_vec], axis=-1) + [self.item_deep_feature, self.item_augmented_vec], axis=-1 + ) item_tower_emb = item_dnn(item_tower_feature) item_tower_emb = tf.layers.dense( - inputs=item_tower_emb, - units=last_item_hidden, - kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1)) + inputs=item_tower_emb, + units=last_item_hidden, + kernel_regularizer=self._l2_reg, + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1) + ) user_tower_emb = self.norm(user_tower_emb) item_tower_emb = self.norm(item_tower_emb) @@ -90,7 +101,8 @@ def build_predict_graph(self): if self._loss_type == LossType.CLASSIFICATION: raise ValueError( - 'Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.') + 'Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.' + ) elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: y_pred = self._mask_in_batch(y_pred) self._prediction_dict['logits'] = y_pred @@ -101,9 +113,11 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb self._prediction_dict['user_emb'] = tf.reduce_join( - tf.as_string(user_tower_emb), axis=-1, separator=',') + tf.as_string(user_tower_emb), axis=-1, separator=',' + ) self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_tower_emb), axis=-1, separator=',') + tf.as_string(item_tower_emb), axis=-1, separator=',' + ) augmented_p_u = tf.stop_gradient(user_tower_emb) augmented_p_i = tf.stop_gradient(item_tower_emb) @@ -119,16 +133,19 @@ def build_predict_graph(self): def get_outputs(self): if self._loss_type == LossType.CLASSIFICATION: raise ValueError( - 'Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.') + 'Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.' + ) elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: self._prediction_dict['logits'] = tf.squeeze( - self._prediction_dict['logits'], axis=-1) + self._prediction_dict['logits'], axis=-1 + ) self._prediction_dict['probs'] = tf.nn.sigmoid( - self._prediction_dict['logits']) + self._prediction_dict['logits'] + ) return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb', 'augmented_p_u', 'augmented_p_i', 'augmented_a_u', - 'augmented_a_i' + 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', + 'item_tower_emb', 'augmented_p_u', 'augmented_p_i', 'augmented_a_u', + 'augmented_a_i' ] else: raise ValueError('invalid loss type: %s' % str(self._loss_type)) diff --git a/easy_rec/python/model/dbmtl.py b/easy_rec/python/model/dbmtl.py index 6c69d33ca..261799e36 100644 --- a/easy_rec/python/model/dbmtl.py +++ b/easy_rec/python/model/dbmtl.py @@ -2,10 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import cmbf -from easy_rec.python.layers import dnn -from easy_rec.python.layers import mmoe -from easy_rec.python.layers import uniter +from easy_rec.python.layers import cmbf, dnn, mmoe, uniter from easy_rec.python.model.multi_task_model import MultiTaskModel from easy_rec.python.protos.dbmtl_pb2 import DBMTL as DBMTLConfig @@ -15,31 +12,36 @@ class DBMTL(MultiTaskModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(DBMTL, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(DBMTL, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'dbmtl', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.dbmtl assert isinstance(self._model_config, DBMTLConfig) if self._model_config.HasField('bottom_cmbf'): - self._cmbf_layer = cmbf.CMBF(model_config, feature_configs, features, - self._model_config.bottom_cmbf, - self._input_layer) + self._cmbf_layer = cmbf.CMBF( + model_config, feature_configs, features, + self._model_config.bottom_cmbf, self._input_layer + ) elif self._model_config.HasField('bottom_uniter'): - self._uniter_layer = uniter.Uniter(model_config, feature_configs, - features, - self._model_config.bottom_uniter, - self._input_layer) + self._uniter_layer = uniter.Uniter( + model_config, feature_configs, features, + self._model_config.bottom_uniter, self._input_layer + ) elif not self.has_backbone: self._features, self._feature_list = self._input_layer( - self._feature_dict, 'all') + self._feature_dict, 'all' + ) else: assert False, 'invalid code branch' self._init_towers(self._model_config.task_towers) @@ -53,10 +55,11 @@ def build_predict_graph(self): bottom_fea = self._uniter_layer(self._is_training, l2_reg=self._l2_reg) elif self._model_config.HasField('bottom_dnn'): bottom_dnn = dnn.DNN( - self._model_config.bottom_dnn, - self._l2_reg, - name='bottom_dnn', - is_training=self._is_training) + self._model_config.bottom_dnn, + self._l2_reg, + name='bottom_dnn', + is_training=self._is_training + ) bottom_fea = bottom_dnn(self._features) else: bottom_fea = self._features @@ -64,10 +67,11 @@ def build_predict_graph(self): # MMOE block if self._model_config.HasField('expert_dnn'): mmoe_layer = mmoe.MMOE( - self._model_config.expert_dnn, - l2_reg=self._l2_reg, - num_task=self._task_num, - num_expert=self._model_config.num_expert) + self._model_config.expert_dnn, + l2_reg=self._l2_reg, + num_task=self._task_num, + num_expert=self._model_config.num_expert + ) task_input_list = mmoe_layer(bottom_fea) else: task_input_list = [bottom_fea] * self._task_num @@ -78,10 +82,11 @@ def build_predict_graph(self): tower_name = task_tower_cfg.tower_name if task_tower_cfg.HasField('dnn'): tower_dnn = dnn.DNN( - task_tower_cfg.dnn, - self._l2_reg, - name=tower_name + '/dnn', - is_training=self._is_training) + task_tower_cfg.dnn, + self._l2_reg, + name=tower_name + '/dnn', + is_training=self._is_training + ) tower_fea = tower_dnn(task_input_list[i]) tower_features[tower_name] = tower_fea else: @@ -93,23 +98,26 @@ def build_predict_graph(self): for task_tower_cfg in self._model_config.task_towers: tower_name = task_tower_cfg.tower_name relation_dnn = dnn.DNN( - task_tower_cfg.relation_dnn, - self._l2_reg, - name=tower_name + '/relation_dnn', - is_training=self._is_training) + task_tower_cfg.relation_dnn, + self._l2_reg, + name=tower_name + '/relation_dnn', + is_training=self._is_training + ) tower_inputs = [tower_features[tower_name]] for relation_tower_name in task_tower_cfg.relation_tower_names: tower_inputs.append(relation_features[relation_tower_name]) relation_input = tf.concat( - tower_inputs, axis=-1, name=tower_name + '/relation_input') + tower_inputs, axis=-1, name=tower_name + '/relation_input' + ) relation_fea = relation_dnn(relation_input) relation_features[tower_name] = relation_fea output_logits = tf.layers.dense( - relation_fea, - task_tower_cfg.num_class, - kernel_regularizer=self._l2_reg, - name=tower_name + '/output') + relation_fea, + task_tower_cfg.num_class, + kernel_regularizer=self._l2_reg, + name=tower_name + '/output' + ) tower_outputs[tower_name] = output_logits self._add_to_prediction_dict(tower_outputs) diff --git a/easy_rec/python/model/dcn.py b/easy_rec/python/model/dcn.py index fcfa7e780..d172b5c4c 100644 --- a/easy_rec/python/model/dcn.py +++ b/easy_rec/python/model/dcn.py @@ -5,7 +5,6 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.dcn_pb2 import DCN as DCNConfig # NOQA if tf.__version__ >= '2.0': @@ -14,14 +13,17 @@ class DCN(RankModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(DCN, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(DCN, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'dcn', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.dcn @@ -35,11 +37,13 @@ def _cross_net(self, tensor, num_cross_layers): for i in range(num_cross_layers): name = 'cross_layer_%s' % i w = tf.get_variable( - name=name + '_w', - dtype=tf.float32, - shape=(input_dim), + name=name + '_w', + dtype=tf.float32, + shape=(input_dim), + ) + b = tf.get_variable( + name=name + '_b', dtype=tf.float32, shape=(input_dim) ) - b = tf.get_variable(name=name + '_b', dtype=tf.float32, shape=(input_dim)) xw = tf.reduce_sum(x * w, axis=1, keepdims=True) # (B, 1) x = tf.math.add(tf.math.add(x0 * xw, b), x) return x @@ -49,8 +53,9 @@ def build_predict_graph(self): # deep tower deep_tower_config = self._model_config.deep_tower - dnn_layer = dnn.DNN(deep_tower_config.dnn, self._l2_reg, 'dnn', - self._is_training) + dnn_layer = dnn.DNN( + deep_tower_config.dnn, self._l2_reg, 'dnn', self._is_training + ) deep_tensor = dnn_layer(self._features) tower_fea_arr.append(deep_tensor) # cross tower @@ -60,8 +65,10 @@ def build_predict_graph(self): tower_fea_arr.append(cross_tensor) # final tower all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, - 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN( + self._model_config.final_dnn, self._l2_reg, 'final_dnn', + self._is_training + ) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/deepfm.py b/easy_rec/python/model/deepfm.py index d1414c050..c6d851b87 100644 --- a/easy_rec/python/model/deepfm.py +++ b/easy_rec/python/model/deepfm.py @@ -2,8 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import dnn -from easy_rec.python.layers import fm +from easy_rec.python.layers import dnn, fm from easy_rec.python.model.rank_model import RankModel from easy_rec.python.protos.deepfm_pb2 import DeepFM as DeepFMConfig @@ -13,14 +12,17 @@ class DeepFM(RankModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(DeepFM, self).__init__(model_config, feature_configs, features, - labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(DeepFM, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'deepfm', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.deepfm @@ -29,11 +31,13 @@ def __init__(self, # backward compatibility if self._model_config.HasField('wide_regularization'): tf.logging.warn( - 'wide_regularization is deprecated, please use l2_regularization') + 'wide_regularization is deprecated, please use l2_regularization' + ) self._wide_features, _ = self._input_layer(self._feature_dict, 'wide') self._deep_features, self._fm_features = self._input_layer( - self._feature_dict, 'deep') + self._feature_dict, 'deep' + ) if 'fm' in self._input_layer._feature_groups: _, self._fm_features = self._input_layer(self._feature_dict, 'fm') @@ -48,42 +52,49 @@ def build_input_layer(self, model_config, feature_configs): def build_predict_graph(self): # Wide wide_fea = tf.reduce_sum( - self._wide_features, axis=1, keepdims=True, name='wide_feature') + self._wide_features, axis=1, keepdims=True, name='wide_feature' + ) # FM fm_fea = fm.FM(name='fm_feature')(self._fm_features) self._fm_outputs = fm_fea # Deep - deep_layer = dnn.DNN(self._model_config.dnn, self._l2_reg, 'deep_feature', - self._is_training) + deep_layer = dnn.DNN( + self._model_config.dnn, self._l2_reg, 'deep_feature', self._is_training + ) deep_fea = deep_layer(self._deep_features) # Final if len(self._model_config.final_dnn.hidden_units) > 0: all_fea = tf.concat([wide_fea, fm_fea, deep_fea], axis=1) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, - 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN( + self._model_config.final_dnn, self._l2_reg, 'final_dnn', + self._is_training + ) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense( - all_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='output') + all_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='output' + ) else: if self._num_class > 1: fm_fea = tf.layers.dense( - fm_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='fm_logits') + fm_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='fm_logits' + ) else: fm_fea = tf.reduce_sum(fm_fea, 1, keepdims=True) deep_fea = tf.layers.dense( - deep_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='deep_logits') + deep_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='deep_logits' + ) output = wide_fea + fm_fea + deep_fea self._add_to_prediction_dict(output) @@ -92,15 +103,18 @@ def build_predict_graph(self): def build_feature_output_dict(self): outputs = super(DeepFM, self).build_feature_output_dict() - outputs.update({ + outputs.update( + { 'wide_features': - tf.reduce_join( - tf.as_string(self._wide_features), axis=-1, separator=','), + tf.reduce_join( + tf.as_string(self._wide_features), axis=-1, separator=',' + ), 'deep_features': - tf.reduce_join( - tf.as_string(self._deep_features), axis=-1, separator=','), + tf.reduce_join( + tf.as_string(self._deep_features), axis=-1, separator=',' + ), 'fm_outputs': - tf.reduce_join( - tf.as_string(self._fm_outputs), axis=-1, separator=',') - }) + tf.reduce_join(tf.as_string(self._fm_outputs), axis=-1, separator=',') + } + ) return outputs diff --git a/easy_rec/python/model/dlrm.py b/easy_rec/python/model/dlrm.py index 1b542ac58..548e7bea1 100755 --- a/easy_rec/python/model/dlrm.py +++ b/easy_rec/python/model/dlrm.py @@ -1,12 +1,10 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.dlrm_pb2 import DLRM as DLRMConfig # NOQA if tf.__version__ >= '2.0': @@ -16,30 +14,36 @@ class DLRM(RankModel): """Implements Deep Learning Recommendation Model for Personalization and Recommendation Systems(FaceBook).""" - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(DLRM, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(DLRM, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert model_config.WhichOneof('model') == 'dlrm', \ 'invalid model config: %s' % model_config.WhichOneof('model') self._model_config = model_config.dlrm assert isinstance(self._model_config, DLRMConfig) assert self._input_layer.has_group( - 'sparse'), 'sparse group is not specified' + 'sparse' + ), 'sparse group is not specified' _, self._sparse_features = self._input_layer(self._feature_dict, 'sparse') assert self._input_layer.has_group('dense'), 'dense group is not specified' self._dense_feature, _ = self._input_layer(self._feature_dict, 'dense') def build_predict_graph(self): - bot_dnn = dnn.DNN(self._model_config.bot_dnn, self._l2_reg, 'bot_dnn', - self._is_training) + bot_dnn = dnn.DNN( + self._model_config.bot_dnn, self._l2_reg, 'bot_dnn', self._is_training + ) dense_fea = bot_dnn(self._dense_feature) - logging.info('arch_interaction_op = %s' % - self._model_config.arch_interaction_op) + logging.info( + 'arch_interaction_op = %s' % self._model_config.arch_interaction_op + ) if self._model_config.arch_interaction_op == 'cat': all_fea = tf.concat([dense_fea] + self._sparse_features, axis=1) elif self._model_config.arch_interaction_op == 'dot': @@ -62,11 +66,13 @@ def build_predict_graph(self): concat_feas.append(dense_fea) all_fea = tf.concat(concat_feas, axis=1) - top_dnn = dnn.DNN(self._model_config.top_dnn, self._l2_reg, 'top_dnn', - self._is_training) + top_dnn = dnn.DNN( + self._model_config.top_dnn, self._l2_reg, 'top_dnn', self._is_training + ) all_fea = top_dnn(all_fea) logits = tf.layers.dense( - all_fea, 1, kernel_regularizer=self._l2_reg, name='output') + all_fea, 1, kernel_regularizer=self._l2_reg, name='output' + ) self._add_to_prediction_dict(logits) diff --git a/easy_rec/python/model/dropoutnet.py b/easy_rec/python/model/dropoutnet.py index 683677531..3d95cac66 100644 --- a/easy_rec/python/model/dropoutnet.py +++ b/easy_rec/python/model/dropoutnet.py @@ -4,13 +4,12 @@ from easy_rec.python.layers import dnn from easy_rec.python.loss.pairwise_loss import pairwise_loss +from easy_rec.python.loss.softmax_loss_with_negative_mining import softmax_loss_with_negative_mining # NOQA from easy_rec.python.model.easy_rec_model import EasyRecModel +from easy_rec.python.protos.dropoutnet_pb2 import DropoutNet as DropoutNetConfig # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.utils.proto_util import copy_obj -from easy_rec.python.protos.dropoutnet_pb2 import DropoutNet as DropoutNetConfig # NOQA -from easy_rec.python.loss.softmax_loss_with_negative_mining import softmax_loss_with_negative_mining # NOQA -from easy_rec.python.protos.dropoutnet_pb2 import DropoutNet as DropoutNetConfig # NOQA if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses @@ -18,7 +17,8 @@ def cosine_similarity(user_emb, item_emb): user_item_sim = tf.reduce_sum( - tf.multiply(user_emb, item_emb), axis=1, name='cosine') + tf.multiply(user_emb, item_emb), axis=1, name='cosine' + ) return user_item_sim @@ -33,19 +33,23 @@ def bernoulli_dropout(x, rate, training=False): class DropoutNet(EasyRecModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(DropoutNet, self).__init__(model_config, feature_configs, features, - labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(DropoutNet, self).__init__( + model_config, feature_configs, features, labels, is_training + ) self._losses = self._model_config.losses assert self._model_config.WhichOneof( - 'model' + 'model' ) == 'dropoutnet', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model') + 'model' + ) self._model_config = self._model_config.dropoutnet assert isinstance(self._model_config, DropoutNetConfig) @@ -55,11 +59,13 @@ def __init__(self, self.user_tower_layers = copy_obj(self._model_config.user_tower) self.user_content_feature, self.user_preference_feature = None, None if self._input_layer.has_group('user_content'): - self.user_content_feature, _ = self._input_layer(self._feature_dict, - 'user_content') + self.user_content_feature, _ = self._input_layer( + self._feature_dict, 'user_content' + ) if self._input_layer.has_group('user_preference'): self.user_preference_feature, _ = self._input_layer( - self._feature_dict, 'user_preference') + self._feature_dict, 'user_preference' + ) assert self.user_content_feature is not None or self.user_preference_feature is not None, 'no user feature' # copy_obj so that any modification will not affect original config @@ -68,11 +74,13 @@ def __init__(self, self.item_tower_layers = copy_obj(self._model_config.item_tower) self.item_content_feature, self.item_preference_feature = None, None if self._input_layer.has_group('item_content'): - self.item_content_feature, _ = self._input_layer(self._feature_dict, - 'item_content') + self.item_content_feature, _ = self._input_layer( + self._feature_dict, 'item_content' + ) if self._input_layer.has_group('item_preference'): self.item_preference_feature, _ = self._input_layer( - self._feature_dict, 'item_preference') + self._feature_dict, 'item_preference' + ) assert self.item_content_feature is not None or self.item_preference_feature is not None, 'no item feature' def build_predict_graph(self): @@ -86,57 +94,71 @@ def build_predict_graph(self): with tf.name_scope('user_tower'): user_features = [] if self.user_content_feature is not None: - user_content_dnn = dnn.DNN(self.user_content_layers, self._l2_reg, - 'user_content', self._is_training) + user_content_dnn = dnn.DNN( + self.user_content_layers, self._l2_reg, 'user_content', + self._is_training + ) content_feature = user_content_dnn(self.user_content_feature) user_features.append(content_feature) if self.user_preference_feature is not None: user_prefer_feature = bernoulli_dropout( - self.user_preference_feature, self._model_config.user_dropout_rate, - self._is_training) - user_prefer_dnn = dnn.DNN(self.user_preference_layers, self._l2_reg, - 'user_preference', self._is_training) + self.user_preference_feature, self._model_config.user_dropout_rate, + self._is_training + ) + user_prefer_dnn = dnn.DNN( + self.user_preference_layers, self._l2_reg, 'user_preference', + self._is_training + ) prefer_feature = user_prefer_dnn(user_prefer_feature) user_features.append(prefer_feature) user_tower_feature = tf.concat(user_features, axis=-1) - user_dnn = dnn.DNN(self.user_tower_layers, self._l2_reg, 'user_dnn', - self._is_training) + user_dnn = dnn.DNN( + self.user_tower_layers, self._l2_reg, 'user_dnn', self._is_training + ) user_hidden = user_dnn(user_tower_feature) user_tower_emb = tf.layers.dense( - inputs=user_hidden, - units=last_user_hidden, - kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1)) + inputs=user_hidden, + units=last_user_hidden, + kernel_regularizer=self._l2_reg, + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1) + ) # --------------------------build item tower----------------------------------- with tf.name_scope('item_tower'): item_features = [] if self.item_content_feature is not None: - item_content_dnn = dnn.DNN(self.item_content_layers, self._l2_reg, - 'item_content', self._is_training) + item_content_dnn = dnn.DNN( + self.item_content_layers, self._l2_reg, 'item_content', + self._is_training + ) content_feature = item_content_dnn(self.item_content_feature) item_features.append(content_feature) if self.item_preference_feature is not None: item_prefer_feature = bernoulli_dropout( - self.item_preference_feature, self._model_config.item_dropout_rate, - self._is_training) - item_prefer_dnn = dnn.DNN(self.item_preference_layers, self._l2_reg, - 'item_preference', self._is_training) + self.item_preference_feature, self._model_config.item_dropout_rate, + self._is_training + ) + item_prefer_dnn = dnn.DNN( + self.item_preference_layers, self._l2_reg, 'item_preference', + self._is_training + ) prefer_feature = item_prefer_dnn(item_prefer_feature) item_features.append(prefer_feature) item_tower_feature = tf.concat(item_features, axis=-1) - item_dnn = dnn.DNN(self.item_tower_layers, self._l2_reg, 'item_dnn', - self._is_training) + item_dnn = dnn.DNN( + self.item_tower_layers, self._l2_reg, 'item_dnn', self._is_training + ) item_hidden = item_dnn(item_tower_feature) item_tower_emb = tf.layers.dense( - inputs=item_hidden, - units=last_item_hidden, - kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1)) + inputs=item_hidden, + units=last_item_hidden, + kernel_regularizer=self._l2_reg, + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1) + ) user_emb = tf.nn.l2_normalize(user_tower_emb, axis=-1) item_emb = tf.nn.l2_normalize(item_tower_emb, axis=-1) @@ -145,9 +167,11 @@ def build_predict_graph(self): self._prediction_dict['float_user_emb'] = user_emb self._prediction_dict['float_item_emb'] = item_emb self._prediction_dict['user_emb'] = tf.reduce_join( - tf.as_string(user_emb), axis=-1, separator=',') + tf.as_string(user_emb), axis=-1, separator=',' + ) self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_emb), axis=-1, separator=',') + tf.as_string(item_emb), axis=-1, separator=',' + ) return self._prediction_dict def build_loss_graph(self): @@ -156,26 +180,29 @@ def build_loss_graph(self): for loss in self._losses: if loss.loss_type == LossType.SOFTMAX_CROSS_ENTROPY_WITH_NEGATIVE_MINING: assert self._model_config.HasField( - 'softmax_loss'), '`softmax_loss` must be configured' + 'softmax_loss' + ), '`softmax_loss` must be configured' user_emb = self._prediction_dict['float_user_emb'] item_emb = self._prediction_dict['float_item_emb'] loss_value = softmax_loss_with_negative_mining( - user_emb, - item_emb, - labels, - self._model_config.softmax_loss.num_negative_samples, - embed_normed=True, - weights=self._sample_weight, - margin=self._model_config.softmax_loss.margin, - gamma=self._model_config.softmax_loss.gamma, - t=self._model_config.softmax_loss.coefficient_of_support_vector) + user_emb, + item_emb, + labels, + self._model_config.softmax_loss.num_negative_samples, + embed_normed=True, + weights=self._sample_weight, + margin=self._model_config.softmax_loss.margin, + gamma=self._model_config.softmax_loss.gamma, + t=self._model_config.softmax_loss.coefficient_of_support_vector + ) self._loss_dict['softmax_loss'] = loss_value * loss.weight elif loss.loss_type == LossType.PAIR_WISE_LOSS: loss_value = pairwise_loss(labels, logits) self._loss_dict['pairwise_loss'] = loss_value * loss.weight elif loss.loss_type == LossType.CLASSIFICATION: - loss_value = tf.losses.sigmoid_cross_entropy(labels, logits, - self._sample_weight) + loss_value = tf.losses.sigmoid_cross_entropy( + labels, logits, self._sample_weight + ) self._loss_dict['sigmoid_loss'] = loss_value * loss.weight return self._loss_dict @@ -189,16 +216,20 @@ def build_metric_graph(self, eval_config): for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'auc': metric_dict['auc'] = metrics.auc( - labels, prob, weights=self._sample_weight) + labels, prob, weights=self._sample_weight + ) elif metric.WhichOneof('metric') == 'accuracy': metric_dict['accuracy'] = metrics.accuracy( - tf.cast(labels, tf.bool), predict, weights=self._sample_weight) + tf.cast(labels, tf.bool), predict, weights=self._sample_weight + ) elif metric.WhichOneof('metric') == 'precision': metric_dict['precision'] = metrics.precision( - labels, predict, weights=self._sample_weight) + labels, predict, weights=self._sample_weight + ) elif metric.WhichOneof('metric') == 'recall': metric_dict['recall'] = metrics.recall( - labels, predict, weights=self._sample_weight) + labels, predict, weights=self._sample_weight + ) else: ValueError('invalid metric type: %s' % str(metric)) return metric_dict diff --git a/easy_rec/python/model/dssm.py b/easy_rec/python/model/dssm.py index e35d69030..2dbeb282e 100644 --- a/easy_rec/python/model/dssm.py +++ b/easy_rec/python/model/dssm.py @@ -16,14 +16,17 @@ class DSSM(MatchModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(DSSM, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(DSSM, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'dssm', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.dssm @@ -41,25 +44,29 @@ def __init__(self, def build_predict_graph(self): num_user_dnn_layer = len(self.user_tower.dnn.hidden_units) last_user_hidden = self.user_tower.dnn.hidden_units.pop() - user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', - self._is_training) + user_dnn = dnn.DNN( + self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training + ) user_tower_emb = user_dnn(self.user_tower_feature) user_tower_emb = tf.layers.dense( - inputs=user_tower_emb, - units=last_user_hidden, - kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1)) + inputs=user_tower_emb, + units=last_user_hidden, + kernel_regularizer=self._l2_reg, + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1) + ) num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() - item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', - self._is_training) + item_dnn = dnn.DNN( + self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training + ) item_tower_emb = item_dnn(self.item_tower_feature) item_tower_emb = tf.layers.dense( - inputs=item_tower_emb, - units=last_item_hidden, - kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1)) + inputs=item_tower_emb, + units=last_item_hidden, + kernel_regularizer=self._l2_reg, + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1) + ) if self._model_config.simi_func == Similarity.COSINE: user_tower_emb = self.norm(user_tower_emb) @@ -71,15 +78,17 @@ def build_predict_graph(self): user_item_sim = self.sim(user_tower_emb, item_tower_emb) / temperature if self._model_config.scale_simi: sim_w = tf.get_variable( - 'sim_w', - dtype=tf.float32, - shape=(1), - initializer=tf.ones_initializer()) + 'sim_w', + dtype=tf.float32, + shape=(1), + initializer=tf.ones_initializer() + ) sim_b = tf.get_variable( - 'sim_b', - dtype=tf.float32, - shape=(1), - initializer=tf.zeros_initializer()) + 'sim_b', + dtype=tf.float32, + shape=(1), + initializer=tf.zeros_initializer() + ) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -100,25 +109,29 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb self._prediction_dict['user_emb'] = tf.reduce_join( - tf.as_string(user_tower_emb), axis=-1, separator=',') + tf.as_string(user_tower_emb), axis=-1, separator=',' + ) self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_tower_emb), axis=-1, separator=',') + tf.as_string(item_tower_emb), axis=-1, separator=',' + ) return self._prediction_dict def get_outputs(self): if self._loss_type == LossType.CLASSIFICATION: return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' + 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', + 'item_tower_emb' ] elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: self._prediction_dict['logits'] = tf.squeeze( - self._prediction_dict['logits'], axis=-1) + self._prediction_dict['logits'], axis=-1 + ) self._prediction_dict['probs'] = tf.nn.sigmoid( - self._prediction_dict['logits']) + self._prediction_dict['logits'] + ) return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' + 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', + 'item_tower_emb' ] elif self._loss_type == LossType.L2_LOSS: return ['y', 'user_emb', 'item_emb', 'user_tower_emb', 'item_tower_emb'] @@ -128,27 +141,35 @@ def get_outputs(self): def build_output_dict(self): output_dict = super(DSSM, self).build_output_dict() output_dict['user_tower_feature'] = tf.reduce_join( - tf.as_string(self.user_tower_feature), axis=-1, separator=',') + tf.as_string(self.user_tower_feature), axis=-1, separator=',' + ) output_dict['item_tower_feature'] = tf.reduce_join( - tf.as_string(self.item_tower_feature), axis=-1, separator=',') + tf.as_string(self.item_tower_feature), axis=-1, separator=',' + ) return output_dict def build_rtp_output_dict(self): output_dict = super(DSSM, self).build_rtp_output_dict() if 'user_tower_emb' not in self._prediction_dict: raise ValueError( - 'User tower embedding does not exist. Please checking predict graph.') + 'User tower embedding does not exist. Please checking predict graph.' + ) output_dict['user_embedding_output'] = tf.identity( - self._prediction_dict['user_tower_emb'], name='user_embedding_output') + self._prediction_dict['user_tower_emb'], name='user_embedding_output' + ) if 'item_tower_emb' not in self._prediction_dict: raise ValueError( - 'Item tower embedding does not exist. Please checking predict graph.') + 'Item tower embedding does not exist. Please checking predict graph.' + ) output_dict['item_embedding_output'] = tf.identity( - self._prediction_dict['item_tower_emb'], name='item_embedding_output') + self._prediction_dict['item_tower_emb'], name='item_embedding_output' + ) if self._loss_type == LossType.CLASSIFICATION: if 'probs' not in self._prediction_dict: raise ValueError( - 'Probs output does not exist. Please checking predict graph.') + 'Probs output does not exist. Please checking predict graph.' + ) output_dict['rank_predict'] = tf.identity( - self._prediction_dict['probs'], name='rank_predict') + self._prediction_dict['probs'], name='rank_predict' + ) return output_dict diff --git a/easy_rec/python/model/dssm_senet.py b/easy_rec/python/model/dssm_senet.py index c84d52161..cef6f709e 100644 --- a/easy_rec/python/model/dssm_senet.py +++ b/easy_rec/python/model/dssm_senet.py @@ -2,16 +2,14 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import dnn -from easy_rec.python.layers import senet +from easy_rec.python.layers import dnn, senet from easy_rec.python.model.dssm import DSSM from easy_rec.python.model.match_model import MatchModel +from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.protos.simi_pb2 import Similarity from easy_rec.python.utils.proto_util import copy_obj -from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses @@ -19,15 +17,18 @@ class DSSM_SENet(DSSM): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): - MatchModel.__init__(self, model_config, feature_configs, features, labels, - is_training) + MatchModel.__init__( + self, model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'dssm_senet', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') @@ -38,14 +39,16 @@ def __init__(self, self.user_tower = copy_obj(self._model_config.user_tower) self.user_seq_features, self.user_plain_features, self.user_feature_list = self._input_layer( - self._feature_dict, 'user', is_combine=False) + self._feature_dict, 'user', is_combine=False + ) self.user_num_fields = len(self.user_feature_list) # copy_obj so that any modification will not affect original config self.item_tower = copy_obj(self._model_config.item_tower) self.item_seq_features, self.item_plain_features, self.item_feature_list = self._input_layer( - self._feature_dict, 'item', is_combine=False) + self._feature_dict, 'item', is_combine=False + ) self.item_num_fields = len(self.item_feature_list) self._user_tower_emb = None @@ -53,45 +56,51 @@ def __init__(self, def build_predict_graph(self): user_senet = senet.SENet( - num_fields=self.user_num_fields, - num_squeeze_group=self.user_tower.senet.num_squeeze_group, - reduction_ratio=self.user_tower.senet.reduction_ratio, - l2_reg=self._l2_reg, - name='user_senet') + num_fields=self.user_num_fields, + num_squeeze_group=self.user_tower.senet.num_squeeze_group, + reduction_ratio=self.user_tower.senet.reduction_ratio, + l2_reg=self._l2_reg, + name='user_senet' + ) user_senet_output_list = user_senet(self.user_feature_list) user_senet_output = tf.concat(user_senet_output_list, axis=-1) num_user_dnn_layer = len(self.user_tower.dnn.hidden_units) last_user_hidden = self.user_tower.dnn.hidden_units.pop() - user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', - self._is_training) + user_dnn = dnn.DNN( + self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training + ) user_tower_emb = user_dnn(user_senet_output) user_tower_emb = tf.layers.dense( - inputs=user_tower_emb, - units=last_user_hidden, - kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1)) + inputs=user_tower_emb, + units=last_user_hidden, + kernel_regularizer=self._l2_reg, + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1) + ) item_senet = senet.SENet( - num_fields=self.item_num_fields, - num_squeeze_group=self.item_tower.senet.num_squeeze_group, - reduction_ratio=self.item_tower.senet.reduction_ratio, - l2_reg=self._l2_reg, - name='item_senet') + num_fields=self.item_num_fields, + num_squeeze_group=self.item_tower.senet.num_squeeze_group, + reduction_ratio=self.item_tower.senet.reduction_ratio, + l2_reg=self._l2_reg, + name='item_senet' + ) item_senet_output_list = item_senet(self.item_feature_list) item_senet_output = tf.concat(item_senet_output_list, axis=-1) num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() - item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', - self._is_training) + item_dnn = dnn.DNN( + self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training + ) item_tower_emb = item_dnn(item_senet_output) item_tower_emb = tf.layers.dense( - inputs=item_tower_emb, - units=last_item_hidden, - kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1)) + inputs=item_tower_emb, + units=last_item_hidden, + kernel_regularizer=self._l2_reg, + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1) + ) if self._model_config.simi_func == Similarity.COSINE: user_tower_emb = self.norm(user_tower_emb) @@ -103,15 +112,17 @@ def build_predict_graph(self): user_item_sim = self.sim(user_tower_emb, item_tower_emb) / temperature if self._model_config.scale_simi: sim_w = tf.get_variable( - 'sim_w', - dtype=tf.float32, - shape=(1), - initializer=tf.ones_initializer()) + 'sim_w', + dtype=tf.float32, + shape=(1), + initializer=tf.ones_initializer() + ) sim_b = tf.get_variable( - 'sim_b', - dtype=tf.float32, - shape=(1), - initializer=tf.zeros_initializer()) + 'sim_b', + dtype=tf.float32, + shape=(1), + initializer=tf.zeros_initializer() + ) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -132,9 +143,11 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb self._prediction_dict['user_emb'] = tf.reduce_join( - tf.as_string(user_tower_emb), axis=-1, separator=',') + tf.as_string(user_tower_emb), axis=-1, separator=',' + ) self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_tower_emb), axis=-1, separator=',') + tf.as_string(item_tower_emb), axis=-1, separator=',' + ) return self._prediction_dict def build_output_dict(self): diff --git a/easy_rec/python/model/dummy_model.py b/easy_rec/python/model/dummy_model.py index d32f9d811..ca8b97305 100644 --- a/easy_rec/python/model/dummy_model.py +++ b/easy_rec/python/model/dummy_model.py @@ -8,14 +8,17 @@ class DummyModel(EasyRecModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(DummyModel, self).__init__(model_config, feature_configs, features, - labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(DummyModel, self).__init__( + model_config, feature_configs, features, labels, is_training + ) if self._labels is not None: self._labels = list(self._labels.values()) @@ -36,9 +39,10 @@ def build_predict_graph(self): def build_loss_graph(self): return { - 'cross_ent': - tf.reduce_sum( - tf.square(self._prediction_dict['output'] - self._labels[0])) + 'cross_ent': + tf.reduce_sum( + tf.square(self._prediction_dict['output'] - self._labels[0]) + ) } def get_outputs(self): diff --git a/easy_rec/python/model/easy_rec_estimator.py b/easy_rec/python/model/easy_rec_estimator.py index aca664d05..587c9ec0f 100644 --- a/easy_rec/python/model/easy_rec_estimator.py +++ b/easy_rec/python/model/easy_rec_estimator.py @@ -6,42 +6,29 @@ import logging import os import re +import tensorflow as tf import time from collections import OrderedDict - -import tensorflow as tf from tensorflow.python.client import session as tf_session from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.saved_model import signature_constants -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import saver +from tensorflow.python.training import basic_session_run_hooks, saver from easy_rec.python.builders import optimizer_builder -from easy_rec.python.compat import optimizers -from easy_rec.python.compat import sync_replicas_optimizer -from easy_rec.python.compat.early_stopping import custom_early_stop_hook -from easy_rec.python.compat.early_stopping import deadline_stop_hook -from easy_rec.python.compat.early_stopping import find_early_stop_var -from easy_rec.python.compat.early_stopping import oss_stop_hook -from easy_rec.python.compat.early_stopping import stop_if_no_decrease_hook -from easy_rec.python.compat.early_stopping import stop_if_no_increase_hook +from easy_rec.python.compat import optimizers, sync_replicas_optimizer +from easy_rec.python.compat.early_stopping import custom_early_stop_hook, deadline_stop_hook, find_early_stop_var, oss_stop_hook, stop_if_no_decrease_hook, stop_if_no_increase_hook # NOQA +from easy_rec.python.compat.embedding_parallel_saver import EmbeddingParallelSaver # NOQA from easy_rec.python.compat.ops import GraphKeys from easy_rec.python.input.input import Input from easy_rec.python.layers.utils import _tensor_to_tensorinfo from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.utils import constant -from easy_rec.python.utils import embedding_utils -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import hvd_utils -from easy_rec.python.utils import pai_util +from easy_rec.python.utils import constant, embedding_utils, estimator_utils, hvd_utils, pai_util # NOQA from easy_rec.python.utils.multi_optimizer import MultiOptimizer -from easy_rec.python.compat.embedding_parallel_saver import EmbeddingParallelSaver # NOQA - try: import horovod.tensorflow as hvd except Exception: @@ -49,6 +36,7 @@ try: from sparse_operation_kit import experiment as sok + from easy_rec.python.compat import sok_optimizer except Exception: sok = None @@ -67,51 +55,53 @@ def __init__(self, pipeline_config, model_cls, run_config, params): assert isinstance(self._pipeline_config, EasyRecConfig) super(EasyRecEstimator, self).__init__( - model_fn=self._model_fn, - model_dir=pipeline_config.model_dir, - config=run_config, - params=params) - - def evaluate(self, - input_fn, - steps=None, - hooks=None, - checkpoint_path=None, - name=None): + model_fn=self._model_fn, + model_dir=pipeline_config.model_dir, + config=run_config, + params=params + ) + + def evaluate( + self, input_fn, steps=None, hooks=None, checkpoint_path=None, name=None + ): # support for datahub/kafka offset restore input_fn.input_creator.restore(checkpoint_path) - return super(EasyRecEstimator, self).evaluate(input_fn, steps, hooks, - checkpoint_path, name) - - def train(self, - input_fn, - hooks=None, - steps=None, - max_steps=None, - saving_listeners=None): + return super(EasyRecEstimator, + self).evaluate(input_fn, steps, hooks, checkpoint_path, name) + + def train( + self, + input_fn, + hooks=None, + steps=None, + max_steps=None, + saving_listeners=None + ): # support for datahub/kafka offset restore checkpoint_path = estimator_utils.latest_checkpoint(self.model_dir) if checkpoint_path is not None: input_fn.input_creator.restore(checkpoint_path) elif self.train_config.HasField('fine_tune_checkpoint'): fine_tune_ckpt = self.train_config.fine_tune_checkpoint - if fine_tune_ckpt.endswith('/') or gfile.IsDirectory(fine_tune_ckpt + - '/'): + if fine_tune_ckpt.endswith('/' + ) or gfile.IsDirectory(fine_tune_ckpt + '/'): fine_tune_ckpt = estimator_utils.latest_checkpoint(fine_tune_ckpt) print( - 'fine_tune_checkpoint[%s] is directory, will use the latest checkpoint: %s' - % (self.train_config.fine_tune_checkpoint, fine_tune_ckpt)) + 'fine_tune_checkpoint[%s] is directory, will use the latest checkpoint: %s' + % (self.train_config.fine_tune_checkpoint, fine_tune_ckpt) + ) self.train_config.fine_tune_checkpoint = fine_tune_ckpt input_fn.input_creator.restore(fine_tune_ckpt) - return super(EasyRecEstimator, self).train(input_fn, hooks, steps, - max_steps, saving_listeners) + return super(EasyRecEstimator, self + ).train(input_fn, hooks, steps, max_steps, saving_listeners) @property def feature_configs(self): if len(self._pipeline_config.feature_configs) > 0: return self._pipeline_config.feature_configs elif self._pipeline_config.feature_config and len( - self._pipeline_config.feature_config.features) > 0: + self._pipeline_config.feature_config.features + ) > 0: return self._pipeline_config.feature_config.features else: assert False, 'One of feature_configs and feature_config.features must be configured.' @@ -131,7 +121,8 @@ def train_config(self): @property def incr_save_config(self): return self.train_config.incr_save_config if self.train_config.HasField( - 'incr_save_config') else None + 'incr_save_config' + ) else None @property def export_config(self): @@ -140,8 +131,9 @@ def export_config(self): @property def embedding_parallel(self): return self.train_config.train_distribute in ( - DistributionStrategy.SokStrategy, - DistributionStrategy.EmbeddingParallelStrategy) + DistributionStrategy.SokStrategy, + DistributionStrategy.EmbeddingParallelStrategy + ) @property def saver_cls(self): @@ -155,29 +147,33 @@ def saver_cls(self): def _train_model_fn(self, features, labels, run_config): tf.keras.backend.set_learning_phase(1) model = self._model_cls( - self.model_config, - self.feature_configs, - features, - labels, - is_training=True) + self.model_config, + self.feature_configs, + features, + labels, + is_training=True + ) predict_dict = model.build_predict_graph() loss_dict = model.build_loss_graph() regularization_losses = tf.get_collection( - tf.GraphKeys.REGULARIZATION_LOSSES) + tf.GraphKeys.REGULARIZATION_LOSSES + ) if regularization_losses: regularization_losses = [ - reg_loss.get() if hasattr(reg_loss, 'get') else reg_loss - for reg_loss in regularization_losses + reg_loss.get() if hasattr(reg_loss, 'get') else reg_loss + for reg_loss in regularization_losses ] regularization_losses = tf.add_n( - regularization_losses, name='regularization_loss') + regularization_losses, name='regularization_loss' + ) loss_dict['regularization_loss'] = regularization_losses variational_dropout_loss = tf.get_collection('variational_dropout_loss') if variational_dropout_loss: variational_dropout_loss = tf.add_n( - variational_dropout_loss, name='variational_dropout_loss') + variational_dropout_loss, name='variational_dropout_loss' + ) loss_dict['variational_dropout_loss'] = variational_dropout_loss loss = tf.add_n(list(loss_dict.values())) @@ -188,13 +184,15 @@ def _train_model_fn(self, features, labels, run_config): if Input.DATA_OFFSET in features: task_index, task_num = estimator_utils.get_task_index_and_num() data_offset_var = tf.get_variable( - name=Input.DATA_OFFSET, - dtype=tf.string, - shape=[task_num], - collections=[tf.GraphKeys.GLOBAL_VARIABLES, Input.DATA_OFFSET], - trainable=False) - update_offset = tf.assign(data_offset_var[task_index], - features[Input.DATA_OFFSET]) + name=Input.DATA_OFFSET, + dtype=tf.string, + shape=[task_num], + collections=[tf.GraphKeys.GLOBAL_VARIABLES, Input.DATA_OFFSET], + trainable=False + ) + update_offset = tf.assign( + data_offset_var[task_index], features[Input.DATA_OFFSET] + ) ops.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_offset) else: data_offset_var = None @@ -206,8 +204,9 @@ def _train_model_fn(self, features, labels, run_config): global_vars = {x.name: x for x in tf.global_variables()} for x in update_ops: if isinstance(x, ops.Operation) and x.inputs[0].name in global_vars: - ops.add_to_collection(constant.DENSE_UPDATE_VARIABLES, - global_vars[x.inputs[0].name]) + ops.add_to_collection( + constant.DENSE_UPDATE_VARIABLES, global_vars[x.inputs[0].name] + ) update_op = tf.group(*update_ops, name='update_barrier') with tf.control_dependencies([update_op]): loss = tf.identity(loss, name='total_loss') @@ -243,53 +242,66 @@ def _train_model_fn(self, features, labels, run_config): # for distributed and synced training if self.train_config.sync_replicas and run_config.num_worker_replicas > 1: - logging.info('sync_replicas: num_worker_replias = %d' % - run_config.num_worker_replicas) + logging.info( + 'sync_replicas: num_worker_replias = %d' % + run_config.num_worker_replicas + ) if pai_util.is_on_pai(): optimizer = tf.train.SyncReplicasOptimizer( - optimizer, - replicas_to_aggregate=run_config.num_worker_replicas, - total_num_replicas=run_config.num_worker_replicas, - sparse_accumulator_type=self.train_config.sparse_accumulator_type) + optimizer, + replicas_to_aggregate=run_config.num_worker_replicas, + total_num_replicas=run_config.num_worker_replicas, + sparse_accumulator_type=self.train_config.sparse_accumulator_type + ) else: optimizer = sync_replicas_optimizer.SyncReplicasOptimizer( - optimizer, - replicas_to_aggregate=run_config.num_worker_replicas, - total_num_replicas=run_config.num_worker_replicas) + optimizer, + replicas_to_aggregate=run_config.num_worker_replicas, + total_num_replicas=run_config.num_worker_replicas + ) hooks.append( - optimizer.make_session_run_hook(run_config.is_chief, num_tokens=0)) + optimizer.make_session_run_hook(run_config.is_chief, num_tokens=0) + ) # add barrier for no strategy case if run_config.num_worker_replicas > 1 and \ self.train_config.train_distribute == DistributionStrategy.NoStrategy: hooks.append( - estimator_utils.ExitBarrierHook(run_config.num_worker_replicas, - run_config.is_chief, self.model_dir)) + estimator_utils.ExitBarrierHook( + run_config.num_worker_replicas, run_config.is_chief, self.model_dir + ) + ) if self.export_config.enable_early_stop: eval_dir = os.path.join(self._model_dir, 'eval_val') logging.info('will use early stop, eval_events_dir=%s' % eval_dir) if self.export_config.HasField('early_stop_func'): hooks.append( - custom_early_stop_hook( - self, - eval_dir=eval_dir, - custom_stop_func=self.export_config.early_stop_func, - custom_stop_func_params=self.export_config.early_stop_params)) + custom_early_stop_hook( + self, + eval_dir=eval_dir, + custom_stop_func=self.export_config.early_stop_func, + custom_stop_func_params=self.export_config.early_stop_params + ) + ) elif self.export_config.metric_bigger: hooks.append( - stop_if_no_increase_hook( - self, - self.export_config.best_exporter_metric, - self.export_config.max_check_steps, - eval_dir=eval_dir)) + stop_if_no_increase_hook( + self, + self.export_config.best_exporter_metric, + self.export_config.max_check_steps, + eval_dir=eval_dir + ) + ) else: hooks.append( - stop_if_no_decrease_hook( - self, - self.export_config.best_exporter_metric, - self.export_config.max_check_steps, - eval_dir=eval_dir)) + stop_if_no_decrease_hook( + self, + self.export_config.best_exporter_metric, + self.export_config.max_check_steps, + eval_dir=eval_dir + ) + ) if self.train_config.enable_oss_stop_signal: hooks.append(oss_stop_hook(self)) @@ -307,13 +319,14 @@ def _train_model_fn(self, features, labels, run_config): gradient_multipliers = None if self.train_config.optimizer_config[0].HasField( - 'embedding_learning_rate_multiplier'): + 'embedding_learning_rate_multiplier' + ): gradient_multipliers = { - var: self.train_config.optimizer_config[0] - .embedding_learning_rate_multiplier - for var in tf.trainable_variables() - if 'embedding_weights:' in var.name or - '/embedding_weights/part_' in var.name + var: + self.train_config.optimizer_config[0]. + embedding_learning_rate_multiplier + for var in tf.trainable_variables() if 'embedding_weights:' in var.name + or '/embedding_weights/part_' in var.name } # optimize loss @@ -337,20 +350,21 @@ def _train_model_fn(self, features, labels, run_config): logging.info('embedding_parallel is enabled') train_op = optimizers.optimize_loss( - loss=loss, - global_step=tf.train.get_global_step(), - learning_rate=None, - clip_gradients=gradient_clipping_by_norm, - optimizer=optimizer, - gradient_multipliers=gradient_multipliers, - variables=all_train_vars, - summaries=summaries, - colocate_gradients_with_ops=True, - not_apply_grad_after_first_step=run_config.is_chief and - self._pipeline_config.data_config.chief_redundant, - name='', # Preventing scope prefix on all variables. - incr_save=(self.incr_save_config is not None), - embedding_parallel=self.embedding_parallel) + loss=loss, + global_step=tf.train.get_global_step(), + learning_rate=None, + clip_gradients=gradient_clipping_by_norm, + optimizer=optimizer, + gradient_multipliers=gradient_multipliers, + variables=all_train_vars, + summaries=summaries, + colocate_gradients_with_ops=True, + not_apply_grad_after_first_step=run_config.is_chief + and self._pipeline_config.data_config.chief_redundant, + name='', # Preventing scope prefix on all variables. + incr_save=(self.incr_save_config is not None), + embedding_parallel=self.embedding_parallel + ) # online evaluation metric_update_op_dict = None @@ -364,8 +378,10 @@ def _train_model_fn(self, features, labels, run_config): train_op = tf.group([train_op] + list(metric_update_op_dict.values())) if estimator_utils.is_chief(): hooks.append( - estimator_utils.OnlineEvaluationHook( - metric_dict=metric_dict, output_dir=self.model_dir)) + estimator_utils.OnlineEvaluationHook( + metric_dict=metric_dict, output_dir=self.model_dir + ) + ) if self.train_config.HasField('fine_tune_checkpoint'): fine_tune_ckpt = self.train_config.fine_tune_checkpoint @@ -373,10 +389,11 @@ def _train_model_fn(self, features, labels, run_config): fine_tune_ckpt_var_map = self.train_config.fine_tune_ckpt_var_map force_restore = self.train_config.force_restore_shape_compatible restore_hook = model.restore( - fine_tune_ckpt, - include_global_step=False, - ckpt_var_map_path=fine_tune_ckpt_var_map, - force_restore_shape_compatible=force_restore) + fine_tune_ckpt, + include_global_step=False, + ckpt_var_map_path=fine_tune_ckpt_var_map, + force_restore_shape_compatible=force_restore + ) if restore_hook is not None: hooks.append(restore_hook) @@ -390,23 +407,25 @@ def _train_model_fn(self, features, labels, run_config): log_step_count_steps = self.train_config.log_step_count_steps logging_hook = basic_session_run_hooks.LoggingTensorHook( - logging_dict, - every_n_iter=log_step_count_steps, - formatter=estimator_utils.tensor_log_format_func) + logging_dict, + every_n_iter=log_step_count_steps, + formatter=estimator_utils.tensor_log_format_func + ) hooks.append(logging_hook) if self.train_config.train_distribute in [ - DistributionStrategy.CollectiveAllReduceStrategy, - DistributionStrategy.MirroredStrategy, - DistributionStrategy.MultiWorkerMirroredStrategy + DistributionStrategy.CollectiveAllReduceStrategy, + DistributionStrategy.MirroredStrategy, + DistributionStrategy.MultiWorkerMirroredStrategy ]: # for multi worker strategy, we could not replace the # inner CheckpointSaverHook, so just use it. scaffold = tf.train.Scaffold() else: var_list = ( - tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) + - tf.get_collection(tf.GraphKeys.SAVEABLE_OBJECTS)) + tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) + + tf.get_collection(tf.GraphKeys.SAVEABLE_OBJECTS) + ) # exclude data_offset_var var_list = [x for x in var_list if x != data_offset_var] @@ -416,7 +435,7 @@ def _train_model_fn(self, features, labels, run_config): var_list = [x for x in var_list if x != early_stop_var] initialize_var_list = [ - x for x in var_list if 'WorkQueue' not in str(type(x)) + x for x in var_list if 'WorkQueue' not in str(type(x)) ] # incompatiable shape restore will not be saved in checkpoint @@ -430,56 +449,66 @@ def _train_model_fn(self, features, labels, run_config): local_init_ops.append(tf.initializers.variables([early_stop_var])) if len(incompatiable_shape_restore) > 0: local_init_ops.append( - tf.initializers.variables(incompatiable_shape_restore)) + tf.initializers.variables(incompatiable_shape_restore) + ) scaffold = tf.train.Scaffold( - saver=self.saver_cls( - var_list=var_list, - sharded=True, - max_to_keep=self.train_config.keep_checkpoint_max, - save_relative_paths=True), - local_init_op=tf.group(local_init_ops), - ready_for_local_init_op=tf.report_uninitialized_variables( - var_list=initialize_var_list)) + saver=self.saver_cls( + var_list=var_list, + sharded=True, + max_to_keep=self.train_config.keep_checkpoint_max, + save_relative_paths=True + ), + local_init_op=tf.group(local_init_ops), + ready_for_local_init_op=tf.report_uninitialized_variables( + var_list=initialize_var_list + ) + ) # saver hook saver_hook = estimator_utils.CheckpointSaverHook( - checkpoint_dir=self.model_dir, - save_secs=self._config.save_checkpoints_secs, - save_steps=self._config.save_checkpoints_steps, - scaffold=scaffold, - write_graph=self.train_config.write_graph, - data_offset_var=data_offset_var, - increment_save_config=self.incr_save_config) + checkpoint_dir=self.model_dir, + save_secs=self._config.save_checkpoints_secs, + save_steps=self._config.save_checkpoints_steps, + scaffold=scaffold, + write_graph=self.train_config.write_graph, + data_offset_var=data_offset_var, + increment_save_config=self.incr_save_config + ) if estimator_utils.is_chief() or self.embedding_parallel: hooks.append(saver_hook) if estimator_utils.is_chief(): hooks.append( - basic_session_run_hooks.StepCounterHook( - every_n_steps=log_step_count_steps, output_dir=self.model_dir)) + basic_session_run_hooks.StepCounterHook( + every_n_steps=log_step_count_steps, output_dir=self.model_dir + ) + ) # profiling hook if self.train_config.is_profiling and estimator_utils.is_chief(): profile_hook = tf.train.ProfilerHook( - save_steps=log_step_count_steps, output_dir=self.model_dir) + save_steps=log_step_count_steps, output_dir=self.model_dir + ) hooks.append(profile_hook) return tf.estimator.EstimatorSpec( - mode=tf.estimator.ModeKeys.TRAIN, - loss=loss, - predictions=predict_dict, - train_op=train_op, - scaffold=scaffold, - training_hooks=hooks) + mode=tf.estimator.ModeKeys.TRAIN, + loss=loss, + predictions=predict_dict, + train_op=train_op, + scaffold=scaffold, + training_hooks=hooks + ) def _eval_model_fn(self, features, labels, run_config): tf.keras.backend.set_learning_phase(0) start = time.time() model = self._model_cls( - self.model_config, - self.feature_configs, - features, - labels, - is_training=False) + self.model_config, + self.feature_configs, + features, + labels, + is_training=False + ) predict_dict = model.build_predict_graph() loss_dict = model.build_loss_graph() loss = tf.add_n(list(loss_dict.values())) @@ -493,33 +522,40 @@ def _eval_model_fn(self, features, labels, run_config): tf.logging.info('metric_dict keys: %s' % metric_dict.keys()) var_list = ( - ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + - ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS)) + ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + + ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS) + ) metric_variables = ops.get_collection(ops.GraphKeys.METRIC_VARIABLES) model_ready_for_local_init_op = tf.variables_initializer(metric_variables) scaffold = tf.train.Scaffold( - saver=self.saver_cls( - var_list=var_list, sharded=True, save_relative_paths=True), - ready_for_local_init_op=model_ready_for_local_init_op) + saver=self.saver_cls( + var_list=var_list, sharded=True, save_relative_paths=True + ), + ready_for_local_init_op=model_ready_for_local_init_op + ) end = time.time() - tf.logging.info('eval graph construct finished. Time %.3fs' % (end - start)) + tf.logging.info( + 'eval graph construct finished. Time %.3fs' % (end - start) + ) return tf.estimator.EstimatorSpec( - mode=tf.estimator.ModeKeys.EVAL, - loss=loss, - scaffold=scaffold, - predictions=predict_dict, - eval_metric_ops=metric_dict) + mode=tf.estimator.ModeKeys.EVAL, + loss=loss, + scaffold=scaffold, + predictions=predict_dict, + eval_metric_ops=metric_dict + ) def _distribute_eval_model_fn(self, features, labels, run_config): tf.keras.backend.set_learning_phase(0) start = time.time() model = self._model_cls( - self.model_config, - self.feature_configs, - features, - labels, - is_training=False) + self.model_config, + self.feature_configs, + features, + labels, + is_training=False + ) predict_dict = model.build_predict_graph() loss_dict = model.build_loss_graph() loss = tf.add_n(list(loss_dict.values())) @@ -532,7 +568,9 @@ def _distribute_eval_model_fn(self, features, labels, run_config): tf.logging.info('metric_dict keys: %s' % metric_dict.keys()) end = time.time() - tf.logging.info('eval graph construct finished. Time %.3fs' % (end - start)) + tf.logging.info( + 'eval graph construct finished. Time %.3fs' % (end - start) + ) metric_name_list = [] for metric_i in self.eval_config.metrics_set: metric_name_list.append(metric_i.WhichOneof('metric')) @@ -553,25 +591,29 @@ def _distribute_eval_model_fn(self, features, labels, run_config): metric_variables = tf.get_collection(tf.GraphKeys.METRIC_VARIABLES) model_ready_for_local_init_op = tf.variables_initializer(metric_variables) remain_variables = list( - set(global_variables).difference(set(metric_variables))) + set(global_variables).difference(set(metric_variables)) + ) cur_saver = tf.train.Saver(var_list=remain_variables, sharded=True) scaffold = tf.train.Scaffold( - saver=cur_saver, ready_for_local_init_op=model_ready_for_local_init_op) + saver=cur_saver, ready_for_local_init_op=model_ready_for_local_init_op + ) return tf.estimator.EstimatorSpec( - mode=tf.estimator.ModeKeys.EVAL, - loss=loss, - predictions=predict_dict, - eval_metric_ops=metric_dict, - scaffold=scaffold) + mode=tf.estimator.ModeKeys.EVAL, + loss=loss, + predictions=predict_dict, + eval_metric_ops=metric_dict, + scaffold=scaffold + ) def _export_model_fn(self, features, labels, run_config, params): tf.keras.backend.set_learning_phase(0) model = self._model_cls( - self.model_config, - self.feature_configs, - features, - labels=None, - is_training=False) + self.model_config, + self.feature_configs, + features, + labels=None, + is_training=False + ) model.build_predict_graph() export_config = self._pipeline_config.export_config @@ -587,37 +629,41 @@ def _export_model_fn(self, features, labels, run_config, params): for out in outputs: tf.logging.info( - 'output %s shape: %s type: %s' % - (out, outputs[out].get_shape().as_list(), outputs[out].dtype)) + 'output %s shape: %s type: %s' % + (out, outputs[out].get_shape().as_list(), outputs[out].dtype) + ) export_outputs = { - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: - tf.estimator.export.PredictOutput(outputs) + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + tf.estimator.export.PredictOutput(outputs) } # save train pipeline.config for debug purpose pipeline_path = os.path.join(self._model_dir, 'pipeline.config') if gfile.Exists(pipeline_path): ops.add_to_collection( - tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(pipeline_path, dtype=tf.string, name='pipeline.config')) + tf.GraphKeys.ASSET_FILEPATHS, + tf.constant(pipeline_path, dtype=tf.string, name='pipeline.config') + ) else: print('train pipeline_path(%s) does not exist' % pipeline_path) # restore DENSE_UPDATE_VARIABLES collection - dense_train_var_path = os.path.join(self.model_dir, - constant.DENSE_UPDATE_VARIABLES) + dense_train_var_path = os.path.join( + self.model_dir, constant.DENSE_UPDATE_VARIABLES + ) if gfile.Exists(dense_train_var_path): with gfile.GFile(dense_train_var_path, 'r') as fin: var_name_to_id_map = json.load(fin) var_name_id_lst = [ - (x, var_name_to_id_map[x]) for x in var_name_to_id_map + (x, var_name_to_id_map[x]) for x in var_name_to_id_map ] var_name_id_lst.sort(key=lambda x: x[1]) all_vars = {x.op.name: x for x in tf.global_variables()} for var_name, var_id in var_name_id_lst: assert var_name in all_vars, 'dense_train_var[%s] is not found' % var_name - ops.add_to_collection(constant.DENSE_UPDATE_VARIABLES, - all_vars[var_name]) + ops.add_to_collection( + constant.DENSE_UPDATE_VARIABLES, all_vars[var_name] + ) # add more asset files if len(export_config.asset_files) > 0: @@ -625,42 +671,49 @@ def _export_model_fn(self, features, labels, run_config, params): if asset_file.startswith('!'): asset_file = asset_file[1:] if ':' not in asset_file or asset_file.startswith( - 'oss:') or asset_file.startswith('hdfs:'): + 'oss:' + ) or asset_file.startswith('hdfs:'): _, asset_name = os.path.split(asset_file) else: asset_name, asset_file = asset_file.split(':', 1) ops.add_to_collection( - ops.GraphKeys.ASSET_FILEPATHS, - tf.constant(asset_file, dtype=tf.string, name=asset_name)) + ops.GraphKeys.ASSET_FILEPATHS, + tf.constant(asset_file, dtype=tf.string, name=asset_name) + ) elif 'asset_files' in params: for asset_name in params['asset_files']: asset_file = params['asset_files'][asset_name] ops.add_to_collection( - tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(asset_file, dtype=tf.string, name=asset_name)) + tf.GraphKeys.ASSET_FILEPATHS, + tf.constant(asset_file, dtype=tf.string, name=asset_name) + ) if self._pipeline_config.HasField('fg_json_path'): fg_path = self._pipeline_config.fg_json_path if fg_path[0] == '!': fg_path = fg_path[1:] ops.add_to_collection( - tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(fg_path, dtype=tf.string, name='fg.json')) + tf.GraphKeys.ASSET_FILEPATHS, + tf.constant(fg_path, dtype=tf.string, name='fg.json') + ) var_list = ( - ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + - ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS)) + ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + + ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS) + ) scaffold = tf.train.Scaffold( - saver=self.saver_cls( - var_list=var_list, sharded=True, save_relative_paths=True)) + saver=self. + saver_cls(var_list=var_list, sharded=True, save_relative_paths=True) + ) return tf.estimator.EstimatorSpec( - mode=tf.estimator.ModeKeys.PREDICT, - loss=None, - scaffold=scaffold, - predictions=outputs, - export_outputs=export_outputs) + mode=tf.estimator.ModeKeys.PREDICT, + loss=None, + scaffold=scaffold, + predictions=outputs, + export_outputs=export_outputs + ) def _model_fn(self, features, labels, mode, config, params): os.environ['tf.estimator.mode'] = mode @@ -669,7 +722,8 @@ def _model_fn(self, features, labels, mode, config, params): os.environ['place_embedding_on_cpu'] = 'True' if self._pipeline_config.fg_json_path: EasyRecEstimator._write_rtp_fg_config_to_col( - fg_config_path=self._pipeline_config.fg_json_path) + fg_config_path=self._pipeline_config.fg_json_path + ) EasyRecEstimator._write_rtp_inputs_to_col(features) if self.embedding_parallel: @@ -718,26 +772,32 @@ def _write_rtp_inputs_to_col(features): else: col[0] = json.dumps(feature_info_map) - def export_checkpoint(self, - export_path=None, - serving_input_receiver_fn=None, - checkpoint_path=None, - mode=tf.estimator.ModeKeys.PREDICT): + def export_checkpoint( + self, + export_path=None, + serving_input_receiver_fn=None, + checkpoint_path=None, + mode=tf.estimator.ModeKeys.PREDICT + ): with context.graph_mode(): if not checkpoint_path: # Locate the latest checkpoint checkpoint_path = estimator_utils.latest_checkpoint(self._model_dir) if not checkpoint_path: - raise ValueError("Couldn't find trained model at %s." % self._model_dir) + raise ValueError( + "Couldn't find trained model at %s." % self._model_dir + ) with ops.Graph().as_default(): input_receiver = serving_input_receiver_fn() estimator_spec = self._call_model_fn( - features=input_receiver.features, - labels=getattr(input_receiver, 'labels', None), - mode=mode, - config=self.config) + features=input_receiver.features, + labels=getattr(input_receiver, 'labels', None), + mode=mode, + config=self.config + ) with tf_session.Session(config=self._session_config) as session: graph_saver = estimator_spec.scaffold.saver or saver.Saver( - sharded=True) + sharded=True + ) graph_saver.restore(session, checkpoint_path) graph_saver.save(session, export_path) diff --git a/easy_rec/python/model/easy_rec_model.py b/easy_rec/python/model/easy_rec_model.py index f2408ba47..b98fbea46 100644 --- a/easy_rec/python/model/easy_rec_model.py +++ b/easy_rec/python/model/easy_rec_model.py @@ -4,33 +4,30 @@ import logging import os import re -from abc import abstractmethod - import six import tensorflow as tf -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape +from abc import abstractmethod +from tensorflow.python.framework import ops, tensor_shape from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from easy_rec.python.compat import regularizers from easy_rec.python.layers import input_layer from easy_rec.python.layers.backbone import Backbone -from easy_rec.python.utils import constant -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import restore_filter +from easy_rec.python.utils import constant, estimator_utils, restore_filter from easy_rec.python.utils.load_class import get_register_class_meta try: import horovod.tensorflow as hvd - from sparse_operation_kit.experiment import raw_ops as dynamic_variable_ops from sparse_operation_kit import experiment as sok + from sparse_operation_kit.experiment import raw_ops as dynamic_variable_ops except Exception: dynamic_variable_ops = None sok = None try: from tensorflow.python.framework.load_library import load_op_library + import easy_rec load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) @@ -43,17 +40,20 @@ _EASY_REC_MODEL_CLASS_MAP = {} _meta_type = get_register_class_meta( - _EASY_REC_MODEL_CLASS_MAP, have_abstract_class=True) + _EASY_REC_MODEL_CLASS_MAP, have_abstract_class=True +) class EasyRecModel(six.with_metaclass(_meta_type, object)): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): self._base_model_config = model_config self._model_config = model_config self._is_training = is_training @@ -66,7 +66,9 @@ def __init__(self, self._global_ev_params = model_config.ev_params if self.embedding_regularization > 0: - self._emb_reg = regularizers.l2_regularizer(self.embedding_regularization) + self._emb_reg = regularizers.l2_regularizer( + self.embedding_regularization + ) else: self._emb_reg = None @@ -102,10 +104,11 @@ def __init__(self, def build_backbone_network(self): if self.has_backbone: return Backbone( - self._base_model_config.backbone, - self._feature_dict, - input_layer=self._input_layer, - l2_reg=self._l2_reg) + self._base_model_config.backbone, + self._feature_dict, + input_layer=self._input_layer, + l2_reg=self._l2_reg + ) return None @property @@ -118,11 +121,11 @@ def backbone(self): return self._backbone_output if self._backbone_net: kwargs = { - 'loss_dict': self._loss_dict, - 'metric_dict': self._metric_dict, - 'prediction_dict': self._prediction_dict, - 'labels': self._labels, - constant.SAMPLE_WEIGHT: self._sample_weight + 'loss_dict': self._loss_dict, + 'metric_dict': self._metric_dict, + 'prediction_dict': self._prediction_dict, + 'labels': self._labels, + constant.SAMPLE_WEIGHT: self._sample_weight } return self._backbone_net(self._is_training, **kwargs) return None @@ -141,14 +144,16 @@ def feature_groups(self): @property def l2_regularization(self): - model_config = getattr(self._base_model_config, - self._base_model_config.WhichOneof('model')) + model_config = getattr( + self._base_model_config, self._base_model_config.WhichOneof('model') + ) l2_regularization = 0.0 if hasattr(model_config, 'dense_regularization') and \ model_config.HasField('dense_regularization'): # backward compatibility logging.warn( - 'dense_regularization is deprecated, please use l2_regularization') + 'dense_regularization is deprecated, please use l2_regularization' + ) l2_regularization = model_config.dense_regularization elif hasattr(model_config, 'l2_regularization'): l2_regularization = model_config.l2_regularization @@ -156,16 +161,17 @@ def l2_regularization(self): def build_input_layer(self, model_config, feature_configs): self._input_layer = input_layer.InputLayer( - feature_configs, - model_config.feature_groups, - wide_output_dim=self._wide_output_dim, - ev_params=self._global_ev_params, - embedding_regularizer=self._emb_reg, - kernel_regularizer=self._l2_reg, - variational_dropout_config=model_config.variational_dropout - if model_config.HasField('variational_dropout') else None, - is_training=self._is_training, - is_predicting=self._is_predicting) + feature_configs, + model_config.feature_groups, + wide_output_dim=self._wide_output_dim, + ev_params=self._global_ev_params, + embedding_regularizer=self._emb_reg, + kernel_regularizer=self._l2_reg, + variational_dropout_config=model_config.variational_dropout + if model_config.HasField('variational_dropout') else None, + is_training=self._is_training, + is_predicting=self._is_predicting + ) @abstractmethod def build_predict_graph(self): @@ -188,8 +194,9 @@ def build_output_dict(self): for name in self.get_outputs(): if name not in self._prediction_dict: raise KeyError( - 'output node {} not in prediction_dict, can not be exported'.format( - name)) + 'output node {} not in prediction_dict, can not be exported'. + format(name) + ) outputs[name] = self._prediction_dict[name] return outputs @@ -203,9 +210,9 @@ def build_feature_output_dict(self): sparse_values = feature_value.values if sparse_values.dtype != tf.string: sparse_values = tf.as_string(sparse_values) - feature_value = tf.sparse_to_dense(feature_value.indices, - feature_value.dense_shape, - sparse_values, '') + feature_value = tf.sparse_to_dense( + feature_value.indices, feature_value.dense_shape, sparse_values, '' + ) elif feature_value.dtype != tf.string: feature_value = tf.as_string(feature_value) feature_value = tf.reduce_join(feature_value, axis=-1, separator=',') @@ -216,11 +223,13 @@ def build_rtp_output_dict(self): """For exporting: get output nodes for RTP infering.""" return {} - def restore(self, - ckpt_path, - include_global_step=False, - ckpt_var_map_path='', - force_restore_shape_compatible=False): + def restore( + self, + ckpt_path, + include_global_step=False, + ckpt_var_map_path='', + force_restore_shape_compatible=False + ): """Restore variables from ckpt_path. steps: @@ -262,42 +271,50 @@ def restore(self, var_shape[0] += x[0] var_shape = tensor_shape.TensorShape(var_shape) variable = variables.PartitionedVariable( - variable_name, - var_shape, - variable[0].dtype, - variable, - partitions=[len(variable)] + [1] * (len(var_shape) - 1)) + variable_name, + var_shape, + variable[0].dtype, + variable, + partitions=[len(variable)] + [1] * (len(var_shape) - 1) + ) else: var_shape = variable.shape.as_list() if ckpt_var_shape == var_shape: vars_in_ckpt[variable_name] = list(variable) if isinstance( - variable, variables.PartitionedVariable) else variable + variable, variables.PartitionedVariable + ) else variable elif len(ckpt_var_shape) == len(var_shape): if force_restore_shape_compatible: # create a variable compatible with checkpoint to restore - dtype = variable[0].dtype if isinstance(variable, - list) else variable.dtype + dtype = variable[0].dtype if isinstance( + variable, list + ) else variable.dtype with tf.variable_scope('incompatible_shape_restore'): tmp_var = tf.get_variable( - name=variable_name + '_T_E_M_P', - shape=ckpt_var_shape, - trainable=False, - # add to a special collection for easy reference - # by tf.get_collection('T_E_M_P_RESTROE') - collections=['T_E_M_P_RESTROE'], - dtype=dtype) + name=variable_name + '_T_E_M_P', + shape=ckpt_var_shape, + trainable=False, + # add to a special collection for easy reference + # by tf.get_collection('T_E_M_P_RESTROE') + collections=['T_E_M_P_RESTROE'], + dtype=dtype + ) vars_in_ckpt[variable_name] = tmp_var incompatible_shape_var_map[variable] = tmp_var - print('incompatible restore %s[%s, %s]' % - (variable_name, str(var_shape), str(ckpt_var_shape))) + print( + 'incompatible restore %s[%s, %s]' % + (variable_name, str(var_shape), str(ckpt_var_shape)) + ) else: logging.warning( - 'Variable [%s] is available in checkpoint, but ' - 'incompatible shape with model variable.', variable_name) + 'Variable [%s] is available in checkpoint, but ' + 'incompatible shape with model variable.', variable_name + ) else: logging.warning( - 'Variable [%s] is available in checkpoint, but ' - 'incompatible shape dims with model variable.', variable_name) + 'Variable [%s] is available in checkpoint, but ' + 'incompatible shape dims with model variable.', variable_name + ) elif 'EmbeddingVariable' in str(type(variable)): if '%s-keys' % variable_name not in ckpt_var2shape_map: continue @@ -311,7 +328,8 @@ def restore(self, init_op = saveable_objects[0].restore([ckpt_path], None) variable._initializer_op = init_op elif type(variable) == list and 'EmbeddingVariable' in str( - type(variable[0])): + type(variable[0]) + ): if '%s/part_0-keys' % variable_name not in ckpt_var2shape_map: continue print('restore partitioned embedding_variable %s' % variable_name) @@ -327,26 +345,30 @@ def restore(self, elif sok is not None and isinstance(variable, sok.DynamicVariable): print('restore dynamic_variable %s' % variable_name) keys, vals = load_embed_lib.load_kv_embed( - task_index=hvd.rank(), - task_num=hvd.size(), - embed_dim=variable._dimension, - var_name='embed-' + variable.name.replace('/', '__'), - ckpt_path=ckpt_path) + task_index=hvd.rank(), + task_num=hvd.size(), + embed_dim=variable._dimension, + var_name='embed-' + variable.name.replace('/', '__'), + ckpt_path=ckpt_path + ) with ops.control_dependencies([variable._initializer_op]): variable._initializer_op = dynamic_variable_ops.dummy_var_assign( - variable.handle, keys, vals) + variable.handle, keys, vals + ) else: fail_restore_vars.append(variable_name) for variable_name in fail_restore_vars: if 'Momentum' not in variable_name: - logging.warning('Variable [%s] is not available in checkpoint', - variable_name) + logging.warning( + 'Variable [%s] is not available in checkpoint', variable_name + ) tf.train.init_from_checkpoint(ckpt_path, vars_in_ckpt) if force_restore_shape_compatible: return estimator_utils.IncompatibleShapeRestoreHook( - incompatible_shape_var_map) + incompatible_shape_var_map + ) else: return None @@ -369,8 +391,9 @@ def _get_restore_vars(self, ckpt_var_map_path): name2var = {} for one_var in all_vars: var_name = re.sub(VAR_SUFIX_PATTERN, '', one_var.name) - if re.search(PARTITION_PATTERN, - var_name) and one_var._save_slice_info is not None: + if re.search( + PARTITION_PATTERN, var_name + ) and one_var._save_slice_info is not None: var_name = re.sub(PARTITION_PATTERN, '', var_name) is_part = True else: @@ -411,14 +434,14 @@ def _get_restore_vars(self, ckpt_var_map_path): var_filter, scope_update = self.get_restore_filter() if var_filter is not None: name2var = { - var_name: name2var[var_name] - for var in name2var - if var_filter.keep(var.name) + var_name: name2var[var_name] + for var in name2var if var_filter.keep(var.name) } # drop scope prefix if necessary if scope_update is not None: name2var = { - scope_update(var_name): name2var[var_name] for var_name in name2var + scope_update(var_name): name2var[var_name] + for var_name in name2var } return name2var @@ -436,12 +459,13 @@ def get_restore_filter(self): logging.info('restore will filter out pattern %s' % x) all_filters = [ - restore_filter.KeywordFilter(x, True) - for x in self._base_model_config.restore_filters + restore_filter.KeywordFilter(x, True) + for x in self._base_model_config.restore_filters ] - return restore_filter.CombineFilter(all_filters, - restore_filter.Logical.AND), None + return restore_filter.CombineFilter( + all_filters, restore_filter.Logical.AND + ), None def get_grouped_vars(self, opt_num): """Group the vars into different optimization groups. @@ -460,7 +484,8 @@ def get_grouped_vars(self, opt_num): deep_vars = [] for tmp_var in variables.trainable_variables(): if tmp_var.name.startswith( - 'input_layer') or '/embedding_weights' in tmp_var.name: + 'input_layer' + ) or '/embedding_weights' in tmp_var.name: embedding_vars.append(tmp_var) else: deep_vars.append(tmp_var) diff --git a/easy_rec/python/model/esmm.py b/easy_rec/python/model/esmm.py index 50567ae63..adb2d884f 100644 --- a/easy_rec/python/model/esmm.py +++ b/easy_rec/python/model/esmm.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.layers import dnn @@ -16,14 +15,17 @@ class ESMM(MultiTaskModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(ESMM, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(ESMM, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'esmm', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.esmm @@ -67,33 +69,37 @@ def build_loss_graph(self): ctr_label_name = self._label_name_dict[ctr_tower_name] if self._cvr_tower_cfg.loss_type == LossType.CLASSIFICATION: ctcvr_label = tf.cast( - self._labels[cvr_label_name] * self._labels[ctr_label_name], - tf.float32) + self._labels[cvr_label_name] * self._labels[ctr_label_name], tf.float32 + ) cvr_losses = tf.keras.backend.binary_crossentropy( - ctcvr_label, self._prediction_dict['probs_ctcvr']) + ctcvr_label, self._prediction_dict['probs_ctcvr'] + ) cvr_loss = tf.reduce_sum(cvr_losses, name='ctcvr_loss') # The weight defaults to 1. - self._loss_dict['weighted_cross_entropy_loss_%s' % - cvr_tower_name] = self._cvr_tower_cfg.weight * cvr_loss + self._loss_dict['weighted_cross_entropy_loss_%s' % cvr_tower_name + ] = self._cvr_tower_cfg.weight * cvr_loss elif self._cvr_tower_cfg.loss_type == LossType.L2_LOSS: logging.info('l2 loss is used') cvr_dtype = self._labels[cvr_label_name].dtype ctcvr_label = self._labels[cvr_label_name] * tf.cast( - self._labels[ctr_label_name], cvr_dtype) + self._labels[ctr_label_name], cvr_dtype + ) cvr_loss = tf.losses.mean_squared_error( - labels=ctcvr_label, - predictions=self._prediction_dict['y_ctcvr'], - weights=self._sample_weight) - self._loss_dict['weighted_l2_loss_%s' % - cvr_tower_name] = self._cvr_tower_cfg.weight * cvr_loss + labels=ctcvr_label, + predictions=self._prediction_dict['y_ctcvr'], + weights=self._sample_weight + ) + self._loss_dict['weighted_l2_loss_%s' % cvr_tower_name + ] = self._cvr_tower_cfg.weight * cvr_loss _labels = tf.cast(self._labels[ctr_label_name], tf.float32) _logits = self._prediction_dict['logits_%s' % ctr_tower_name] cross = tf.nn.sigmoid_cross_entropy_with_logits( - labels=_labels, logits=_logits, name='ctr_loss') + labels=_labels, logits=_logits, name='ctr_loss' + ) ctr_loss = tf.reduce_sum(cross) - self._loss_dict['weighted_cross_entropy_loss_%s' % - ctr_tower_name] = self._ctr_tower_cfg.weight * ctr_loss + self._loss_dict['weighted_cross_entropy_loss_%s' % ctr_tower_name + ] = self._ctr_tower_cfg.weight * ctr_loss return self._loss_dict def build_metric_graph(self, eval_config): @@ -116,56 +122,67 @@ def build_metric_graph(self, eval_config): ctcvr_label_name = cvr_label_name + '_ctcvr' cvr_dtype = self._labels[cvr_label_name].dtype self._labels[ctcvr_label_name] = self._labels[cvr_label_name] * tf.cast( - self._labels[ctr_label_name], cvr_dtype) + self._labels[ctr_label_name], cvr_dtype + ) metric_dict.update( - self._build_metric_impl( - metric, - loss_type=self._cvr_tower_cfg.loss_type, - label_name=ctcvr_label_name, - num_class=self._cvr_tower_cfg.num_class, - suffix='_ctcvr')) + self._build_metric_impl( + metric, + loss_type=self._cvr_tower_cfg.loss_type, + label_name=ctcvr_label_name, + num_class=self._cvr_tower_cfg.num_class, + suffix='_ctcvr' + ) + ) # CVR metric cvr_label_masked_name = cvr_label_name + '_masked' ctr_mask = self._labels[ctr_label_name] > 0 self._labels[cvr_label_masked_name] = tf.boolean_mask( - self._labels[cvr_label_name], ctr_mask) + self._labels[cvr_label_name], ctr_mask + ) pred_prefix = 'probs' if self._cvr_tower_cfg.loss_type == LossType.CLASSIFICATION else 'y' pred_name = '%s_%s' % (pred_prefix, cvr_tower_name) self._prediction_dict[pred_name + '_masked'] = tf.boolean_mask( - self._prediction_dict[pred_name], ctr_mask) + self._prediction_dict[pred_name], ctr_mask + ) metric_dict.update( - self._build_metric_impl( - metric, - loss_type=self._cvr_tower_cfg.loss_type, - label_name=cvr_label_masked_name, - num_class=self._cvr_tower_cfg.num_class, - suffix='_%s_masked' % cvr_tower_name)) + self._build_metric_impl( + metric, + loss_type=self._cvr_tower_cfg.loss_type, + label_name=cvr_label_masked_name, + num_class=self._cvr_tower_cfg.num_class, + suffix='_%s_masked' % cvr_tower_name + ) + ) for metric in self._ctr_tower_cfg.metrics_set: # CTR metric metric_dict.update( - self._build_metric_impl( - metric, - loss_type=self._ctr_tower_cfg.loss_type, - label_name=ctr_label_name, - num_class=self._ctr_tower_cfg.num_class, - suffix='_%s' % ctr_tower_name)) + self._build_metric_impl( + metric, + loss_type=self._ctr_tower_cfg.loss_type, + label_name=ctr_label_name, + num_class=self._ctr_tower_cfg.num_class, + suffix='_%s' % ctr_tower_name + ) + ) return metric_dict def _add_to_prediction_dict(self, output): super(ESMM, self)._add_to_prediction_dict(output) if self._cvr_tower_cfg.loss_type == LossType.CLASSIFICATION: prob = tf.multiply( - self._prediction_dict['probs_%s' % self._cvr_tower_cfg.tower_name], - self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name]) + self._prediction_dict['probs_%s' % self._cvr_tower_cfg.tower_name], + self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name] + ) # pctcvr = pctr * pcvr self._prediction_dict['probs_ctcvr'] = prob else: prob = tf.multiply( - self._prediction_dict['y_%s' % self._cvr_tower_cfg.tower_name], - self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name]) + self._prediction_dict['y_%s' % self._cvr_tower_cfg.tower_name], + self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name] + ) # pctcvr = pctr * pcvr self._prediction_dict['y_ctcvr'] = prob @@ -184,8 +201,9 @@ def build_predict_graph(self): group_fea = self._group_features[group_id] group = self._model_config.groups[group_id] group_name = group.input - dnn_model = dnn.DNN(group.dnn, self._l2_reg, group_name, - self._is_training) + dnn_model = dnn.DNN( + group.dnn, self._l2_reg, group_name, self._is_training + ) group_fea = dnn_model(group_fea) group_fea_arr.append(group_fea) all_fea = tf.concat(group_fea_arr, axis=1) @@ -194,33 +212,37 @@ def build_predict_graph(self): cvr_tower_name = self._cvr_tower_cfg.tower_name dnn_model = dnn.DNN( - self._cvr_tower_cfg.dnn, - self._l2_reg, - name=cvr_tower_name, - is_training=self._is_training) + self._cvr_tower_cfg.dnn, + self._l2_reg, + name=cvr_tower_name, + is_training=self._is_training + ) cvr_tower_output = dnn_model(all_fea) cvr_tower_output = tf.layers.dense( - inputs=cvr_tower_output, - units=1, - kernel_regularizer=self._l2_reg, - name='%s/dnn_output' % cvr_tower_name) + inputs=cvr_tower_output, + units=1, + kernel_regularizer=self._l2_reg, + name='%s/dnn_output' % cvr_tower_name + ) ctr_tower_name = self._ctr_tower_cfg.tower_name dnn_model = dnn.DNN( - self._ctr_tower_cfg.dnn, - self._l2_reg, - name=ctr_tower_name, - is_training=self._is_training) + self._ctr_tower_cfg.dnn, + self._l2_reg, + name=ctr_tower_name, + is_training=self._is_training + ) ctr_tower_output = dnn_model(all_fea) ctr_tower_output = tf.layers.dense( - inputs=ctr_tower_output, - units=1, - kernel_regularizer=self._l2_reg, - name='%s/dnn_output' % ctr_tower_name) + inputs=ctr_tower_output, + units=1, + kernel_regularizer=self._l2_reg, + name='%s/dnn_output' % ctr_tower_name + ) tower_outputs = { - cvr_tower_name: cvr_tower_output, - ctr_tower_name: ctr_tower_output + cvr_tower_name: cvr_tower_output, + ctr_tower_name: ctr_tower_output } self._add_to_prediction_dict(tower_outputs) return self._prediction_dict @@ -237,6 +259,7 @@ def get_outputs(self): elif self._cvr_tower_cfg.loss_type == LossType.L2_LOSS: outputs.append('y_ctcvr') else: - raise ValueError('invalid cvr_tower loss type: %s' % - str(self._cvr_tower_cfg.loss_type)) + raise ValueError( + 'invalid cvr_tower loss type: %s' % str(self._cvr_tower_cfg.loss_type) + ) return outputs diff --git a/easy_rec/python/model/fm.py b/easy_rec/python/model/fm.py index 357628d3e..5ee3c216c 100644 --- a/easy_rec/python/model/fm.py +++ b/easy_rec/python/model/fm.py @@ -14,14 +14,17 @@ class FM(RankModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(FM, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(FM, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'fm', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.fm @@ -29,7 +32,8 @@ def __init__(self, self._wide_features, _ = self._input_layer(self._feature_dict, 'wide') self._deep_features, self._fm_features = self._input_layer( - self._feature_dict, 'deep') + self._feature_dict, 'deep' + ) def build_input_layer(self, model_config, feature_configs): # overwrite create input_layer to support wide_output_dim @@ -38,23 +42,26 @@ def build_input_layer(self, model_config, feature_configs): def build_predict_graph(self): wide_fea = tf.reduce_sum( - self._wide_features, axis=1, keepdims=True, name='wide_feature') + self._wide_features, axis=1, keepdims=True, name='wide_feature' + ) fm_fea = fm.FM(name='fm_feature')(self._fm_features) if self._num_class > 1: fm_fea = tf.layers.dense( - fm_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='fm_logits') + fm_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='fm_logits' + ) else: fm_fea = tf.reduce_sum(fm_fea, 1, keepdims=True) bias = tf.get_variable( - 'fm_bias', [self._num_class], - initializer=tf.zeros_initializer(), - trainable=True) + 'fm_bias', [self._num_class], + initializer=tf.zeros_initializer(), + trainable=True + ) output = wide_fea + fm_fea output = tf.nn.bias_add(output, bias) diff --git a/easy_rec/python/model/match_model.py b/easy_rec/python/model/match_model.py index e9c4d2d44..7045fac5b 100644 --- a/easy_rec/python/model/match_model.py +++ b/easy_rec/python/model/match_model.py @@ -2,7 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os - import tensorflow as tf from easy_rec.python.builders import loss_builder @@ -17,14 +16,17 @@ class MatchModel(EasyRecModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(MatchModel, self).__init__(model_config, feature_configs, features, - labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(MatchModel, self).__init__( + model_config, feature_configs, features, labels, is_training + ) self._loss_type = self._model_config.loss_type self._num_class = self._model_config.num_class @@ -51,16 +53,20 @@ def _mask_in_batch(self, logits): batch_size = tf.shape(logits)[0] if getattr(self._model_config, 'ignore_in_batch_neg_sam', False): in_batch = logits[:, :batch_size] - ( - 1 - tf.diag(tf.ones([batch_size], dtype=tf.float32))) * 1e32 + 1 - tf.diag(tf.ones([batch_size], dtype=tf.float32)) + ) * 1e32 return tf.concat([in_batch, logits[:, batch_size:]], axis=1) else: if self._item_ids is not None: mask_in_batch_neg = tf.to_float( - tf.equal(self._item_ids[None, :batch_size], - self._item_ids[:batch_size, None])) - tf.diag( - tf.ones([batch_size], dtype=tf.float32)) - tf.summary.scalar('in_batch_neg_conflict', - tf.reduce_sum(mask_in_batch_neg)) + tf.equal( + self._item_ids[None, :batch_size], self._item_ids[:batch_size, + None] + ) + ) - tf.diag(tf.ones([batch_size], dtype=tf.float32)) + tf.summary.scalar( + 'in_batch_neg_conflict', tf.reduce_sum(mask_in_batch_neg) + ) return tf.concat([ logits[:, :batch_size] - mask_in_batch_neg * 1e32, logits[:, batch_size:]], @@ -78,7 +84,8 @@ def _list_wise_sim(self, user_emb, item_emb): # pos_item_emb, neg_item_emb, hard_neg_item_emb = tf.split( # item_emb, [batch_size, -1, noclk_size], axis=0) simple_item_emb, hard_neg_item_emb = tf.split( - item_emb, [-1, noclk_size], axis=0) + item_emb, [-1, noclk_size], axis=0 + ) else: # pos_item_emb = item_emb[:batch_size] # neg_item_emb = item_emb[batch_size:] @@ -92,7 +99,8 @@ def _list_wise_sim(self, user_emb, item_emb): _mode = os.environ['tf.estimator.mode'] if _mode == tf.estimator.ModeKeys.PREDICT: simple_user_item_sim = tf.reduce_sum( - tf.multiply(user_emb, simple_item_emb), axis=1, keep_dims=True) + tf.multiply(user_emb, simple_item_emb), axis=1, keep_dims=True + ) else: simple_user_item_sim = tf.matmul(user_emb, tf.transpose(simple_item_emb)) @@ -101,15 +109,18 @@ def _list_wise_sim(self, user_emb, item_emb): else: user_emb_expand = tf.gather(user_emb, hard_neg_indices[:, 0]) hard_neg_user_item_sim = tf.reduce_sum( - tf.multiply(user_emb_expand, hard_neg_item_emb), axis=1) + tf.multiply(user_emb_expand, hard_neg_item_emb), axis=1 + ) max_num_neg = tf.reduce_max(hard_neg_indices[:, 1]) + 1 hard_neg_shape = tf.stack([tf.to_int64(batch_size), max_num_neg]) - hard_neg_sim = tf.scatter_nd(hard_neg_indices, hard_neg_user_item_sim, - hard_neg_shape) + hard_neg_sim = tf.scatter_nd( + hard_neg_indices, hard_neg_user_item_sim, hard_neg_shape + ) hard_neg_mask = tf.scatter_nd( - hard_neg_indices, - tf.ones_like(hard_neg_user_item_sim, dtype=tf.float32), - shape=hard_neg_shape) + hard_neg_indices, + tf.ones_like(hard_neg_user_item_sim, dtype=tf.float32), + shape=hard_neg_shape + ) # set tail positions to -1e32, so that after exp(x), will be zero hard_neg_user_item_sim = hard_neg_sim - (1 - hard_neg_mask) * 1e32 @@ -122,7 +133,8 @@ def _list_wise_sim(self, user_emb, item_emb): def _point_wise_sim(self, user_emb, item_emb): user_item_sim = tf.reduce_sum( - tf.multiply(user_emb, item_emb), axis=1, keep_dims=True) + tf.multiply(user_emb, item_emb), axis=1, keep_dims=True + ) return user_item_sim def sim(self, user_emb, item_emb): @@ -145,7 +157,7 @@ def norm(self, fea): def build_predict_graph(self): if not self.has_backbone: raise NotImplementedError( - 'method `build_predict_graph` must be implemented when you donot use backbone network' + 'method `build_predict_graph` must be implemented when you donot use backbone network' ) model = self._model_config.WhichOneof('model') assert model == 'model_params', '`model_params` must be configured' @@ -169,15 +181,17 @@ def build_predict_graph(self): if model_params.scale_simi: sim_w = tf.get_variable( - 'sim_w', - dtype=tf.float32, - shape=(1), - initializer=tf.ones_initializer()) + 'sim_w', + dtype=tf.float32, + shape=(1), + initializer=tf.ones_initializer() + ) sim_b = tf.get_variable( - 'sim_b', - dtype=tf.float32, - shape=(1), - initializer=tf.zeros_initializer()) + 'sim_b', + dtype=tf.float32, + shape=(1), + initializer=tf.zeros_initializer() + ) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -198,9 +212,11 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb self._prediction_dict['user_emb'] = tf.reduce_join( - tf.as_string(user_tower_emb), axis=-1, separator=',') + tf.as_string(user_tower_emb), axis=-1, separator=',' + ) self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_tower_emb), axis=-1, separator=',') + tf.as_string(item_tower_emb), axis=-1, separator=',' + ) return self._prediction_dict @@ -216,12 +232,13 @@ def _build_list_wise_loss_graph(self): indices = tf.range(batch_size) indices = tf.concat([indices[:, None], indices[:, None]], axis=1) hit_prob = tf.gather_nd( - self._prediction_dict['probs'][:batch_size, :batch_size], indices) + self._prediction_dict['probs'][:batch_size, :batch_size], indices + ) sample_weights = tf.cast(tf.squeeze(self._sample_weight), tf.float32) self._loss_dict['cross_entropy_loss'] = -tf.reduce_mean( - tf.log(hit_prob + 1e-12) * - sample_weights) / tf.reduce_mean(sample_weights) + tf.log(hit_prob + 1e-12) * sample_weights + ) / tf.reduce_mean(sample_weights) logging.info('softmax cross entropy loss is used') @@ -231,23 +248,30 @@ def _build_list_wise_loss_graph(self): # if pos_simi < 0, produce loss reg_pos_loss = tf.nn.relu(-pos_simi) self._loss_dict['reg_pos_loss'] = tf.reduce_mean( - reg_pos_loss * sample_weights) / tf.reduce_mean(sample_weights) + reg_pos_loss * sample_weights + ) / tf.reduce_mean(sample_weights) # the AMM loss for DAT model - if all([ + if all( + [ k in self._prediction_dict.keys() for k in ['augmented_p_u', 'augmented_p_i', 'augmented_a_u', 'augmented_a_i'] - ]): - self._loss_dict[ - 'amm_loss_u'] = self._model_config.amm_u_weight * tf.reduce_mean( - tf.square(self._prediction_dict['augmented_a_u'] - - self._prediction_dict['augmented_p_i'][:batch_size]) * - sample_weights) / tf.reduce_mean(sample_weights) + ] + ): + self._loss_dict['amm_loss_u' + ] = self._model_config.amm_u_weight * tf.reduce_mean( + tf.square( + self._prediction_dict['augmented_a_u'] - + self._prediction_dict['augmented_p_i'][:batch_size] + ) * sample_weights + ) / tf.reduce_mean(sample_weights) self._loss_dict[ - 'amm_loss_i'] = self._model_config.amm_i_weight * tf.reduce_mean( - tf.square(self._prediction_dict['augmented_a_i'][:batch_size] - - self._prediction_dict['augmented_p_u']) * - sample_weights) / tf.reduce_mean(sample_weights) + 'amm_loss_i'] = self._model_config.amm_i_weight * tf.reduce_mean( + tf.square( + self._prediction_dict['augmented_a_i'][:batch_size] - + self._prediction_dict['augmented_p_u'] + ) * sample_weights + ) / tf.reduce_mean(sample_weights) else: raise ValueError('invalid loss type: %s' % str(self._loss_type)) @@ -266,15 +290,17 @@ def _build_point_wise_loss_graph(self): kwargs = {'loss_name': loss_name} self._loss_dict[loss_name] = loss_builder.build( - self._loss_type, - label=label, - pred=pred, - loss_weight=self._sample_weight, - **kwargs) + self._loss_type, + label=label, + pred=pred, + loss_weight=self._sample_weight, + **kwargs + ) # build kd loss - kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, - self._labels, self._feature_dict) + kd_loss_dict = loss_builder.build_kd_loss( + self.kd, self._prediction_dict, self._labels, self._feature_dict + ) self._loss_dict.update(kd_loss_dict) return self._loss_dict @@ -297,21 +323,23 @@ def _build_list_wise_metric_graph(self, eval_config): metric_dict = {} for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'recall_at_topk': - metric_dict['recall@%d' % - metric.recall_at_topk.topk] = metrics_tf.recall_at_k( - label, logits, metric.recall_at_topk.topk) + metric_dict[ + 'recall@%d' % metric.recall_at_topk.topk + ] = metrics_tf.recall_at_k(label, logits, metric.recall_at_topk.topk) - logits_v2 = tf.concat([pos_item_sim[:, None], logits[:, batch_size:]], - axis=1) + logits_v2 = tf.concat( + [pos_item_sim[:, None], logits[:, batch_size:]], axis=1 + ) labels_v2 = tf.zeros_like(logits_v2[:, :1], dtype=tf.int64) - metric_dict['recall_neg_sam@%d' % - metric.recall_at_topk.topk] = metrics_tf.recall_at_k( - labels_v2, logits_v2, metric.recall_at_topk.topk) - - metric_dict['recall_in_batch@%d' % - metric.recall_at_topk.topk] = metrics_tf.recall_at_k( - label, logits[:, :batch_size], - metric.recall_at_topk.topk) + metric_dict['recall_neg_sam@%d' % metric.recall_at_topk.topk + ] = metrics_tf.recall_at_k( + labels_v2, logits_v2, metric.recall_at_topk.topk + ) + + metric_dict['recall_in_batch@%d' % metric.recall_at_topk.topk + ] = metrics_tf.recall_at_k( + label, logits[:, :batch_size], metric.recall_at_topk.topk + ) else: raise ValueError('invalid metric type: %s' % str(metric)) return metric_dict @@ -323,12 +351,14 @@ def _build_point_wise_metric_graph(self, eval_config): for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'auc': assert self._loss_type == LossType.CLASSIFICATION - metric_dict['auc'] = metrics_tf.auc(label, - self._prediction_dict['probs']) + metric_dict['auc'] = metrics_tf.auc( + label, self._prediction_dict['probs'] + ) elif metric.WhichOneof('metric') == 'mean_absolute_error': assert self._loss_type == LossType.L2_LOSS metric_dict['mean_absolute_error'] = metrics_tf.mean_absolute_error( - tf.to_float(label), self._prediction_dict['y']) + tf.to_float(label), self._prediction_dict['y'] + ) else: raise ValueError('invalid metric type: %s' % str(metric)) return metric_dict @@ -336,20 +366,23 @@ def _build_point_wise_metric_graph(self, eval_config): def get_outputs(self): if not self.has_backbone: raise NotImplementedError( - 'could not call get_outputs on abstract class MatchModel') + 'could not call get_outputs on abstract class MatchModel' + ) if self._loss_type == LossType.CLASSIFICATION: return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' + 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', + 'item_tower_emb' ] elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: self._prediction_dict['logits'] = tf.squeeze( - self._prediction_dict['logits'], axis=-1) + self._prediction_dict['logits'], axis=-1 + ) self._prediction_dict['probs'] = tf.nn.sigmoid( - self._prediction_dict['logits']) + self._prediction_dict['logits'] + ) return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' + 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', + 'item_tower_emb' ] elif self._loss_type == LossType.L2_LOSS: return ['y', 'user_emb', 'item_emb', 'user_tower_emb', 'item_tower_emb'] diff --git a/easy_rec/python/model/mind.py b/easy_rec/python/model/mind.py index 0e47f79d8..943f6f620 100644 --- a/easy_rec/python/model/mind.py +++ b/easy_rec/python/model/mind.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.compat import regularizers @@ -20,20 +19,24 @@ class MIND(MatchModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(MIND, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(MIND, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'mind', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.mind self._hist_seq_features, _, _ = self._input_layer( - self._feature_dict, 'hist', is_combine=False) + self._feature_dict, 'hist', is_combine=False + ) self._user_features, _ = self._input_layer(self._feature_dict, 'user') self._item_features, _ = self._input_layer(self._feature_dict, 'item') @@ -45,29 +48,31 @@ def __init__(self, self.concat_dnn = copy_obj(self._model_config.concat_dnn) self._l2_reg = regularizers.l2_regularizer( - self._model_config.l2_regularization) + self._model_config.l2_regularization + ) def build_predict_graph(self): - capsule_layer = CapsuleLayer(self._model_config.capsule_config, - self._is_training) + capsule_layer = CapsuleLayer( + self._model_config.capsule_config, self._is_training + ) if self._model_config.time_id_fea: time_id_fea = [ - x[0] - for x in self._hist_seq_features - if self._model_config.time_id_fea in x[0].name + x[0] for x in self._hist_seq_features + if self._model_config.time_id_fea in x[0].name ] - logging.info('time_id_fea is set(%s), find num: %d' % - (self._model_config.time_id_fea, len(time_id_fea))) + logging.info( + 'time_id_fea is set(%s), find num: %d' % + (self._model_config.time_id_fea, len(time_id_fea)) + ) else: time_id_fea = [] time_id_fea = time_id_fea[0] if len(time_id_fea) > 0 else None if time_id_fea is not None: hist_seq_feas = [ - x[0] - for x in self._hist_seq_features - if self._model_config.time_id_fea not in x[0].name + x[0] for x in self._hist_seq_features + if self._model_config.time_id_fea not in x[0].name ] else: hist_seq_feas = [x[0] for x in self._hist_seq_features] @@ -88,8 +93,10 @@ def build_predict_graph(self): if self._model_config.HasField('pre_capsule_dnn') and \ len(self._model_config.pre_capsule_dnn.hidden_units) > 0: - pre_dnn_layer = dnn.DNN(self._model_config.pre_capsule_dnn, self._l2_reg, - 'pre_capsule_dnn', self._is_training) + pre_dnn_layer = dnn.DNN( + self._model_config.pre_capsule_dnn, self._l2_reg, 'pre_capsule_dnn', + self._is_training + ) hist_seq_feas = pre_dnn_layer(hist_seq_feas) if time_id_fea is not None: @@ -103,8 +110,9 @@ def build_predict_graph(self): tf.summary.histogram('hist_seq_len', hist_seq_len) # batch_size x max_k x high_capsule_dim - high_capsules, num_high_capsules = capsule_layer(hist_seq_feas, - hist_seq_len) + high_capsules, num_high_capsules = capsule_layer( + hist_seq_feas, hist_seq_len + ) tf.summary.histogram('num_high_capsules', num_high_capsules) @@ -113,52 +121,63 @@ def build_predict_graph(self): # trainable=True, name='capsule_bn') # high_capsules = high_capsules * 0.1 - tf.summary.scalar('high_capsules_norm', - tf.reduce_mean(tf.norm(high_capsules, axis=-1))) - tf.summary.scalar('num_high_capsules', - tf.reduce_mean(tf.to_float(num_high_capsules))) + tf.summary.scalar( + 'high_capsules_norm', tf.reduce_mean(tf.norm(high_capsules, axis=-1)) + ) + tf.summary.scalar( + 'num_high_capsules', tf.reduce_mean(tf.to_float(num_high_capsules)) + ) user_features = tf.layers.batch_normalization( - self._user_features, - training=self._is_training, - trainable=True, - name='user_fea_bn') - user_dnn = dnn.DNN(self.user_dnn, self._l2_reg, 'user_dnn', - self._is_training) + self._user_features, + training=self._is_training, + trainable=True, + name='user_fea_bn' + ) + user_dnn = dnn.DNN( + self.user_dnn, self._l2_reg, 'user_dnn', self._is_training + ) user_features = user_dnn(user_features) - tf.summary.scalar('user_features_norm', - tf.reduce_mean(tf.norm(self._user_features, axis=-1))) + tf.summary.scalar( + 'user_features_norm', + tf.reduce_mean(tf.norm(self._user_features, axis=-1)) + ) # concatenate with user features - user_features_tile = tf.tile(user_features[:, None, :], - [1, tf.shape(high_capsules)[1], 1]) + user_features_tile = tf.tile( + user_features[:, None, :], [1, tf.shape(high_capsules)[1], 1] + ) user_interests = tf.concat([high_capsules, user_features_tile], axis=2) num_concat_dnn_layer = len(self.concat_dnn.hidden_units) last_hidden = self.concat_dnn.hidden_units.pop() - concat_dnn = dnn.DNN(self.concat_dnn, self._l2_reg, 'concat_dnn', - self._is_training) + concat_dnn = dnn.DNN( + self.concat_dnn, self._l2_reg, 'concat_dnn', self._is_training + ) user_interests = concat_dnn(user_interests) user_interests = tf.layers.dense( - inputs=user_interests, - units=last_hidden, - kernel_regularizer=self._l2_reg, - name='concat_dnn/dnn_%d' % (num_concat_dnn_layer - 1)) + inputs=user_interests, + units=last_hidden, + kernel_regularizer=self._l2_reg, + name='concat_dnn/dnn_%d' % (num_concat_dnn_layer - 1) + ) num_item_dnn_layer = len(self.item_dnn.hidden_units) last_item_hidden = self.item_dnn.hidden_units.pop() - item_dnn = dnn.DNN(self.item_dnn, self._l2_reg, 'item_dnn', - self._is_training) + item_dnn = dnn.DNN( + self.item_dnn, self._l2_reg, 'item_dnn', self._is_training + ) item_tower_emb = item_dnn(self._item_features) item_tower_emb = tf.layers.dense( - inputs=item_tower_emb, - units=last_item_hidden, - kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1)) + inputs=item_tower_emb, + units=last_item_hidden, + kernel_regularizer=self._l2_reg, + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1) + ) assert self._model_config.simi_func in [ - Similarity.COSINE, Similarity.INNER_PRODUCT + Similarity.COSINE, Similarity.INNER_PRODUCT ] if self._model_config.simi_func == Similarity.COSINE: @@ -170,16 +189,19 @@ def build_predict_graph(self): batch_size = tf.shape(user_interests)[0] pos_item_fea = item_tower_emb[:batch_size] simi = tf.einsum('bhe,be->bh', user_interests, pos_item_fea) - tf.summary.histogram('interest_item_simi/pre_scale', - tf.reduce_max(simi, axis=1)) + tf.summary.histogram( + 'interest_item_simi/pre_scale', tf.reduce_max(simi, axis=1) + ) # simi = tf.Print(simi, [tf.reduce_max(simi, axis=1), tf.reduce_min(simi, axis=1)], message='simi_max_0') # simi = tf.pow(simi, self._model_config.simi_pow) simi = simi * self._model_config.simi_pow - tf.summary.histogram('interest_item_simi/scaled', - tf.reduce_max(simi, axis=1)) + tf.summary.histogram( + 'interest_item_simi/scaled', tf.reduce_max(simi, axis=1) + ) # simi = tf.Print(simi, [tf.reduce_max(simi, axis=1), tf.reduce_min(simi, axis=1)], message='simi_max') - simi_mask = tf.sequence_mask(num_high_capsules, - self._model_config.capsule_config.max_k) + simi_mask = tf.sequence_mask( + num_high_capsules, self._model_config.capsule_config.max_k + ) user_interests = user_interests * tf.to_float(simi_mask[:, :, None]) self._prediction_dict['user_interests'] = user_interests @@ -187,13 +209,15 @@ def build_predict_graph(self): max_thresh = (tf.cast(simi_mask, tf.float32) * 2 - 1) * 1e32 simi = tf.minimum(simi, max_thresh) simi = tf.nn.softmax(simi, axis=1) - tf.summary.histogram('interest_item_simi/softmax', - tf.reduce_max(simi, axis=1)) + tf.summary.histogram( + 'interest_item_simi/softmax', tf.reduce_max(simi, axis=1) + ) if self._model_config.simi_pow >= 100: logging.info( - 'simi_pow=%d, will change to argmax, only use the most similar interests for calculate loss.' - % self._model_config.simi_pow) + 'simi_pow=%d, will change to argmax, only use the most similar interests for calculate loss.' + % self._model_config.simi_pow + ) simi_max_id = tf.argmax(simi, axis=1) simi = tf.one_hot(simi_max_id, tf.shape(simi)[1], dtype=tf.float32) @@ -203,15 +227,17 @@ def build_predict_graph(self): user_item_sim = self.sim(user_tower_emb, item_tower_emb) if self._model_config.scale_simi: sim_w = tf.get_variable( - 'sim_w', - dtype=tf.float32, - shape=(1), - initializer=tf.ones_initializer()) + 'sim_w', + dtype=tf.float32, + shape=(1), + initializer=tf.ones_initializer() + ) sim_b = tf.get_variable( - 'sim_b', - dtype=tf.float32, - shape=(1), - initializer=tf.zeros_initializer()) + 'sim_b', + dtype=tf.float32, + shape=(1), + initializer=tf.zeros_initializer() + ) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -234,12 +260,14 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb self._prediction_dict['user_emb'] = tf.reduce_join( - tf.reduce_join(tf.as_string(user_interests), axis=-1, separator=','), - axis=-1, - separator='|') + tf.reduce_join(tf.as_string(user_interests), axis=-1, separator=','), + axis=-1, + separator='|' + ) self._prediction_dict['user_emb_num'] = num_high_capsules self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_tower_emb), axis=-1, separator=',') + tf.as_string(item_tower_emb), axis=-1, separator=',' + ) if self._labels is not None: # for summary purpose @@ -253,14 +281,16 @@ def build_loss_graph(self): loss_dict = super(MIND, self).build_loss_graph() if self._model_config.max_interests_simi < 1.0: loss_dict['reg_interest_simi'] = tf.nn.relu( - self._prediction_dict['interests_simi'] - - self._model_config.max_interests_simi) + self._prediction_dict['interests_simi'] - + self._model_config.max_interests_simi + ) return loss_dict def _build_interest_simi(self): user_emb_num = self._prediction_dict['user_emb_num'] high_capsule_mask = tf.sequence_mask( - user_emb_num, self._model_config.capsule_config.max_k) + user_emb_num, self._model_config.capsule_config.max_k + ) user_interests = self._prediction_dict['user_interests'] high_capsule_mask = tf.to_float(high_capsule_mask[:, :, None]) @@ -278,7 +308,8 @@ def _build_interest_simi(self): # normalize by interest number interest_div = tf.maximum( - tf.to_float(user_emb_num * (user_emb_num - 1)), 1.0) + tf.to_float(user_emb_num * (user_emb_num - 1)), 1.0 + ) interest_simi = tf.reduce_sum(interest_simi, axis=1) / interest_div high_capsule_simi = tf.reduce_sum(high_capsule_simi, axis=1) / interest_div @@ -286,12 +317,14 @@ def _build_interest_simi(self): # normalize by batch_size multi_interest = tf.to_float(user_emb_num > 1) sum_interest_simi = tf.reduce_sum( - (interest_simi + 1) * multi_interest) / 2.0 + (interest_simi + 1) * multi_interest + ) / 2.0 sum_div = tf.maximum(tf.reduce_sum(multi_interest), 1.0) avg_interest_simi = sum_interest_simi / sum_div sum_capsule_simi = tf.reduce_sum( - (high_capsule_simi + 1) * multi_interest) / 2.0 + (high_capsule_simi + 1) * multi_interest + ) / 2.0 avg_capsule_simi = sum_capsule_simi / sum_div tf.summary.scalar('interest_similarity', avg_interest_simi) @@ -300,11 +333,12 @@ def _build_interest_simi(self): def build_metric_graph(self, eval_config): from easy_rec.python.core.easyrec_metrics import metrics_tf as metrics + # build interest metric interest_simi, capsule_simi = self._build_interest_simi() metric_dict = { - 'interest_similarity': metrics.mean(interest_simi), - 'capsule_similarity': metrics.mean(capsule_simi) + 'interest_similarity': metrics.mean(interest_simi), + 'capsule_similarity': metrics.mean(capsule_simi) } if self._is_point_wise: metric_dict.update(self._build_point_wise_metric_graph(eval_config)) @@ -314,7 +348,7 @@ def build_metric_graph(self, eval_config): for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'recall_at_topk': assert self._loss_type in [ - LossType.CLASSIFICATION, LossType.SOFTMAX_CROSS_ENTROPY + LossType.CLASSIFICATION, LossType.SOFTMAX_CROSS_ENTROPY ] if metric.recall_at_topk.topk not in recall_at_topks: recall_at_topks.append(metric.recall_at_topk.topk) @@ -333,7 +367,8 @@ def build_metric_graph(self, eval_config): logging.info('With hard negative examples') noclk_size = tf.shape(hard_neg_indices)[0] simple_item_emb, hard_neg_item_emb = tf.split( - item_tower_emb, [-1, noclk_size], axis=0) + item_tower_emb, [-1, noclk_size], axis=0 + ) else: simple_item_emb = item_tower_emb hard_neg_item_emb = None @@ -346,100 +381,118 @@ def build_metric_graph(self, eval_config): # labels = tf.zeros_like(logits[:, :1], dtype=tf.int64) pos_indices = tf.range(batch_size) - pos_indices = tf.concat([pos_indices[:, None], pos_indices[:, None]], - axis=1) - pos_item_sim = tf.gather_nd(simple_item_sim[:batch_size, :batch_size], - pos_indices) + pos_indices = tf.concat( + [pos_indices[:, None], pos_indices[:, None]], axis=1 + ) + pos_item_sim = tf.gather_nd( + simple_item_sim[:batch_size, :batch_size], pos_indices + ) simple_item_sim_v2 = tf.concat( - [pos_item_sim[:, None], simple_item_sim[:, batch_size:]], axis=1) + [pos_item_sim[:, None], simple_item_sim[:, batch_size:]], axis=1 + ) simple_lbls_v2 = tf.zeros_like(simple_item_sim_v2[:, :1], dtype=tf.int64) for topk in recall_at_topks: metric_dict['interests_recall@%d' % topk] = metrics.recall_at_k( - labels=simple_lbls, - predictions=simple_item_sim, - k=topk, - name='interests_recall_at_%d' % topk) + labels=simple_lbls, + predictions=simple_item_sim, + k=topk, + name='interests_recall_at_%d' % topk + ) metric_dict['interests_neg_sam_recall@%d' % topk] = metrics.recall_at_k( - labels=simple_lbls_v2, - predictions=simple_item_sim_v2, - k=topk, - name='interests_recall_neg_sam_at_%d' % topk) + labels=simple_lbls_v2, + predictions=simple_item_sim_v2, + k=topk, + name='interests_recall_neg_sam_at_%d' % topk + ) logits = self._prediction_dict['logits'] - pos_item_logits = tf.gather_nd(logits[:batch_size, :batch_size], - pos_indices) - logits_v2 = tf.concat([pos_item_logits[:, None], logits[:, batch_size:]], - axis=1) + pos_item_logits = tf.gather_nd( + logits[:batch_size, :batch_size], pos_indices + ) + logits_v2 = tf.concat( + [pos_item_logits[:, None], logits[:, batch_size:]], axis=1 + ) labels_v2 = tf.zeros_like(logits_v2[:, :1], dtype=tf.int64) for topk in recall_at_topks: metric_dict['recall@%d' % topk] = metrics.recall_at_k( - labels=simple_lbls, - predictions=logits, - k=topk, - name='recall_at_%d' % topk) + labels=simple_lbls, + predictions=logits, + k=topk, + name='recall_at_%d' % topk + ) metric_dict['recall_neg_sam@%d' % topk] = metrics.recall_at_k( - labels=labels_v2, - predictions=logits_v2, - k=topk, - name='recall_neg_sam_at_%d' % topk) + labels=labels_v2, + predictions=logits_v2, + k=topk, + name='recall_neg_sam_at_%d' % topk + ) eval_logits = logits[:, :batch_size] eval_logits = tf.cond( - batch_size < topk, lambda: tf.pad( - eval_logits, [[0, 0], [0, topk - batch_size]], - mode='CONSTANT', - constant_values=-1e32, - name='pad_eval_logits'), lambda: eval_logits) + batch_size < topk, lambda: tf.pad( + eval_logits, [[0, 0], [0, topk - batch_size]], + mode='CONSTANT', + constant_values=-1e32, + name='pad_eval_logits' + ), lambda: eval_logits + ) metric_dict['recall_in_batch@%d' % topk] = metrics.recall_at_k( - labels=simple_lbls, - predictions=eval_logits, - k=topk, - name='recall_in_batch_at_%d' % topk) + labels=simple_lbls, + predictions=eval_logits, + k=topk, + name='recall_in_batch_at_%d' % topk + ) # batch_size num_interest if hard_neg_indices is not None: hard_neg_user_emb = tf.gather(user_interests, hard_neg_indices[:, 0]) - hard_neg_sim = tf.einsum('nhe,ne->nh', hard_neg_user_emb, - hard_neg_item_emb) + hard_neg_sim = tf.einsum( + 'nhe,ne->nh', hard_neg_user_emb, hard_neg_item_emb + ) hard_neg_sim = tf.reduce_max(hard_neg_sim, axis=1) max_num_neg = tf.reduce_max(hard_neg_indices[:, 1]) + 1 hard_neg_shape = tf.stack([tf.to_int64(batch_size), max_num_neg]) hard_neg_mask = tf.scatter_nd( - hard_neg_indices, - tf.ones_like(hard_neg_sim, dtype=tf.float32), - shape=hard_neg_shape) - hard_neg_sim = tf.scatter_nd(hard_neg_indices, hard_neg_sim, - hard_neg_shape) + hard_neg_indices, + tf.ones_like(hard_neg_sim, dtype=tf.float32), + shape=hard_neg_shape + ) + hard_neg_sim = tf.scatter_nd( + hard_neg_indices, hard_neg_sim, hard_neg_shape + ) hard_neg_sim = hard_neg_sim - (1 - hard_neg_mask) * 1e32 hard_logits = tf.concat([pos_item_logits[:, None], hard_neg_sim], axis=1) hard_lbls = tf.zeros_like(hard_logits[:, :1], dtype=tf.int64) metric_dict['hard_neg_acc'] = metrics.accuracy( - hard_lbls, tf.argmax(hard_logits, axis=1)) + hard_lbls, tf.argmax(hard_logits, axis=1) + ) return metric_dict def get_outputs(self): if self._loss_type == LossType.CLASSIFICATION: return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_emb_num', - 'user_interests', 'item_tower_emb' + 'logits', 'probs', 'user_emb', 'item_emb', 'user_emb_num', + 'user_interests', 'item_tower_emb' ] elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: self._prediction_dict['logits'] = tf.squeeze( - self._prediction_dict['logits'], axis=-1) + self._prediction_dict['logits'], axis=-1 + ) self._prediction_dict['probs'] = tf.nn.sigmoid( - self._prediction_dict['logits']) + self._prediction_dict['logits'] + ) return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_emb_num', - 'user_interests', 'item_tower_emb' + 'logits', 'probs', 'user_emb', 'item_emb', 'user_emb_num', + 'user_interests', 'item_tower_emb' ] elif self._loss_type == LossType.L2_LOSS: return [ - 'y', 'user_emb', 'item_emb', 'user_emb_num', 'user_interests', - 'item_tower_emb' + 'y', 'user_emb', 'item_emb', 'user_emb_num', 'user_interests', + 'item_tower_emb' ] else: raise ValueError('invalid loss type: %s' % str(self._loss_type)) diff --git a/easy_rec/python/model/mmoe.py b/easy_rec/python/model/mmoe.py index 3cc644f6d..f432eacf7 100644 --- a/easy_rec/python/model/mmoe.py +++ b/easy_rec/python/model/mmoe.py @@ -2,8 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import dnn -from easy_rec.python.layers import mmoe +from easy_rec.python.layers import dnn, mmoe from easy_rec.python.model.multi_task_model import MultiTaskModel from easy_rec.python.protos.mmoe_pb2 import MMoE as MMoEConfig @@ -13,14 +12,17 @@ class MMoE(MultiTaskModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(MMoE, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(MMoE, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'mmoe', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.mmoe @@ -35,15 +37,18 @@ def __init__(self, def build_predict_graph(self): if self._model_config.HasField('expert_dnn'): mmoe_layer = mmoe.MMOE( - self._model_config.expert_dnn, - l2_reg=self._l2_reg, - num_task=self._task_num, - num_expert=self._model_config.num_expert) + self._model_config.expert_dnn, + l2_reg=self._l2_reg, + num_task=self._task_num, + num_expert=self._model_config.num_expert + ) else: # For backward compatibility with original mmoe layer config - mmoe_layer = mmoe.MMOE([x.dnn for x in self._model_config.experts], - l2_reg=self._l2_reg, - num_task=self._task_num) + mmoe_layer = mmoe.MMOE( + [x.dnn for x in self._model_config.experts], + l2_reg=self._l2_reg, + num_task=self._task_num + ) task_input_list = mmoe_layer(self._features) tower_outputs = {} @@ -52,18 +57,20 @@ def build_predict_graph(self): if task_tower_cfg.HasField('dnn'): tower_dnn = dnn.DNN( - task_tower_cfg.dnn, - self._l2_reg, - name=tower_name, - is_training=self._is_training) + task_tower_cfg.dnn, + self._l2_reg, + name=tower_name, + is_training=self._is_training + ) tower_output = tower_dnn(task_input_list[i]) else: tower_output = task_input_list[i] tower_output = tf.layers.dense( - inputs=tower_output, - units=task_tower_cfg.num_class, - kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i) + inputs=tower_output, + units=task_tower_cfg.num_class, + kernel_regularizer=self._l2_reg, + name='dnn_output_%d' % i + ) tower_outputs[tower_name] = tower_output self._add_to_prediction_dict(tower_outputs) diff --git a/easy_rec/python/model/multi_task_model.py b/easy_rec/python/model/multi_task_model.py index aa102104c..83c1be50d 100644 --- a/easy_rec/python/model/multi_task_model.py +++ b/easy_rec/python/model/multi_task_model.py @@ -1,9 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging -from collections import OrderedDict - import tensorflow as tf +from collections import OrderedDict from easy_rec.python.builders import loss_builder from easy_rec.python.layers.dnn import DNN @@ -18,14 +17,17 @@ class MultiTaskModel(RankModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(MultiTaskModel, self).__init__(model_config, feature_configs, - features, labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(MultiTaskModel, self).__init__( + model_config, feature_configs, features, labels, is_training + ) self._task_towers = [] self._task_num = None self._label_name_dict = {} @@ -33,7 +35,7 @@ def __init__(self, def build_predict_graph(self): if not self.has_backbone: raise NotImplementedError( - 'method `build_predict_graph` must be implemented when backbone network do not exists' + 'method `build_predict_graph` must be implemented when backbone network do not exists' ) model = self._model_config.WhichOneof('model') assert model == 'model_params', '`model_params` must be configured' @@ -47,7 +49,8 @@ def build_predict_graph(self): if type(backbone) in (list, tuple): if len(backbone) != len(config.task_towers): raise ValueError( - 'The number of backbone outputs and task towers must be equal') + 'The number of backbone outputs and task towers must be equal' + ) task_input_list = backbone else: task_input_list = [backbone] * len(config.task_towers) @@ -58,10 +61,11 @@ def build_predict_graph(self): with tf.name_scope(tower_name): if task_tower_cfg.HasField('dnn'): tower_dnn = DNN( - task_tower_cfg.dnn, - self._l2_reg, - name=tower_name, - is_training=self._is_training) + task_tower_cfg.dnn, + self._l2_reg, + name=tower_name, + is_training=self._is_training + ) tower_output = tower_dnn(task_input_list[i]) else: tower_output = task_input_list[i] @@ -75,25 +79,28 @@ def build_predict_graph(self): with tf.name_scope(tower_name): if task_tower_cfg.HasField('relation_dnn'): relation_dnn = DNN( - task_tower_cfg.relation_dnn, - self._l2_reg, - name=tower_name + '/relation_dnn', - is_training=self._is_training) + task_tower_cfg.relation_dnn, + self._l2_reg, + name=tower_name + '/relation_dnn', + is_training=self._is_training + ) tower_inputs = [tower_features[tower_name]] for relation_tower_name in task_tower_cfg.relation_tower_names: tower_inputs.append(relation_features[relation_tower_name]) relation_input = tf.concat( - tower_inputs, axis=-1, name=tower_name + '/relation_input') + tower_inputs, axis=-1, name=tower_name + '/relation_input' + ) relation_fea = relation_dnn(relation_input) relation_features[tower_name] = relation_fea else: relation_fea = tower_features[tower_name] output_logits = tf.layers.dense( - relation_fea, - task_tower_cfg.num_class, - kernel_regularizer=self._l2_reg, - name=tower_name + '/output') + relation_fea, + task_tower_cfg.num_class, + kernel_regularizer=self._l2_reg, + name=tower_name + '/output' + ) tower_outputs[tower_name] = output_logits self._add_to_prediction_dict(tower_outputs) @@ -116,8 +123,9 @@ def _init_towers(self, task_tower_configs): else: # If label name is not specified, task_tower and label will be matched by order label_name = list(self._labels.keys())[i] - logging.info('Task Tower [%s] use label [%s]' % - (tower_name, label_name)) + logging.info( + 'Task Tower [%s] use label [%s]' % (tower_name, label_name) + ) assert label_name in self._labels, 'label [%s] must exists in labels' % label_name self._label_name_dict[tower_name] = label_name @@ -126,19 +134,23 @@ def _add_to_prediction_dict(self, output): tower_name = task_tower_cfg.tower_name if len(task_tower_cfg.losses) == 0: self._prediction_dict.update( - self._output_to_prediction_impl( - output[tower_name], - loss_type=task_tower_cfg.loss_type, - num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name)) + self._output_to_prediction_impl( + output[tower_name], + loss_type=task_tower_cfg.loss_type, + num_class=task_tower_cfg.num_class, + suffix='_%s' % tower_name + ) + ) else: for loss in task_tower_cfg.losses: self._prediction_dict.update( - self._output_to_prediction_impl( - output[tower_name], - loss_type=loss.loss_type, - num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name)) + self._output_to_prediction_impl( + output[tower_name], + loss_type=loss.loss_type, + num_class=task_tower_cfg.num_class, + suffix='_%s' % tower_name + ) + ) def build_metric_graph(self, eval_config): """Build metric graph for multi task model.""" @@ -149,12 +161,14 @@ def build_metric_graph(self, eval_config): if len(task_tower_cfg.losses) > 0: loss_types = {loss.loss_type for loss in task_tower_cfg.losses} self._metric_dict.update( - self._build_metric_impl( - metric, - loss_type=loss_types, - label_name=self._label_name_dict[tower_name], - num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name)) + self._build_metric_impl( + metric, + loss_type=loss_types, + label_name=self._label_name_dict[tower_name], + num_class=task_tower_cfg.num_class, + suffix='_%s' % tower_name + ) + ) return self._metric_dict def build_loss_weight(self): @@ -166,7 +180,7 @@ def build_loss_weight(self): n = len(losses) if n > 0: loss_weights[tower_name] = [ - loss.weight * task_tower_cfg.weight for loss in losses + loss.weight * task_tower_cfg.weight for loss in losses ] num_loss += n else: @@ -188,7 +202,8 @@ def get_learnt_loss(self, loss_type, name, value): strategy = self._base_model_config.loss_weight_strategy if strategy == self._base_model_config.Uncertainty: uncertainty = tf.Variable( - 0, name='%s_loss_weight' % name, dtype=tf.float32) + 0, name='%s_loss_weight' % name, dtype=tf.float32 + ) tf.summary.scalar('loss/%s_uncertainty' % name, uncertainty) if loss_type in {LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS}: return 0.5 * tf.exp(-uncertainty) * value + 0.5 * uncertainty @@ -210,31 +225,37 @@ def build_loss_graph(self): if hasattr(task_tower_cfg, 'task_space_indicator_label') and \ task_tower_cfg.HasField('task_space_indicator_label'): in_task_space = tf.to_float( - self._labels[task_tower_cfg.task_space_indicator_label] > 0) + self._labels[task_tower_cfg.task_space_indicator_label] > 0 + ) loss_weight = loss_weight * ( - task_tower_cfg.in_task_space_weight * in_task_space + - task_tower_cfg.out_task_space_weight * (1 - in_task_space)) + task_tower_cfg.in_task_space_weight * in_task_space + + task_tower_cfg.out_task_space_weight * (1 - in_task_space) + ) if task_tower_cfg.HasField('task_space_indicator_name') and \ task_tower_cfg.HasField('task_space_indicator_value'): in_task_space = tf.to_float( - tf.equal( - self._feature_dict[task_tower_cfg.task_space_indicator_name], - task_tower_cfg.task_space_indicator_value)) + tf.equal( + self._feature_dict[task_tower_cfg.task_space_indicator_name], + task_tower_cfg.task_space_indicator_value + ) + ) loss_weight = loss_weight * ( - task_tower_cfg.in_task_space_weight * in_task_space + - task_tower_cfg.out_task_space_weight * (1 - in_task_space)) + task_tower_cfg.in_task_space_weight * in_task_space + + task_tower_cfg.out_task_space_weight * (1 - in_task_space) + ) task_loss_weight = task_loss_weights[tower_name] loss_dict = {} losses = task_tower_cfg.losses if len(losses) == 0: loss_dict = self._build_loss_impl( - task_tower_cfg.loss_type, - label_name=self._label_name_dict[tower_name], - loss_weight=loss_weight, - num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name) + task_tower_cfg.loss_type, + label_name=self._label_name_dict[tower_name], + loss_weight=loss_weight, + num_class=task_tower_cfg.num_class, + suffix='_%s' % tower_name + ) for loss_name in loss_dict.keys(): loss_dict[loss_name] = loss_dict[loss_name] * task_loss_weight[0] else: @@ -246,25 +267,28 @@ def build_loss_graph(self): y_rt = self._prediction_dict['probs_%s' % relation_tower_name] cali_loss = tf.reduce_mean(tf.nn.relu(y_t - y_rt)) calibrate_loss.append(cali_loss * loss.weight) - logging.info('calibrate loss: %s -> %s' % - (relation_tower_name, tower_name)) + logging.info( + 'calibrate loss: %s -> %s' % (relation_tower_name, tower_name) + ) continue loss_param = loss.WhichOneof('loss_param') if loss_param is not None: loss_param = getattr(loss, loss_param) loss_ops = self._build_loss_impl( - loss.loss_type, - label_name=self._label_name_dict[tower_name], - loss_weight=loss_weight, - num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name, - loss_name=loss.loss_name, - loss_param=loss_param) + loss.loss_type, + label_name=self._label_name_dict[tower_name], + loss_weight=loss_weight, + num_class=task_tower_cfg.num_class, + suffix='_%s' % tower_name, + loss_name=loss.loss_name, + loss_param=loss_param + ) for i, loss_name in enumerate(loss_ops): loss_value = loss_ops[loss_name] if loss.learn_loss_weight: loss_dict[loss_name] = self.get_learnt_loss( - loss.loss_type, loss_name, loss_value) + loss.loss_type, loss_name, loss_value + ) else: loss_dict[loss_name] = loss_value * task_loss_weight[i] if calibrate_loss: @@ -273,8 +297,9 @@ def build_loss_graph(self): tf.summary.scalar('loss/order_calibrate_loss', cali_loss) self._loss_dict.update(loss_dict) - kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, - self._labels, self._feature_dict) + kd_loss_dict = loss_builder.build_kd_loss( + self.kd, self._prediction_dict, self._labels, self._feature_dict + ) self._loss_dict.update(kd_loss_dict) return self._loss_dict @@ -287,17 +312,21 @@ def get_outputs(self): tower_name = task_tower_cfg.tower_name if len(task_tower_cfg.losses) == 0: outputs.extend( - self._get_outputs_impl( - task_tower_cfg.loss_type, - task_tower_cfg.num_class, - suffix='_%s' % tower_name)) + self._get_outputs_impl( + task_tower_cfg.loss_type, + task_tower_cfg.num_class, + suffix='_%s' % tower_name + ) + ) else: for loss in task_tower_cfg.losses: if loss.loss_type == LossType.ORDER_CALIBRATE_LOSS: continue outputs.extend( - self._get_outputs_impl( - loss.loss_type, - task_tower_cfg.num_class, - suffix='_%s' % tower_name)) + self._get_outputs_impl( + loss.loss_type, + task_tower_cfg.num_class, + suffix='_%s' % tower_name + ) + ) return list(set(outputs)) diff --git a/easy_rec/python/model/multi_tower.py b/easy_rec/python/model/multi_tower.py index 5cdd89ba5..702a74374 100644 --- a/easy_rec/python/model/multi_tower.py +++ b/easy_rec/python/model/multi_tower.py @@ -5,7 +5,6 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': @@ -14,16 +13,20 @@ class MultiTower(RankModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(MultiTower, self).__init__(model_config, feature_configs, features, - labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(MultiTower, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'multi_tower', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model')) + 'invalid model config: %s' % self._model_config.WhichOneof('model') + ) self._model_config = self._model_config.multi_tower assert isinstance(self._model_config, MultiTowerConfig) @@ -41,19 +44,23 @@ def build_predict_graph(self): tower = self._model_config.towers[tower_id] tower_name = tower.input tower_fea = tf.layers.batch_normalization( - tower_fea, - training=self._is_training, - trainable=True, - name='%s_fea_bn' % tower_name) + tower_fea, + training=self._is_training, + trainable=True, + name='%s_fea_bn' % tower_name + ) - tower_dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, - self._is_training) + tower_dnn_layer = dnn.DNN( + tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training + ) tower_fea = tower_dnn_layer(tower_fea) tower_fea_arr.append(tower_fea) all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, - 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN( + self._model_config.final_dnn, self._l2_reg, 'final_dnn', + self._is_training + ) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/multi_tower_bst.py b/easy_rec/python/model/multi_tower_bst.py index 4cbc9fd29..7e28c0c4f 100644 --- a/easy_rec/python/model/multi_tower_bst.py +++ b/easy_rec/python/model/multi_tower_bst.py @@ -2,15 +2,11 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import math - import tensorflow as tf from easy_rec.python.compat import regularizers -from easy_rec.python.layers import dnn -from easy_rec.python.layers import layer_norm -from easy_rec.python.layers import seq_input_layer +from easy_rec.python.layers import dnn, layer_norm, seq_input_layer from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': @@ -19,19 +15,23 @@ class MultiTowerBST(RankModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(MultiTowerBST, self).__init__(model_config, feature_configs, features, - labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(MultiTowerBST, self).__init__( + model_config, feature_configs, features, labels, is_training + ) self._seq_input_layer = seq_input_layer.SeqInputLayer( - feature_configs, - model_config.seq_att_groups, - embedding_regularizer=self._emb_reg, - ev_params=self._global_ev_params) + feature_configs, + model_config.seq_att_groups, + embedding_regularizer=self._emb_reg, + ev_params=self._global_ev_params + ) assert self._model_config.WhichOneof('model') == 'multi_tower', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.multi_tower @@ -47,17 +47,20 @@ def __init__(self, self._bst_tower_features = [] self._bst_tower_num = len(self._model_config.bst_towers) - logging.info('all tower num: {0}'.format(self._tower_num + - self._bst_tower_num)) + logging.info( + 'all tower num: {0}'.format(self._tower_num + self._bst_tower_num) + ) logging.info('bst tower num: {0}'.format(self._bst_tower_num)) for tower_id in range(self._bst_tower_num): tower = self._model_config.bst_towers[tower_id] tower_feature = self._seq_input_layer(self._feature_dict, tower.input) regularizers.apply_regularization( - self._emb_reg, weights_list=[tower_feature['key']]) + self._emb_reg, weights_list=[tower_feature['key']] + ) regularizers.apply_regularization( - self._emb_reg, weights_list=[tower_feature['hist_seq_emb']]) + self._emb_reg, weights_list=[tower_feature['hist_seq_emb']] + ) self._bst_tower_features.append(tower_feature) def dnn_net(self, net, dnn_units, name): @@ -66,13 +69,15 @@ def dnn_net(self, net, dnn_units, name): for idx, units in enumerate(dnn_units): if idx + 1 < dnn_units_len: net = tf.layers.dense( - net, - units=units, - activation=tf.nn.relu, - name='%s_%d' % (name, idx)) + net, + units=units, + activation=tf.nn.relu, + name='%s_%d' % (name, idx) + ) else: net = tf.layers.dense( - net, units=units, activation=None, name='%s_%d' % (name, idx)) + net, units=units, activation=None, name='%s_%d' % (name, idx) + ) return net def attention_net(self, net, dim, cur_seq_len, seq_size, name): @@ -80,15 +85,19 @@ def attention_net(self, net, dim, cur_seq_len, seq_size, name): key_net = self.dnn_net(net, [dim], name + '_key') value_net = self.dnn_net(net, [dim], name + '_value') scores = tf.matmul( - query_net, key_net, transpose_b=True) # [B, seq_size, seq_size] + query_net, key_net, transpose_b=True + ) # [B, seq_size, seq_size] hist_mask = tf.sequence_mask( - cur_seq_len, maxlen=seq_size - 1) # [B, seq_size-1] + cur_seq_len, maxlen=seq_size - 1 + ) # [B, seq_size-1] cur_id_mask = tf.ones( - tf.stack([tf.shape(hist_mask)[0], 1]), dtype=tf.bool) # [B, 1] + tf.stack([tf.shape(hist_mask)[0], 1]), dtype=tf.bool + ) # [B, 1] mask = tf.concat([hist_mask, cur_id_mask], axis=1) # [B, seq_size] - masks = tf.reshape(tf.tile(mask, [1, seq_size]), - (-1, seq_size, seq_size)) # [B, seq_size, seq_size] + masks = tf.reshape( + tf.tile(mask, [1, seq_size]), (-1, seq_size, seq_size) + ) # [B, seq_size, seq_size] padding = tf.ones_like(scores) * (-2**32 + 1) scores = tf.where(masks, scores, padding) # [B, seq_size, seq_size] @@ -97,24 +106,29 @@ def attention_net(self, net, dim, cur_seq_len, seq_size, name): att_res_net = tf.matmul(scores, value_net) # [B, seq_size, emb_dim] return att_res_net - def multi_head_att_net(self, id_cols, head_count, emb_dim, seq_len, seq_size): + def multi_head_att_net( + self, id_cols, head_count, emb_dim, seq_len, seq_size + ): multi_head_attention_res = [] part_cols_emd_dim = int(math.ceil(emb_dim / head_count)) for start_idx in range(0, emb_dim, part_cols_emd_dim): if start_idx + part_cols_emd_dim > emb_dim: part_cols_emd_dim = emb_dim - start_idx - part_id_col = tf.slice(id_cols, [0, 0, start_idx], - [-1, -1, part_cols_emd_dim]) + part_id_col = tf.slice( + id_cols, [0, 0, start_idx], [-1, -1, part_cols_emd_dim] + ) part_attention_net = self.attention_net( - part_id_col, - part_cols_emd_dim, - seq_len, - seq_size, - name='multi_head_%d' % start_idx) + part_id_col, + part_cols_emd_dim, + seq_len, + seq_size, + name='multi_head_%d' % start_idx + ) multi_head_attention_res.append(part_attention_net) multi_head_attention_res_net = tf.concat(multi_head_attention_res, axis=2) multi_head_attention_res_net = self.dnn_net( - multi_head_attention_res_net, [emb_dim], name='multi_head_attention') + multi_head_attention_res_net, [emb_dim], name='multi_head_attention' + ) return multi_head_attention_res_net def add_and_norm(self, net_1, net_2, emb_dim, name): @@ -126,27 +140,32 @@ def add_and_norm(self, net_1, net_2, emb_dim, name): def bst(self, bst_fea, seq_size, head_count, name): cur_id, hist_id_col, seq_len = bst_fea['key'], bst_fea[ - 'hist_seq_emb'], bst_fea['hist_seq_len'] + 'hist_seq_emb'], bst_fea['hist_seq_len'] cur_batch_max_seq_len = tf.shape(hist_id_col)[1] hist_id_col = tf.cond( - tf.constant(seq_size) > cur_batch_max_seq_len, lambda: tf.pad( - hist_id_col, [[0, 0], [0, seq_size - cur_batch_max_seq_len - 1], - [0, 0]], 'CONSTANT'), - lambda: tf.slice(hist_id_col, [0, 0, 0], [-1, seq_size - 1, -1])) - all_ids = tf.concat([hist_id_col, tf.expand_dims(cur_id, 1)], - axis=1) # b, seq_size, emb_dim + tf.constant(seq_size) > cur_batch_max_seq_len, lambda: tf.pad( + hist_id_col, + [[0, 0], [0, seq_size - cur_batch_max_seq_len - 1], [0, 0]], 'CONSTANT' + ), lambda: tf.slice(hist_id_col, [0, 0, 0], [-1, seq_size - 1, -1]) + ) + all_ids = tf.concat( + [hist_id_col, tf.expand_dims(cur_id, 1)], axis=1 + ) # b, seq_size, emb_dim emb_dim = int(all_ids.shape[2]) - attention_net = self.multi_head_att_net(all_ids, head_count, emb_dim, - seq_len, seq_size) + attention_net = self.multi_head_att_net( + all_ids, head_count, emb_dim, seq_len, seq_size + ) tmp_net = self.add_and_norm( - all_ids, attention_net, emb_dim, name='add_and_norm_1') + all_ids, attention_net, emb_dim, name='add_and_norm_1' + ) feed_forward_net = self.dnn_net(tmp_net, [emb_dim], 'feed_forward_net') net = self.add_and_norm( - tmp_net, feed_forward_net, emb_dim, name='add_and_norm_2') + tmp_net, feed_forward_net, emb_dim, name='add_and_norm_2' + ) bst_output = tf.reshape(net, [-1, seq_size * emb_dim]) return bst_output @@ -157,12 +176,14 @@ def build_predict_graph(self): tower = self._model_config.towers[tower_id] tower_name = tower.input tower_fea = tf.layers.batch_normalization( - tower_fea, - training=self._is_training, - trainable=True, - name='%s_fea_bn' % tower_name) - tower_dnn = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, - self._is_training) + tower_fea, + training=self._is_training, + trainable=True, + name='%s_fea_bn' % tower_name + ) + tower_dnn = dnn.DNN( + tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training + ) tower_fea = tower_dnn(tower_fea) tower_fea_arr.append(tower_fea) @@ -173,15 +194,18 @@ def build_predict_graph(self): tower_seq_len = tower.seq_len tower_multi_head_size = tower.multi_head_size tower_fea = self.bst( - tower_fea, - seq_size=tower_seq_len, - head_count=tower_multi_head_size, - name=tower_name) + tower_fea, + seq_size=tower_seq_len, + head_count=tower_multi_head_size, + name=tower_name + ) tower_fea_arr.append(tower_fea) all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', - self._is_training) + final_dnn = dnn.DNN( + self._model_config.final_dnn, self._l2_reg, 'final_dnn', + self._is_training + ) all_fea = final_dnn(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/multi_tower_din.py b/easy_rec/python/model/multi_tower_din.py index e586da1cf..6e50b0f51 100644 --- a/easy_rec/python/model/multi_tower_din.py +++ b/easy_rec/python/model/multi_tower_din.py @@ -1,14 +1,11 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.compat import regularizers -from easy_rec.python.layers import dnn -from easy_rec.python.layers import seq_input_layer +from easy_rec.python.layers import dnn, seq_input_layer from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': @@ -17,19 +14,23 @@ class MultiTowerDIN(RankModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(MultiTowerDIN, self).__init__(model_config, feature_configs, features, - labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(MultiTowerDIN, self).__init__( + model_config, feature_configs, features, labels, is_training + ) self._seq_input_layer = seq_input_layer.SeqInputLayer( - feature_configs, - model_config.seq_att_groups, - embedding_regularizer=self._emb_reg, - ev_params=self._global_ev_params) + feature_configs, + model_config.seq_att_groups, + embedding_regularizer=self._emb_reg, + ev_params=self._global_ev_params + ) assert self._model_config.WhichOneof('model') == 'multi_tower', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.multi_tower @@ -45,8 +46,9 @@ def __init__(self, self._din_tower_features = [] self._din_tower_num = len(self._model_config.din_towers) - logging.info('all tower num: {0}'.format(self._tower_num + - self._din_tower_num)) + logging.info( + 'all tower num: {0}'.format(self._tower_num + self._din_tower_num) + ) logging.info('din tower num: {0}'.format(self._din_tower_num)) for tower_id in range(self._din_tower_num): @@ -56,31 +58,35 @@ def __init__(self, # apply regularization for sequence feature key in seq_input_layer. regularizers.apply_regularization( - self._emb_reg, weights_list=[tower_feature['hist_seq_emb']]) + self._emb_reg, weights_list=[tower_feature['hist_seq_emb']] + ) self._din_tower_features.append(tower_feature) def din(self, dnn_config, deep_fea, name): cur_id, hist_id_col, seq_len = deep_fea['key'], deep_fea[ - 'hist_seq_emb'], deep_fea['hist_seq_len'] + 'hist_seq_emb'], deep_fea['hist_seq_len'] seq_max_len = tf.shape(hist_id_col)[1] emb_dim = hist_id_col.shape[2] cur_ids = tf.tile(cur_id, [1, seq_max_len]) - cur_ids = tf.reshape(cur_ids, - tf.shape(hist_id_col)) # (B, seq_max_len, emb_dim) + cur_ids = tf.reshape( + cur_ids, tf.shape(hist_id_col) + ) # (B, seq_max_len, emb_dim) din_net = tf.concat( - [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1) # (B, seq_max_len, emb_dim*4) + [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], + axis=-1 + ) # (B, seq_max_len, emb_dim*4) din_layer = dnn.DNN( - dnn_config, - self._l2_reg, - name, - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True) + dnn_config, + self._l2_reg, + name, + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True + ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) @@ -103,12 +109,14 @@ def build_predict_graph(self): tower = self._model_config.towers[tower_id] tower_name = tower.input tower_fea = tf.layers.batch_normalization( - tower_fea, - training=self._is_training, - trainable=True, - name='%s_fea_bn' % tower_name) - dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, - self._is_training) + tower_fea, + training=self._is_training, + trainable=True, + name='%s_fea_bn' % tower_name + ) + dnn_layer = dnn.DNN( + tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training + ) tower_fea = dnn_layer(tower_fea) tower_fea_arr.append(tower_fea) @@ -120,8 +128,10 @@ def build_predict_graph(self): tower_fea_arr.append(tower_fea) all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, - 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN( + self._model_config.final_dnn, self._l2_reg, 'final_dnn', + self._is_training + ) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/multi_tower_recall.py b/easy_rec/python/model/multi_tower_recall.py index 8f576944e..45de92329 100644 --- a/easy_rec/python/model/multi_tower_recall.py +++ b/easy_rec/python/model/multi_tower_recall.py @@ -5,7 +5,6 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.multi_tower_recall_pb2 import MultiTowerRecall as MultiTowerRecallConfig # NOQA if tf.__version__ >= '2.0': @@ -14,16 +13,20 @@ class MultiTowerRecall(RankModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(MultiTowerRecall, self).__init__(model_config, feature_configs, - features, labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(MultiTowerRecall, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'multi_tower_recall', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model')) + 'invalid model config: %s' % self._model_config.WhichOneof('model') + ) self._model_config = self._model_config.multi_tower_recall assert isinstance(self._model_config, MultiTowerRecallConfig) @@ -36,19 +39,26 @@ def build_predict_graph(self): batch_size = tf.shape(user_tower_feature)[0] pos_item_feature = self.item_tower_feature[:batch_size] neg_item_feature = self.item_tower_feature[batch_size:] - item_tower_feature = tf.concat([ + item_tower_feature = tf.concat( + [ pos_item_feature[:, tf.newaxis, :], tf.tile( - neg_item_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1]) - ], - axis=1) # noqa: E126 - - user_dnn = dnn.DNN(self._model_config.user_tower.dnn, self._l2_reg, - 'user_dnn', self._is_training) + neg_item_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1] + ) + ], + axis=1 + ) # noqa: E126 + + user_dnn = dnn.DNN( + self._model_config.user_tower.dnn, self._l2_reg, 'user_dnn', + self._is_training + ) user_tower_emb = user_dnn(user_tower_feature) - item_dnn = dnn.DNN(self._model_config.item_tower.dnn, self._l2_reg, - 'item_dnn', self._is_training) + item_dnn = dnn.DNN( + self._model_config.item_tower.dnn, self._l2_reg, 'item_dnn', + self._is_training + ) item_tower_emb = item_dnn(item_tower_feature) item_tower_emb = tf.reshape(item_tower_emb, tf.shape(user_tower_emb)) @@ -57,8 +67,10 @@ def build_predict_graph(self): tower_fea_arr.append(item_tower_emb) all_fea = tf.concat(tower_fea_arr, axis=-1) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, - 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN( + self._model_config.final_dnn, self._l2_reg, 'final_dnn', + self._is_training + ) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, 1, name='output') output = output[:, 0] diff --git a/easy_rec/python/model/pdn.py b/easy_rec/python/model/pdn.py index 7325beb1c..16ffde39a 100644 --- a/easy_rec/python/model/pdn.py +++ b/easy_rec/python/model/pdn.py @@ -15,14 +15,17 @@ class PDN(MatchModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(PDN, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(PDN, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'pdn', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.pdn @@ -45,8 +48,9 @@ def build_predict_graph(self): logits = tf.multiply(sim_out, trigger_out) seq_mask = tf.to_float( - tf.sequence_mask(self._seq_len, - tf.shape(sim_out)[1])) + tf.sequence_mask(self._seq_len, + tf.shape(sim_out)[1]) + ) logits = tf.reduce_sum(logits * seq_mask[:, :, None], axis=1) direct_logits = self._build_direct_net() @@ -62,7 +66,8 @@ def build_predict_graph(self): self._prediction_dict['probs'] = probs self._prediction_dict['logits'] = tf.log( - tf.clip_by_value(probs, 1e-8, 1 - 1e-8)) + tf.clip_by_value(probs, 1e-8, 1 - 1e-8) + ) return self._prediction_dict def _get_seq_features(self, name): @@ -72,23 +77,26 @@ def _get_seq_features(self, name): return seq, seq_len def _build_trigger_net(self): - user_dnn_layer = dnn.DNN(self._model_config.user_dnn, self._l2_reg, - 'user_dnn', self._is_training) + user_dnn_layer = dnn.DNN( + self._model_config.user_dnn, self._l2_reg, 'user_dnn', self._is_training + ) user_fea = user_dnn_layer(self._user_features) trigger_seq = tf.concat([self._u2i_seq, self._i_seq], axis=2) - u2i_dnn_layer = dnn.DNN(self._model_config.u2i_dnn, self._l2_reg, 'u2i_dnn', - self._is_training) + u2i_dnn_layer = dnn.DNN( + self._model_config.u2i_dnn, self._l2_reg, 'u2i_dnn', self._is_training + ) trigger_seq_fea = u2i_dnn_layer(trigger_seq) trigger_merge_fea = trigger_seq_fea + user_fea[:, None, :] trigger_dnn_layer = dnn.DNN( - self._model_config.trigger_dnn, - self._l2_reg, - 'trigger_dnn', - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True) + self._model_config.trigger_dnn, + self._l2_reg, + 'trigger_dnn', + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True + ) # output: N x seq_len x d, d is usually set to 1 trigger_out = trigger_dnn_layer(trigger_merge_fea) @@ -96,67 +104,78 @@ def _build_trigger_net(self): trigger_out = tf.exp(trigger_out) self._prediction_dict['trigger_out'] = tf.reduce_join( - tf.reduce_join( - tf.as_string(trigger_out, precision=4, shortest=True), - axis=2, - separator=','), - axis=1, - separator=';') + tf.reduce_join( + tf.as_string(trigger_out, precision=4, shortest=True), + axis=2, + separator=',' + ), + axis=1, + separator=';' + ) return trigger_out def _build_similarity_net(self): - item_dnn_layer = dnn.DNN(self._model_config.item_dnn, self._l2_reg, - 'item_dnn', self._is_training) + item_dnn_layer = dnn.DNN( + self._model_config.item_dnn, self._l2_reg, 'item_dnn', self._is_training + ) item_fea = item_dnn_layer(self._item_features) - sim_side_dnn_layer = dnn.DNN(self._model_config.i2i_dnn, self._l2_reg, - 'i2i_dnn', self._is_training) + sim_side_dnn_layer = dnn.DNN( + self._model_config.i2i_dnn, self._l2_reg, 'i2i_dnn', self._is_training + ) sim_seq_fea = sim_side_dnn_layer(self._i_seq) sim_seq_cross = sim_seq_fea * item_fea[:, None, :] - item_fea_tile = tf.tile(item_fea[:, None, :], - [1, tf.shape(sim_seq_fea)[1], 1]) + item_fea_tile = tf.tile( + item_fea[:, None, :], [1, tf.shape(sim_seq_fea)[1], 1] + ) sim_seq_concat = tf.concat( - [sim_seq_cross, sim_seq_cross, self._i2i_seq, item_fea_tile], axis=2) + [sim_seq_cross, sim_seq_cross, self._i2i_seq, item_fea_tile], axis=2 + ) sim_dnn_layer = dnn.DNN( - self._model_config.sim_dnn, - self._l2_reg, - 'sim_dnn', - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True) + self._model_config.sim_dnn, + self._l2_reg, + 'sim_dnn', + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True + ) # output: N x seq_len x 1 sim_out = sim_dnn_layer(sim_seq_concat) # exp(x): map (-inf, inf) to (0, inf) sim_out = tf.exp(sim_out) self._prediction_dict['sim_out'] = tf.reduce_join( - tf.reduce_join( - tf.as_string(sim_out, precision=4, shortest=True), - axis=2, - separator=','), - axis=1, - separator=';') + tf.reduce_join( + tf.as_string(sim_out, precision=4, shortest=True), + axis=2, + separator=',' + ), + axis=1, + separator=';' + ) return sim_out def _build_direct_net(self): if self._model_config.HasField('direct_user_dnn') and \ self._model_config.HasField('direct_item_dnn'): direct_user_layer = dnn.DNN( - self._model_config.direct_user_dnn, - 'direct_user_dnn', - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True) + self._model_config.direct_user_dnn, + 'direct_user_dnn', + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True + ) direct_user_out = direct_user_layer(self._user_features) direct_item_layer = dnn.DNN( - self._model_config.direct_item_dnn, - 'direct_item_dnn', - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True) + self._model_config.direct_item_dnn, + 'direct_item_dnn', + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True + ) direct_item_out = direct_item_layer(self._item_features) if self._model_config.simi_func == Similarity.COSINE: @@ -169,15 +188,17 @@ def _build_direct_net(self): if self._model_config.scale_simi: sim_w = tf.get_variable( - 'direct_net/sim_w', - dtype=tf.float32, - shape=(1), - initializer=tf.ones_initializer()) + 'direct_net/sim_w', + dtype=tf.float32, + shape=(1), + initializer=tf.ones_initializer() + ) sim_b = tf.get_variable( - 'direct_net/sim_b', - dtype=tf.float32, - shape=(1), - initializer=tf.zeros_initializer()) + 'direct_net/sim_b', + dtype=tf.float32, + shape=(1), + initializer=tf.zeros_initializer() + ) direct_logits = direct_logits * tf.abs(sim_w) + sim_b return tf.nn.softplus(direct_logits) @@ -188,12 +209,13 @@ def _build_bias_net(self): if self._model_config.HasField('bias_dnn'): assert self._bias_features is not None, 'bias group must be defined' bias_dnn_layer = dnn.DNN( - self._model_config.bias_dnn, - self._l2_reg, - 'bias_dnn', - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True) + self._model_config.bias_dnn, + self._l2_reg, + 'bias_dnn', + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True + ) bias_logits = bias_dnn_layer(self._bias_features) return tf.nn.softplus(bias_logits) else: diff --git a/easy_rec/python/model/ple.py b/easy_rec/python/model/ple.py index e04781bcd..942d52eee 100644 --- a/easy_rec/python/model/ple.py +++ b/easy_rec/python/model/ple.py @@ -12,14 +12,17 @@ class PLE(MultiTaskModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(PLE, self).__init__(model_config, feature_configs, features, labels, - is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(PLE, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'ple', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.ple @@ -36,11 +39,12 @@ def __init__(self, def gate(self, selector_fea, vec_feas, name): vec = tf.stack(vec_feas, axis=1) gate = tf.layers.dense( - inputs=selector_fea, - units=len(vec_feas), - kernel_regularizer=self._l2_reg, - activation=None, - name=name + '_gate/dnn') + inputs=selector_fea, + units=len(vec_feas), + kernel_regularizer=self._l2_reg, + activation=None, + name=name + '_gate/dnn' + ) gate = tf.nn.softmax(gate, axis=1) gate = tf.expand_dims(gate, -1) task_input = tf.multiply(vec, gate) @@ -51,40 +55,47 @@ def experts_layer(self, deep_fea, expert_num, experts_cfg, name): tower_outputs = [] for expert_id in range(expert_num): tower_dnn = dnn.DNN( - experts_cfg, - self._l2_reg, - name=name + '_expert_%d/dnn' % expert_id, - is_training=self._is_training) + experts_cfg, + self._l2_reg, + name=name + '_expert_%d/dnn' % expert_id, + is_training=self._is_training + ) tower_output = tower_dnn(deep_fea) tower_outputs.append(tower_output) return tower_outputs - def CGC_layer(self, extraction_networks_cfg, extraction_network_fea, - shared_expert_fea, final_flag): + def CGC_layer( + self, extraction_networks_cfg, extraction_network_fea, shared_expert_fea, + final_flag + ): layer_name = extraction_networks_cfg.network_name expert_shared_out = self.experts_layer( - shared_expert_fea, extraction_networks_cfg.share_num, - extraction_networks_cfg.share_expert_net, layer_name + '_share/dnn') + shared_expert_fea, extraction_networks_cfg.share_num, + extraction_networks_cfg.share_expert_net, layer_name + '_share/dnn' + ) experts_outs = [] cgc_layer_outs = [] for task_idx in range(self._task_nums): name = layer_name + '_task_%d' % task_idx experts_out = self.experts_layer( - extraction_network_fea[task_idx], - extraction_networks_cfg.expert_num_per_task, - extraction_networks_cfg.task_expert_net, name) - cgc_layer_out = self.gate(extraction_network_fea[task_idx], - experts_out + expert_shared_out, name) + extraction_network_fea[task_idx], + extraction_networks_cfg.expert_num_per_task, + extraction_networks_cfg.task_expert_net, name + ) + cgc_layer_out = self.gate( + extraction_network_fea[task_idx], experts_out + expert_shared_out, name + ) experts_outs.extend(experts_out) cgc_layer_outs.append(cgc_layer_out) if final_flag: shared_layer_out = None else: - shared_layer_out = self.gate(shared_expert_fea, - experts_outs + expert_shared_out, - layer_name + '_share') + shared_layer_out = self.gate( + shared_expert_fea, experts_outs + expert_shared_out, + layer_name + '_share' + ) return cgc_layer_outs, shared_layer_out def build_predict_graph(self): @@ -96,24 +107,27 @@ def build_predict_graph(self): if idx == len(self._model_config.extraction_networks) - 1: final_flag = True extraction_network_fea, shared_expert_fea = self.CGC_layer( - extraction_network, extraction_network_fea, shared_expert_fea, - final_flag) + extraction_network, extraction_network_fea, shared_expert_fea, + final_flag + ) tower_outputs = {} for i, task_tower_cfg in enumerate(self._model_config.task_towers): tower_name = task_tower_cfg.tower_name tower_dnn = dnn.DNN( - task_tower_cfg.dnn, - self._l2_reg, - name=tower_name, - is_training=self._is_training) + task_tower_cfg.dnn, + self._l2_reg, + name=tower_name, + is_training=self._is_training + ) tower_output = tower_dnn(extraction_network_fea[i]) tower_output = tf.layers.dense( - inputs=tower_output, - units=task_tower_cfg.num_class, - kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i) + inputs=tower_output, + units=task_tower_cfg.num_class, + kernel_regularizer=self._l2_reg, + name='dnn_output_%d' % i + ) tower_outputs[tower_name] = tower_output self._add_to_prediction_dict(tower_outputs) diff --git a/easy_rec/python/model/rank_model.py b/easy_rec/python/model/rank_model.py index ebed369dd..ee01b533e 100644 --- a/easy_rec/python/model/rank_model.py +++ b/easy_rec/python/model/rank_model.py @@ -1,30 +1,31 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from tensorflow.python.ops import math_ops from easy_rec.python.builders import loss_builder +from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_pred # NOQA from easy_rec.python.model.easy_rec_model import EasyRecModel from easy_rec.python.protos.loss_pb2 import LossType -from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_pred # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 class RankModel(EasyRecModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(RankModel, self).__init__(model_config, feature_configs, features, - labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(RankModel, self).__init__( + model_config, feature_configs, features, labels, is_training + ) self._loss_type = self._model_config.loss_type self._num_class = self._model_config.num_class self._losses = self._model_config.losses @@ -38,7 +39,7 @@ def __init__(self, def build_predict_graph(self): if not self.has_backbone: raise NotImplementedError( - 'method `build_predict_graph` must be implemented when backbone network do not exits' + 'method `build_predict_graph` must be implemented when backbone network do not exits' ) model = self._model_config.WhichOneof('model') assert model == 'model_params', '`model_params` must be configured' @@ -53,19 +54,16 @@ def build_predict_graph(self): self._add_to_prediction_dict(output) return self._prediction_dict - def _output_to_prediction_impl(self, - output, - loss_type, - num_class=1, - suffix='', - **kwargs): + def _output_to_prediction_impl( + self, output, loss_type, num_class=1, suffix='', **kwargs + ): prediction_dict = {} binary_loss_type = { - LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, LossType.BINARY_CROSS_ENTROPY_LOSS, - LossType.LISTWISE_DISTILL_LOSS + LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, LossType.BINARY_CROSS_ENTROPY_LOSS, + LossType.LISTWISE_DISTILL_LOSS } if loss_type in binary_loss_type: assert num_class == 1, 'num_class must be 1 when loss type is %s' % loss_type.name @@ -87,10 +85,11 @@ def _output_to_prediction_impl(self, return_log = kwargs.get('return_log_pred_value', False) max_sigma = kwargs.get('max_sigma', 5.0) probs, preds = zero_inflated_lognormal_pred( - output, - max_sigma=max_sigma, - max_log_clip=max_log_clip_val, - return_log=return_log) + output, + max_sigma=max_sigma, + max_log_clip=max_log_clip_val, + return_log=return_log + ) tf.summary.scalar('prediction/probs', tf.reduce_mean(probs)) tf.summary.scalar('prediction/y', tf.reduce_mean(preds)) prediction_dict['logits' + suffix] = output @@ -109,10 +108,10 @@ def _output_to_prediction_impl(self, prediction_dict['logits' + suffix + '_1'] = output[:, 1] prediction_dict['probs' + suffix] = probs prediction_dict['probs' + suffix + '_1'] = probs[:, 1] - prediction_dict['logits' + suffix + '_y'] = math_ops.reduce_max( - output, axis=1) - prediction_dict['probs' + suffix + '_y'] = math_ops.reduce_max( - probs, axis=1) + prediction_dict['logits' + suffix + + '_y'] = math_ops.reduce_max(output, axis=1) + prediction_dict['probs' + suffix + + '_y'] = math_ops.reduce_max(probs, axis=1) prediction_dict['y' + suffix] = tf.argmax(output, axis=1) elif loss_type == LossType.L2_LOSS: output = tf.squeeze(output, axis=1) @@ -125,7 +124,8 @@ def _output_to_prediction_impl(self, def _add_to_prediction_dict(self, output): if len(self._losses) == 0: prediction_dict = self._output_to_prediction_impl( - output, loss_type=self._loss_type, num_class=self._num_class) + output, loss_type=self._loss_type, num_class=self._num_class + ) self._prediction_dict.update(prediction_dict) else: for loss in self._losses: @@ -138,10 +138,11 @@ def _add_to_prediction_dict(self, output): kwargs['return_log_pred_value'] = loss_param.return_log_pred_value kwargs['max_sigma'] = loss_param.max_sigma prediction_dict = self._output_to_prediction_impl( - output, - loss_type=loss.loss_type, - num_class=self._num_class, - **kwargs) + output, + loss_type=loss.loss_type, + num_class=self._num_class, + **kwargs + ) self._prediction_dict.update(prediction_dict) def build_rtp_output_dict(self): @@ -153,9 +154,11 @@ def build_rtp_output_dict(self): op = tf.get_default_graph().get_operation_by_name('rank_predict') if len(op.outputs) != 1: raise ValueError( - ('failed to build RTP rank_predict output: op {}[{}] has output ' + - 'size {}, however 1 is expected.').format(op.name, op.type, - len(op.outputs))) + ( + 'failed to build RTP rank_predict output: op {}[{}] has output ' + + 'size {}, however 1 is expected.' + ).format(op.name, op.type, len(op.outputs)) + ) rank_predict = op.outputs[0] except KeyError: forwarded = None @@ -163,59 +166,64 @@ def build_rtp_output_dict(self): if len(self._losses) > 0: loss_types = {loss.loss_type for loss in self._losses} binary_loss_set = { - LossType.CLASSIFICATION, LossType.F1_REWEIGHTED_LOSS, - LossType.PAIR_WISE_LOSS, LossType.BINARY_FOCAL_LOSS, - LossType.PAIRWISE_FOCAL_LOSS, LossType.PAIRWISE_LOGISTIC_LOSS, - LossType.JRC_LOSS, LossType.LISTWISE_DISTILL_LOSS, - LossType.LISTWISE_RANK_LOSS + LossType.CLASSIFICATION, LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.JRC_LOSS, LossType.LISTWISE_DISTILL_LOSS, + LossType.LISTWISE_RANK_LOSS } if loss_types & binary_loss_set: if 'probs' in self._prediction_dict: forwarded = self._prediction_dict['probs'] else: raise ValueError( - 'failed to build RTP rank_predict output: classification model ' + - "expect 'probs' prediction, which is not found. Please check if" + - ' build_predict_graph() is called.') + 'failed to build RTP rank_predict output: classification model ' + + "expect 'probs' prediction, which is not found. Please check if" + + ' build_predict_graph() is called.' + ) elif loss_types & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS }: if 'y' in self._prediction_dict: forwarded = self._prediction_dict['y'] else: raise ValueError( - 'failed to build RTP rank_predict output: regression model expect' - + - "'y' prediction, which is not found. Please check if build_predic" - + 't_graph() is called.') + 'failed to build RTP rank_predict output: regression model expect' + + + "'y' prediction, which is not found. Please check if build_predic" + + 't_graph() is called.' + ) else: logging.warning( - 'failed to build RTP rank_predict: unsupported loss type {}'.format( - loss_types)) + 'failed to build RTP rank_predict: unsupported loss type {}'. + format(loss_types) + ) if forwarded is not None: rank_predict = tf.identity(forwarded, name='rank_predict') if rank_predict is not None: outputs['rank_predict'] = rank_predict return outputs - def _build_loss_impl(self, - loss_type, - label_name, - loss_weight=1.0, - num_class=1, - suffix='', - loss_name='', - loss_param=None): + def _build_loss_impl( + self, + loss_type, + label_name, + loss_weight=1.0, + num_class=1, + suffix='', + loss_name='', + loss_param=None + ): loss_dict = {} binary_loss_type = { - LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, LossType.JRC_LOSS, - LossType.LISTWISE_DISTILL_LOSS, LossType.ZILN_LOSS + LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, LossType.JRC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, LossType.ZILN_LOSS } if loss_type in { - LossType.CLASSIFICATION, LossType.BINARY_CROSS_ENTROPY_LOSS + LossType.CLASSIFICATION, LossType.BINARY_CROSS_ENTROPY_LOSS }: loss_name = loss_name if loss_name else 'cross_entropy_loss' + suffix pred = self._prediction_dict['logits' + suffix] @@ -231,20 +239,23 @@ def _build_loss_impl(self, else: raise ValueError('invalid loss type: %s' % LossType.Name(loss_type)) - tf.summary.scalar('labels/%s' % label_name, - tf.reduce_mean(tf.to_float(self._labels[label_name]))) + tf.summary.scalar( + 'labels/%s' % label_name, + tf.reduce_mean(tf.to_float(self._labels[label_name])) + ) kwargs = {'loss_name': loss_name} if loss_param is not None: if hasattr(loss_param, 'session_name'): kwargs['session_ids'] = self._feature_dict[loss_param.session_name] loss_dict[loss_name] = loss_builder.build( - loss_type, - self._labels[label_name], - pred, - loss_weight, - num_class, - loss_param=loss_param, - **kwargs) + loss_type, + self._labels[label_name], + pred, + loss_weight, + num_class, + loss_param=loss_param, + **kwargs + ) return loss_dict def build_loss_graph(self): @@ -252,14 +263,17 @@ def build_loss_graph(self): with tf.name_scope('loss'): if len(self._losses) == 0: loss_dict = self._build_loss_impl( - self._loss_type, - label_name=self._label_name, - loss_weight=self._sample_weight, - num_class=self._num_class) + self._loss_type, + label_name=self._label_name, + loss_weight=self._sample_weight, + num_class=self._num_class + ) else: strategy = self._base_model_config.loss_weight_strategy loss_weight = [1.0] - if strategy == self._base_model_config.Random and len(self._losses) > 1: + if strategy == self._base_model_config.Random and len( + self._losses + ) > 1: weights = tf.random_normal([len(self._losses)]) loss_weight = tf.nn.softmax(weights) for i, loss in enumerate(self._losses): @@ -267,59 +281,61 @@ def build_loss_graph(self): if loss_param is not None: loss_param = getattr(loss, loss_param) loss_ops = self._build_loss_impl( - loss.loss_type, - label_name=self._label_name, - loss_weight=self._sample_weight, - num_class=self._num_class, - loss_name=loss.loss_name, - loss_param=loss_param) + loss.loss_type, + label_name=self._label_name, + loss_weight=self._sample_weight, + num_class=self._num_class, + loss_name=loss.loss_name, + loss_param=loss_param + ) for loss_name, loss_value in loss_ops.items(): if strategy == self._base_model_config.Fixed: loss_dict[loss_name] = loss_value * loss.weight elif strategy == self._base_model_config.Uncertainty: if loss.learn_loss_weight: uncertainty = tf.Variable( - 0, name='%s_loss_weight' % loss_name, dtype=tf.float32) + 0, name='%s_loss_weight' % loss_name, dtype=tf.float32 + ) tf.summary.scalar('%s_uncertainty' % loss_name, uncertainty) if loss.loss_type in { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS + LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS }: loss_dict[loss_name] = 0.5 * tf.exp( - -uncertainty) * loss_value + 0.5 * uncertainty + -uncertainty + ) * loss_value + 0.5 * uncertainty else: - loss_dict[loss_name] = tf.exp( - -uncertainty) * loss_value + 0.5 * uncertainty + loss_dict[ + loss_name + ] = tf.exp(-uncertainty) * loss_value + 0.5 * uncertainty else: loss_dict[loss_name] = loss_value * loss.weight elif strategy == self._base_model_config.Random: loss_dict[loss_name] = loss_value * loss_weight[i] else: - raise ValueError('Unsupported loss weight strategy: ' + - strategy.Name) + raise ValueError( + 'Unsupported loss weight strategy: ' + strategy.Name + ) self._loss_dict.update(loss_dict) # build kd loss - kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, - self._labels, - self._feature_dict) + kd_loss_dict = loss_builder.build_kd_loss( + self.kd, self._prediction_dict, self._labels, self._feature_dict + ) self._loss_dict.update(kd_loss_dict) return self._loss_dict - def _build_metric_impl(self, - metric, - loss_type, - label_name, - num_class=1, - suffix=''): + def _build_metric_impl( + self, metric, loss_type, label_name, num_class=1, suffix='' + ): if not isinstance(loss_type, set): loss_type = {loss_type} - from easy_rec.python.core.easyrec_metrics import metrics_tf from easy_rec.python.core import metrics as metrics_lib + from easy_rec.python.core.easyrec_metrics import metrics_tf binary_loss_set = { - LossType.CLASSIFICATION, LossType.F1_REWEIGHTED_LOSS, - LossType.PAIR_WISE_LOSS, LossType.BINARY_FOCAL_LOSS, - LossType.PAIRWISE_FOCAL_LOSS, LossType.PAIRWISE_LOGISTIC_LOSS, - LossType.JRC_LOSS, LossType.LISTWISE_DISTILL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.ZILN_LOSS + LossType.CLASSIFICATION, LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.JRC_LOSS, LossType.LISTWISE_DISTILL_LOSS, + LossType.LISTWISE_RANK_LOSS, LossType.ZILN_LOSS } metric_dict = {} if metric.WhichOneof('metric') == 'auc': @@ -327,15 +343,17 @@ def _build_metric_impl(self, if num_class == 1 or loss_type & {LossType.JRC_LOSS, LossType.ZILN_LOSS}: label = tf.to_int64(self._labels[label_name]) metric_dict['auc' + suffix] = metrics_tf.auc( - label, - self._prediction_dict['probs' + suffix], - num_thresholds=metric.auc.num_thresholds) + label, + self._prediction_dict['probs' + suffix], + num_thresholds=metric.auc.num_thresholds + ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) metric_dict['auc' + suffix] = metrics_tf.auc( - label, - self._prediction_dict['probs' + suffix][:, 1], - num_thresholds=metric.auc.num_thresholds) + label, + self._prediction_dict['probs' + suffix][:, 1], + num_thresholds=metric.auc.num_thresholds + ) else: raise ValueError('Wrong class number') elif metric.WhichOneof('metric') == 'gauc': @@ -345,20 +363,23 @@ def _build_metric_impl(self, uids = self._feature_dict[metric.gauc.uid_field] if isinstance(uids, tf.sparse.SparseTensor): uids = tf.sparse_to_dense( - uids.indices, uids.dense_shape, uids.values, default_value='') + uids.indices, uids.dense_shape, uids.values, default_value='' + ) uids = tf.reshape(uids, [-1]) metric_dict['gauc' + suffix] = metrics_lib.gauc( - label, - self._prediction_dict['probs' + suffix], - uids=uids, - reduction=metric.gauc.reduction) + label, + self._prediction_dict['probs' + suffix], + uids=uids, + reduction=metric.gauc.reduction + ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) metric_dict['gauc' + suffix] = metrics_lib.gauc( - label, - self._prediction_dict['probs' + suffix][:, 1], - uids=self._feature_dict[metric.gauc.uid_field], - reduction=metric.gauc.reduction) + label, + self._prediction_dict['probs' + suffix][:, 1], + uids=self._feature_dict[metric.gauc.uid_field], + reduction=metric.gauc.reduction + ) else: raise ValueError('Wrong class number') elif metric.WhichOneof('metric') == 'session_auc': @@ -366,17 +387,19 @@ def _build_metric_impl(self, if num_class == 1 or loss_type & {LossType.JRC_LOSS, LossType.ZILN_LOSS}: label = tf.to_int64(self._labels[label_name]) metric_dict['session_auc' + suffix] = metrics_lib.session_auc( - label, - self._prediction_dict['probs' + suffix], - session_ids=self._feature_dict[metric.session_auc.session_id_field], - reduction=metric.session_auc.reduction) + label, + self._prediction_dict['probs' + suffix], + session_ids=self._feature_dict[metric.session_auc.session_id_field], + reduction=metric.session_auc.reduction + ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) metric_dict['session_auc' + suffix] = metrics_lib.session_auc( - label, - self._prediction_dict['probs' + suffix][:, 1], - session_ids=self._feature_dict[metric.session_auc.session_id_field], - reduction=metric.session_auc.reduction) + label, + self._prediction_dict['probs' + suffix][:, 1], + session_ids=self._feature_dict[metric.session_auc.session_id_field], + reduction=metric.session_auc.reduction + ) else: raise ValueError('Wrong class number') elif metric.WhichOneof('metric') == 'max_f1': @@ -384,11 +407,13 @@ def _build_metric_impl(self, if num_class == 1 or loss_type & {LossType.JRC_LOSS, LossType.ZILN_LOSS}: label = tf.to_int64(self._labels[label_name]) metric_dict['max_f1' + suffix] = metrics_lib.max_f1( - label, self._prediction_dict['logits' + suffix]) + label, self._prediction_dict['logits' + suffix] + ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) metric_dict['max_f1' + suffix] = metrics_lib.max_f1( - label, self._prediction_dict['logits' + suffix][:, 1]) + label, self._prediction_dict['logits' + suffix][:, 1] + ) else: raise ValueError('Wrong class number') elif metric.WhichOneof('metric') == 'recall_at_topk': @@ -396,48 +421,55 @@ def _build_metric_impl(self, assert num_class > 1 label = tf.to_int64(self._labels[label_name]) metric_dict['recall_at_topk' + suffix] = metrics_tf.recall_at_k( - label, self._prediction_dict['logits' + suffix], - metric.recall_at_topk.topk) + label, self._prediction_dict['logits' + suffix], + metric.recall_at_topk.topk + ) elif metric.WhichOneof('metric') == 'mean_absolute_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS }: metric_dict['mean_absolute_error' + suffix] = metrics_tf.mean_absolute_error( - label, self._prediction_dict['y' + suffix]) + label, self._prediction_dict['y' + suffix] + ) elif loss_type & {LossType.CLASSIFICATION} and num_class == 1: metric_dict['mean_absolute_error' + suffix] = metrics_tf.mean_absolute_error( - label, self._prediction_dict['probs' + suffix]) + label, self._prediction_dict['probs' + suffix] + ) else: assert False, 'mean_absolute_error is not supported for this model' elif metric.WhichOneof('metric') == 'mean_squared_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS }: metric_dict['mean_squared_error' + suffix] = metrics_tf.mean_squared_error( - label, self._prediction_dict['y' + suffix]) + label, self._prediction_dict['y' + suffix] + ) elif num_class == 1 and loss_type & binary_loss_set: metric_dict['mean_squared_error' + suffix] = metrics_tf.mean_squared_error( - label, self._prediction_dict['probs' + suffix]) + label, self._prediction_dict['probs' + suffix] + ) else: assert False, 'mean_squared_error is not supported for this model' elif metric.WhichOneof('metric') == 'root_mean_squared_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS }: metric_dict['root_mean_squared_error' + suffix] = metrics_tf.root_mean_squared_error( - label, self._prediction_dict['y' + suffix]) + label, self._prediction_dict['y' + suffix] + ) elif loss_type & {LossType.CLASSIFICATION} and num_class == 1: metric_dict['root_mean_squared_error' + suffix] = metrics_tf.root_mean_squared_error( - label, self._prediction_dict['probs' + suffix]) + label, self._prediction_dict['probs' + suffix] + ) else: assert False, 'root_mean_squared_error is not supported for this model' elif metric.WhichOneof('metric') == 'accuracy': @@ -445,7 +477,8 @@ def _build_metric_impl(self, assert num_class > 1 label = tf.to_int64(self._labels[label_name]) metric_dict['accuracy' + suffix] = metrics_tf.accuracy( - label, self._prediction_dict['y' + suffix]) + label, self._prediction_dict['y' + suffix] + ) return metric_dict def build_metric_graph(self, eval_config): @@ -454,19 +487,21 @@ def build_metric_graph(self, eval_config): loss_types = {loss.loss_type for loss in self._losses} for metric in eval_config.metrics_set: self._metric_dict.update( - self._build_metric_impl( - metric, - loss_type=loss_types, - label_name=self._label_name, - num_class=self._num_class)) + self._build_metric_impl( + metric, + loss_type=loss_types, + label_name=self._label_name, + num_class=self._num_class + ) + ) return self._metric_dict def _get_outputs_impl(self, loss_type, num_class=1, suffix=''): binary_loss_set = { - LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, LossType.LISTWISE_DISTILL_LOSS + LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, LossType.LISTWISE_DISTILL_LOSS } if loss_type in binary_loss_set: return ['probs' + suffix, 'logits' + suffix] @@ -479,9 +514,9 @@ def _get_outputs_impl(self, loss_type, num_class=1, suffix=''): return ['probs' + suffix, 'logits' + suffix] else: return [ - 'y' + suffix, 'probs' + suffix, 'logits' + suffix, - 'probs' + suffix + '_y', 'logits' + suffix + '_y', - 'probs' + suffix + '_1', 'logits' + suffix + '_1' + 'y' + suffix, 'probs' + suffix, 'logits' + suffix, + 'probs' + suffix + '_y', 'logits' + suffix + '_y', + 'probs' + suffix + '_1', 'logits' + suffix + '_1' ] elif loss_type in [LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS]: return ['y' + suffix] diff --git a/easy_rec/python/model/rocket_launching.py b/easy_rec/python/model/rocket_launching.py index aea29bf52..970a5e6d6 100755 --- a/easy_rec/python/model/rocket_launching.py +++ b/easy_rec/python/model/rocket_launching.py @@ -6,9 +6,8 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel from easy_rec.python.protos.loss_pb2 import LossType -from easy_rec.python.protos.simi_pb2 import Similarity - from easy_rec.python.protos.rocket_launching_pb2 import RocketLaunching as RocketLaunchingConfig # NOQA +from easy_rec.python.protos.simi_pb2 import Similarity if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -16,14 +15,17 @@ class RocketLaunching(RankModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(RocketLaunching, self).__init__(model_config, feature_configs, - features, labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(RocketLaunching, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'rocket_launching', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.rocket_launching @@ -35,7 +37,8 @@ def __init__(self, def sim(self, feature_emb1, feature_emb2): emb1_emb2_sim = tf.reduce_sum( - tf.multiply(feature_emb1, feature_emb2), axis=1, keepdims=True) + tf.multiply(feature_emb1, feature_emb2), axis=1, keepdims=True + ) return emb1_emb2_sim def norm(self, fea): @@ -44,80 +47,102 @@ def norm(self, fea): def feature_based_sim(self, feature_based_distillation, i, j): booster_feature_no_gradient = tf.stop_gradient( - self.booster_feature['hidden_layer' + str(j)]) + self.booster_feature['hidden_layer' + str(j)] + ) if feature_based_distillation == Similarity.COSINE: booster_feature_no_gradient_norm = self.norm(booster_feature_no_gradient) - light_feature_norm = self.norm(self.light_feature['hidden_layer' + - str(i)]) + light_feature_norm = self.norm( + self.light_feature['hidden_layer' + str(i)] + ) sim_middle_layer = tf.reduce_mean( - self.sim(booster_feature_no_gradient_norm, light_feature_norm)) + self.sim(booster_feature_no_gradient_norm, light_feature_norm) + ) return sim_middle_layer else: return tf.sqrt( - tf.reduce_sum( - tf.square(booster_feature_no_gradient - - self.light_feature['hidden_layer' + str(i)]))) + tf.reduce_sum( + tf.square( + booster_feature_no_gradient - + self.light_feature['hidden_layer' + str(i)] + ) + ) + ) def build_predict_graph(self): self.hidden_layer_feature_output = self._model_config.feature_based_distillation if self._model_config.HasField('share_dnn'): - share_dnn_layer = dnn.DNN(self._model_config.share_dnn, self._l2_reg, - 'share_dnn', self._is_training) + share_dnn_layer = dnn.DNN( + self._model_config.share_dnn, self._l2_reg, 'share_dnn', + self._is_training + ) share_feature = share_dnn_layer(self._features) - booster_dnn_layer = dnn.DNN(self._model_config.booster_dnn, self._l2_reg, - 'booster_dnn', self._is_training) - light_dnn_layer = dnn.DNN(self._model_config.light_dnn, self._l2_reg, - 'light_dnn', self._is_training) + booster_dnn_layer = dnn.DNN( + self._model_config.booster_dnn, self._l2_reg, 'booster_dnn', + self._is_training + ) + light_dnn_layer = dnn.DNN( + self._model_config.light_dnn, self._l2_reg, 'light_dnn', + self._is_training + ) if self._model_config.HasField('share_dnn'): - self.booster_feature = booster_dnn_layer(share_feature, - self.hidden_layer_feature_output) + self.booster_feature = booster_dnn_layer( + share_feature, self.hidden_layer_feature_output + ) input_embedding_stop_gradient = tf.stop_gradient(share_feature) - self.light_feature = light_dnn_layer(input_embedding_stop_gradient, - self.hidden_layer_feature_output) + self.light_feature = light_dnn_layer( + input_embedding_stop_gradient, self.hidden_layer_feature_output + ) else: - self.booster_feature = booster_dnn_layer(self._features, - self.hidden_layer_feature_output) + self.booster_feature = booster_dnn_layer( + self._features, self.hidden_layer_feature_output + ) input_embedding_stop_gradient = tf.stop_gradient(self._features) - self.light_feature = light_dnn_layer(input_embedding_stop_gradient, - self.hidden_layer_feature_output) + self.light_feature = light_dnn_layer( + input_embedding_stop_gradient, self.hidden_layer_feature_output + ) if self._model_config.feature_based_distillation: booster_out = tf.layers.dense( - self.booster_feature['hidden_layer_end'], - self._num_class, - kernel_regularizer=self._l2_reg, - name='booster_output') + self.booster_feature['hidden_layer_end'], + self._num_class, + kernel_regularizer=self._l2_reg, + name='booster_output' + ) light_out = tf.layers.dense( - self.light_feature['hidden_layer_end'], - self._num_class, - kernel_regularizer=self._l2_reg, - name='light_output') + self.light_feature['hidden_layer_end'], + self._num_class, + kernel_regularizer=self._l2_reg, + name='light_output' + ) else: booster_out = tf.layers.dense( - self.booster_feature, - self._num_class, - kernel_regularizer=self._l2_reg, - name='booster_output') + self.booster_feature, + self._num_class, + kernel_regularizer=self._l2_reg, + name='booster_output' + ) light_out = tf.layers.dense( - self.light_feature, - self._num_class, - kernel_regularizer=self._l2_reg, - name='light_output') + self.light_feature, + self._num_class, + kernel_regularizer=self._l2_reg, + name='light_output' + ) self._prediction_dict.update( - self._output_to_prediction_impl( - booster_out, - self._loss_type, - num_class=self._num_class, - suffix='_booster')) + self._output_to_prediction_impl( + booster_out, + self._loss_type, + num_class=self._num_class, + suffix='_booster' + ) + ) self._prediction_dict.update( - self._output_to_prediction_impl( - light_out, - self._loss_type, - num_class=self._num_class, - suffix='_light')) + self._output_to_prediction_impl( + light_out, self._loss_type, num_class=self._num_class, suffix='_light' + ) + ) return self._prediction_dict @@ -136,34 +161,40 @@ def build_loss_graph(self): for j, unit_j in enumerate(booster_hidden_units): if light_hidden_units[i] == booster_hidden_units[j]: self._prediction_dict[ - 'similarity_' + str(count)] = self.feature_based_sim( - self._model_config.feature_based_distillation, i, j) + 'similarity_' + str(count)] = self.feature_based_sim( + self._model_config.feature_based_distillation, i, j + ) count += 1 break self._loss_dict.update( - self._build_loss_impl( - LossType.CLASSIFICATION, - label_name=self._label_name, - loss_weight=self._sample_weight, - num_class=self._num_class, - suffix='_booster')) + self._build_loss_impl( + LossType.CLASSIFICATION, + label_name=self._label_name, + loss_weight=self._sample_weight, + num_class=self._num_class, + suffix='_booster' + ) + ) self._loss_dict.update( - self._build_loss_impl( - LossType.CLASSIFICATION, - label_name=self._label_name, - loss_weight=self._sample_weight, - num_class=self._num_class, - suffix='_light')) + self._build_loss_impl( + LossType.CLASSIFICATION, + label_name=self._label_name, + loss_weight=self._sample_weight, + num_class=self._num_class, + suffix='_light' + ) + ) booster_logits_no_grad = tf.stop_gradient(logits_booster) self._loss_dict['hint_loss'] = loss_builder.build( - LossType.L2_LOSS, - label=booster_logits_no_grad, - pred=logits_light, - loss_weight=self._sample_weight) + LossType.L2_LOSS, + label=booster_logits_no_grad, + pred=logits_light, + loss_weight=self._sample_weight + ) if self._model_config.feature_based_distillation: for key, value in self._prediction_dict.items(): @@ -177,27 +208,35 @@ def build_metric_graph(self, eval_config): metric_dict = {} for metric in eval_config.metrics_set: metric_dict.update( - self._build_metric_impl( - metric, - loss_type=LossType.CLASSIFICATION, - label_name=self._label_name, - num_class=self._num_class, - suffix='_light')) + self._build_metric_impl( + metric, + loss_type=LossType.CLASSIFICATION, + label_name=self._label_name, + num_class=self._num_class, + suffix='_light' + ) + ) metric_dict.update( - self._build_metric_impl( - metric, - loss_type=LossType.CLASSIFICATION, - label_name=self._label_name, - num_class=self._num_class, - suffix='_booster')) + self._build_metric_impl( + metric, + loss_type=LossType.CLASSIFICATION, + label_name=self._label_name, + num_class=self._num_class, + suffix='_booster' + ) + ) return metric_dict def get_outputs(self): outputs = [] outputs.extend( - self._get_outputs_impl( - self._loss_type, self._num_class, suffix='_light')) + self._get_outputs_impl( + self._loss_type, self._num_class, suffix='_light' + ) + ) outputs.extend( - self._get_outputs_impl( - self._loss_type, self._num_class, suffix='_booster')) + self._get_outputs_impl( + self._loss_type, self._num_class, suffix='_booster' + ) + ) return outputs diff --git a/easy_rec/python/model/simple_multi_task.py b/easy_rec/python/model/simple_multi_task.py index 05dd7a773..ec2207e48 100644 --- a/easy_rec/python/model/simple_multi_task.py +++ b/easy_rec/python/model/simple_multi_task.py @@ -4,7 +4,6 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.multi_task_model import MultiTaskModel - from easy_rec.python.protos.simple_multi_task_pb2 import SimpleMultiTask as SimpleMultiTaskConfig # NOQA if tf.__version__ >= '2.0': @@ -13,14 +12,17 @@ class SimpleMultiTask(MultiTaskModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(SimpleMultiTask, self).__init__(model_config, feature_configs, - features, labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(SimpleMultiTask, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'simple_multi_task', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') @@ -38,16 +40,18 @@ def build_predict_graph(self): for i, task_tower_cfg in enumerate(self._task_towers): tower_name = task_tower_cfg.tower_name task_dnn = dnn.DNN( - task_tower_cfg.dnn, - self._l2_reg, - name=tower_name, - is_training=self._is_training) + task_tower_cfg.dnn, + self._l2_reg, + name=tower_name, + is_training=self._is_training + ) task_fea = task_dnn(self._features) task_output = tf.layers.dense( - inputs=task_fea, - units=task_tower_cfg.num_class, - kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i) + inputs=task_fea, + units=task_tower_cfg.num_class, + kernel_regularizer=self._l2_reg, + name='dnn_output_%d' % i + ) tower_outputs[tower_name] = task_output self._add_to_prediction_dict(tower_outputs) diff --git a/easy_rec/python/model/uniter.py b/easy_rec/python/model/uniter.py index 40dfc8cb1..3aaf72e66 100644 --- a/easy_rec/python/model/uniter.py +++ b/easy_rec/python/model/uniter.py @@ -2,10 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import dnn -from easy_rec.python.layers import uniter +from easy_rec.python.layers import dnn, uniter from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.uniter_pb2 import Uniter as UNITERConfig # NOQA if tf.__version__ >= '2.0': @@ -19,26 +17,33 @@ class Uniter(RankModel): https://arxiv.org/abs/1909.11740 """ - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(Uniter, self).__init__(model_config, feature_configs, features, - labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(Uniter, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert self._model_config.WhichOneof('model') == 'uniter', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model')) + 'invalid model config: %s' % self._model_config.WhichOneof('model') + ) - self._uniter_layer = uniter.Uniter(model_config, feature_configs, features, - self._model_config.uniter.config, - self._input_layer) + self._uniter_layer = uniter.Uniter( + model_config, feature_configs, features, + self._model_config.uniter.config, self._input_layer + ) self._model_config = self._model_config.uniter def build_predict_graph(self): hidden = self._uniter_layer(self._is_training, l2_reg=self._l2_reg) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, - 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN( + self._model_config.final_dnn, self._l2_reg, 'final_dnn', + self._is_training + ) all_fea = final_dnn_layer(hidden) final = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/wide_and_deep.py b/easy_rec/python/model/wide_and_deep.py index 48b620bd7..bd826416d 100755 --- a/easy_rec/python/model/wide_and_deep.py +++ b/easy_rec/python/model/wide_and_deep.py @@ -1,12 +1,10 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.wide_and_deep_pb2 import WideAndDeep as WideAndDeepConfig # NOQA if tf.__version__ >= '2.0': @@ -15,14 +13,17 @@ class WideAndDeep(RankModel): - def __init__(self, - model_config, - feature_configs, - features, - labels=None, - is_training=False): - super(WideAndDeep, self).__init__(model_config, feature_configs, features, - labels, is_training) + def __init__( + self, + model_config, + feature_configs, + features, + labels=None, + is_training=False + ): + super(WideAndDeep, self).__init__( + model_config, feature_configs, features, labels, is_training + ) assert model_config.WhichOneof('model') == 'wide_and_deep', \ 'invalid model config: %s' % model_config.WhichOneof('model') self._model_config = model_config.wide_and_deep @@ -46,33 +47,40 @@ def build_predict_graph(self): logging.info('wide features dimension: %d' % wide_fea.get_shape()[-1]) self._deep_features = tf.concat(self._deep_features, axis=1) - logging.info('input deep features dimension: %d' % - self._deep_features.get_shape()[-1]) + logging.info( + 'input deep features dimension: %d' % self._deep_features.get_shape()[-1] + ) - deep_layer = dnn.DNN(self._model_config.dnn, self._l2_reg, 'deep_feature', - self._is_training) + deep_layer = dnn.DNN( + self._model_config.dnn, self._l2_reg, 'deep_feature', self._is_training + ) deep_fea = deep_layer(self._deep_features) - logging.info('output deep features dimension: %d' % - deep_fea.get_shape()[-1]) + logging.info( + 'output deep features dimension: %d' % deep_fea.get_shape()[-1] + ) has_final = len(self._model_config.final_dnn.hidden_units) > 0 print('wide_deep has_final_dnn layers = %d' % has_final) if has_final: all_fea = tf.concat([wide_fea, deep_fea], axis=1) - final_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, - 'final_dnn', self._is_training) + final_layer = dnn.DNN( + self._model_config.final_dnn, self._l2_reg, 'final_dnn', + self._is_training + ) all_fea = final_layer(all_fea) output = tf.layers.dense( - all_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='output') + all_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='output' + ) else: deep_out = tf.layers.dense( - deep_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='deep_out') + deep_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='deep_out' + ) output = deep_out + wide_fea self._add_to_prediction_dict(output) @@ -114,7 +122,8 @@ def get_grouped_vars(self, opt_num): (not tmp_var.name.startswith('input_layer_1')): wide_vars.append(tmp_var) elif tmp_var.name.startswith( - 'input_layer') or '/embedding_weights' in tmp_var.name: + 'input_layer' + ) or '/embedding_weights' in tmp_var.name: embedding_vars.append(tmp_var) else: deep_vars.append(tmp_var) diff --git a/easy_rec/python/ops/gen_kafka_ops.py b/easy_rec/python/ops/gen_kafka_ops.py index 16bba500d..bcf99608f 100644 --- a/easy_rec/python/ops/gen_kafka_ops.py +++ b/easy_rec/python/ops/gen_kafka_ops.py @@ -6,10 +6,9 @@ import logging import os -import traceback - import six as _six import tensorflow as tf +import traceback from tensorflow.python import pywrap_tensorflow as _pywrap_tensorflow from tensorflow.python.eager import context as _context from tensorflow.python.eager import core as _core @@ -28,21 +27,24 @@ try: kafka_module = tf.load_op_library(kafka_ops_path) except Exception: - logging.warning('load %s failed: %s' % - (kafka_ops_path, traceback.format_exc())) + logging.warning( + 'load %s failed: %s' % (kafka_ops_path, traceback.format_exc()) + ) @tf_export('io_kafka_dataset_v2') -def io_kafka_dataset_v2(topics, - servers, - group, - eof, - timeout, - config_global, - config_topic, - message_key, - message_offset, - name=None): +def io_kafka_dataset_v2( + topics, + servers, + group, + eof, + timeout, + config_global, + config_topic, + message_key, + message_offset, + name=None +): """Creates a dataset that emits the messages of one or more Kafka topics. Args: @@ -73,29 +75,32 @@ def io_kafka_dataset_v2(topics, A `Tensor` of type `variant`. """ return kafka_module.io_kafka_dataset_v2( - topics=topics, - servers=servers, - group=group, - eof=eof, - timeout=timeout, - config_global=config_global, - config_topic=config_topic, - message_key=message_key, - message_offset=message_offset, - name=name) - - -def io_kafka_dataset_eager_fallback(topics, - servers, - group, - eof, - timeout, - config_global, - config_topic, - message_key, - message_offset, - name=None, - ctx=None): + topics=topics, + servers=servers, + group=group, + eof=eof, + timeout=timeout, + config_global=config_global, + config_topic=config_topic, + message_key=message_key, + message_offset=message_offset, + name=name + ) + + +def io_kafka_dataset_eager_fallback( + topics, + servers, + group, + eof, + timeout, + config_global, + config_topic, + message_key, + message_offset, + name=None, + ctx=None +): """This is the slowpath function for Eager mode. This is for function io_kafka_dataset @@ -111,19 +116,21 @@ def io_kafka_dataset_eager_fallback(topics, message_key = _ops.convert_to_tensor(message_key, _dtypes.bool) message_offset = _ops.convert_to_tensor(message_offset, _dtypes.bool) _inputs_flat = [ - topics, servers, group, eof, timeout, config_global, config_topic, - message_key, message_offset + topics, servers, group, eof, timeout, config_global, config_topic, + message_key, message_offset ] _attrs = None _result = _execute.execute( - b'IOKafkaDataset', - 1, - inputs=_inputs_flat, - attrs=_attrs, - ctx=_ctx, - name=name) - _execute.record_gradient('IOKafkaDataset', _inputs_flat, _attrs, _result, - name) + b'IOKafkaDataset', + 1, + inputs=_inputs_flat, + attrs=_attrs, + ctx=_ctx, + name=name + ) + _execute.record_gradient( + 'IOKafkaDataset', _inputs_flat, _attrs, _result, name + ) _result, = _result return _result @@ -144,24 +151,28 @@ def io_write_kafka_v2(message, topic, servers, name=None): _ctx = _context._context if _ctx is None or not _ctx._eager_context.is_eager: _op = kafka_module.io_write_kafka_v2( - message=message, topic=topic, servers=servers, name=name) + message=message, topic=topic, servers=servers, name=name + ) _result = _op.outputs[:] _inputs_flat = _op.inputs _attrs = None - _execute.record_gradient('IOWriteKafka', _inputs_flat, _attrs, _result, - name) + _execute.record_gradient( + 'IOWriteKafka', _inputs_flat, _attrs, _result, name + ) _result, = _result return _result else: try: _result = _pywrap_tensorflow.TFE_Py_FastPathExecute( - _ctx._context_handle, _ctx._eager_context.device_name, 'IOWriteKafka', - name, _ctx._post_execution_callbacks, message, topic, servers) + _ctx._context_handle, _ctx._eager_context.device_name, 'IOWriteKafka', + name, _ctx._post_execution_callbacks, message, topic, servers + ) return _result except _core._FallbackException: return io_write_kafka_eager_fallback( - message, topic, servers, name=name, ctx=_ctx) + message, topic, servers, name=name, ctx=_ctx + ) except _core._NotOkStatusException as e: if name is not None: message = e.message + ' name: ' + name @@ -170,7 +181,9 @@ def io_write_kafka_v2(message, topic, servers, name=None): _six.raise_from(_core._status_to_exception(e.code, message), None) -def io_write_kafka_eager_fallback(message, topic, servers, name=None, ctx=None): +def io_write_kafka_eager_fallback( + message, topic, servers, name=None, ctx=None +): """This is the slowpath function for Eager mode. This is for function io_write_kafka @@ -182,12 +195,8 @@ def io_write_kafka_eager_fallback(message, topic, servers, name=None, ctx=None): _inputs_flat = [message, topic, servers] _attrs = None _result = _execute.execute( - b'IOWriteKafka', - 1, - inputs=_inputs_flat, - attrs=_attrs, - ctx=_ctx, - name=name) + b'IOWriteKafka', 1, inputs=_inputs_flat, attrs=_attrs, ctx=_ctx, name=name + ) _execute.record_gradient('IOWriteKafka', _inputs_flat, _attrs, _result, name) _result, = _result return _result diff --git a/easy_rec/python/ops/gen_str_avx_op.py b/easy_rec/python/ops/gen_str_avx_op.py index d022d52cb..a42293e01 100644 --- a/easy_rec/python/ops/gen_str_avx_op.py +++ b/easy_rec/python/ops/gen_str_avx_op.py @@ -2,7 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os - import tensorflow as tf from tensorflow.python.ops import string_ops @@ -23,6 +22,8 @@ def str_split_by_chr(input_str, sep, skip_empty): assert len(sep) == 1, \ 'invalid data_config.separator(%s) len(%d) != 1' % ( sep, len(sep)) - return str_avx_op.avx512_string_split(input_str, sep, skip_empty=skip_empty) + return str_avx_op.avx512_string_split( + input_str, sep, skip_empty=skip_empty + ) else: return string_ops.string_split(input_str, sep, skip_empty=skip_empty) diff --git a/easy_rec/python/ops/incr_record.py b/easy_rec/python/ops/incr_record.py index b4bad11e4..cc0c49e30 100644 --- a/easy_rec/python/ops/incr_record.py +++ b/easy_rec/python/ops/incr_record.py @@ -2,7 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os - import tensorflow as tf import easy_rec @@ -20,11 +19,13 @@ get_sparse_indices = None set_sparse_indices = None kv_resource_incr_gather = None - logging.warning('failed to import gen_io_ops.collect_sparse_indices: %s' % - str(ex)) + logging.warning( + 'failed to import gen_io_ops.collect_sparse_indices: %s' % str(ex) + ) except Exception as ex: get_sparse_indices = None set_sparse_indices = None kv_resource_incr_gather = None - logging.warning('failed to import gen_io_ops.collect_sparse_indices: %s' % - str(ex)) + logging.warning( + 'failed to import gen_io_ops.collect_sparse_indices: %s' % str(ex) + ) diff --git a/easy_rec/python/predict.py b/easy_rec/python/predict.py index ced1a7573..2a21c68ea 100644 --- a/easy_rec/python/predict.py +++ b/easy_rec/python/predict.py @@ -4,54 +4,58 @@ import json import logging import os - import tensorflow as tf from tensorflow.python.lib.io import file_io from easy_rec.python.inference.csv_predictor import CSVPredictor +from easy_rec.python.inference.hive_parquet_predictor import HiveParquetPredictor # NOQA from easy_rec.python.inference.hive_predictor import HivePredictor from easy_rec.python.inference.parquet_predictor import ParquetPredictor from easy_rec.python.inference.parquet_predictor_v2 import ParquetPredictorV2 from easy_rec.python.main import predict from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.utils import config_util -from easy_rec.python.utils import numpy_utils +from easy_rec.python.utils import config_util, numpy_utils from easy_rec.python.utils.hive_utils import HiveUtils -from easy_rec.python.inference.hive_parquet_predictor import HiveParquetPredictor # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) tf.app.flags.DEFINE_string('input_path', None, 'predict data path') tf.app.flags.DEFINE_string('output_path', None, 'path to save predict result') tf.app.flags.DEFINE_integer('batch_size', 1024, help='batch size') -tf.app.flags.DEFINE_bool('with_header', False, - 'whether the input csv file has header') +tf.app.flags.DEFINE_bool( + 'with_header', False, 'whether the input csv file has header' +) # predict by checkpoint -tf.app.flags.DEFINE_string('pipeline_config_path', None, - 'Path to pipeline config ' - 'file.') tf.app.flags.DEFINE_string( - 'checkpoint_path', None, 'checkpoint to be evaled ' - ' if not specified, use the latest checkpoint in ' - 'train_config.model_dir') + 'pipeline_config_path', None, 'Path to pipeline config ' + 'file.' +) +tf.app.flags.DEFINE_string( + 'checkpoint_path', None, 'checkpoint to be evaled ' + ' if not specified, use the latest checkpoint in ' + 'train_config.model_dir' +) tf.app.flags.DEFINE_string('model_dir', None, help='will update the model_dir') # predict by saved_model tf.app.flags.DEFINE_string('saved_model_dir', None, help='save model dir') tf.app.flags.DEFINE_string( - 'reserved_cols', 'ALL_COLUMNS', - 'columns to keep from input table, they are separated with ,') + 'reserved_cols', 'ALL_COLUMNS', + 'columns to keep from input table, they are separated with ,' +) +tf.app.flags.DEFINE_string( + 'output_cols', 'ALL_COLUMNS', + 'output columns, such as: score float. multiple columns are separated by ,' +) tf.app.flags.DEFINE_string( - 'output_cols', 'ALL_COLUMNS', - 'output columns, such as: score float. multiple columns are separated by ,') -tf.app.flags.DEFINE_string('output_sep', chr(1), - 'separator of predict result file') + 'output_sep', chr(1), 'separator of predict result file' +) tf.app.flags.DEFINE_string('selected_cols', None, '') tf.app.flags.DEFINE_string('fg_json_path', '', '') tf.app.flags.DEFINE_string('ds_vector_recall', '', '') @@ -76,60 +80,68 @@ def main(argv): pipeline_config_path = FLAGS.pipeline_config_path else: pipeline_config_path = config_util.search_pipeline_config( - FLAGS.saved_model_dir) + FLAGS.saved_model_dir + ) pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False) + pipeline_config_path, False + ) data_config = pipeline_config.data_config input_type = get_input_type(FLAGS.input_type, data_config) if input_type in [data_config.HiveParquetInput, data_config.HiveInput]: all_cols, all_col_types = HiveUtils( - data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input).get_all_cols( - FLAGS.input_path) + data_config=pipeline_config.data_config, + hive_config=pipeline_config.hive_train_input + ).get_all_cols(FLAGS.input_path) if input_type == DatasetConfig.HiveParquetInput: predictor = HiveParquetPredictor( - FLAGS.saved_model_dir, - pipeline_config.data_config, - fg_json_path=FLAGS.fg_json_path, - hive_config=pipeline_config.hive_train_input, - output_sep=FLAGS.output_sep, - all_cols=all_cols, - all_col_types=all_col_types) + FLAGS.saved_model_dir, + pipeline_config.data_config, + fg_json_path=FLAGS.fg_json_path, + hive_config=pipeline_config.hive_train_input, + output_sep=FLAGS.output_sep, + all_cols=all_cols, + all_col_types=all_col_types + ) else: predictor = HivePredictor( - FLAGS.saved_model_dir, - pipeline_config.data_config, - fg_json_path=FLAGS.fg_json_path, - hive_config=pipeline_config.hive_train_input, - output_sep=FLAGS.output_sep, - all_cols=all_cols, - all_col_types=all_col_types) + FLAGS.saved_model_dir, + pipeline_config.data_config, + fg_json_path=FLAGS.fg_json_path, + hive_config=pipeline_config.hive_train_input, + output_sep=FLAGS.output_sep, + all_cols=all_cols, + all_col_types=all_col_types + ) elif input_type in [data_config.ParquetInput, data_config.ParquetInputV2]: predictor_cls = ParquetPredictor if input_type == data_config.ParquetInputV2: predictor_cls = ParquetPredictorV2 predictor = predictor_cls( - FLAGS.saved_model_dir, - pipeline_config.data_config, - ds_vector_recall=FLAGS.ds_vector_recall, - fg_json_path=FLAGS.fg_json_path, - selected_cols=FLAGS.selected_cols, - output_sep=FLAGS.output_sep, - pipeline_config=pipeline_config) + FLAGS.saved_model_dir, + pipeline_config.data_config, + ds_vector_recall=FLAGS.ds_vector_recall, + fg_json_path=FLAGS.fg_json_path, + selected_cols=FLAGS.selected_cols, + output_sep=FLAGS.output_sep, + pipeline_config=pipeline_config + ) elif input_type == data_config.CSVInput: predictor = CSVPredictor( - FLAGS.saved_model_dir, - pipeline_config.data_config, - FLAGS.with_header, - ds_vector_recall=FLAGS.ds_vector_recall, - fg_json_path=FLAGS.fg_json_path, - selected_cols=FLAGS.selected_cols, - output_sep=FLAGS.output_sep) + FLAGS.saved_model_dir, + pipeline_config.data_config, + FLAGS.with_header, + ds_vector_recall=FLAGS.ds_vector_recall, + fg_json_path=FLAGS.fg_json_path, + selected_cols=FLAGS.selected_cols, + output_sep=FLAGS.output_sep + ) else: assert False, 'invalid input type: %s' % input_class_map_r[input_type] - logging.info('input_path = %s, output_path = %s' % - (FLAGS.input_path, FLAGS.output_path)) + logging.info( + 'input_path = %s, output_path = %s' % + (FLAGS.input_path, FLAGS.output_path) + ) if 'TF_CONFIG' in os.environ: tf_config = json.loads(os.environ['TF_CONFIG']) worker_num = len(tf_config['cluster']['worker']) @@ -138,27 +150,31 @@ def main(argv): worker_num = 1 task_index = 0 predictor.predict_impl( - FLAGS.input_path, - FLAGS.output_path, - reserved_cols=FLAGS.reserved_cols, - output_cols=FLAGS.output_cols, - batch_size=FLAGS.batch_size, - slice_id=task_index, - slice_num=worker_num) + FLAGS.input_path, + FLAGS.output_path, + reserved_cols=FLAGS.reserved_cols, + output_cols=FLAGS.output_cols, + batch_size=FLAGS.batch_size, + slice_id=task_index, + slice_num=worker_num + ) else: logging.info('Predict by checkpoint_path.') assert FLAGS.model_dir or FLAGS.pipeline_config_path, 'At least one of model_dir and pipeline_config_path exists.' if FLAGS.model_dir: pipeline_config_path = os.path.join(FLAGS.model_dir, 'pipeline.config') if file_io.file_exists(pipeline_config_path): - logging.info('update pipeline_config_path to %s' % pipeline_config_path) + logging.info( + 'update pipeline_config_path to %s' % pipeline_config_path + ) else: pipeline_config_path = FLAGS.pipeline_config_path else: pipeline_config_path = FLAGS.pipeline_config_path - pred_result = predict(pipeline_config_path, FLAGS.checkpoint_path, - FLAGS.input_path) + pred_result = predict( + pipeline_config_path, FLAGS.checkpoint_path, FLAGS.input_path + ) if FLAGS.output_path is not None: logging.info('will save predict result to %s' % FLAGS.output_path) with tf.gfile.GFile(FLAGS.output_path, 'wb') as fout: diff --git a/easy_rec/python/test/csv_input_test.py b/easy_rec/python/test/csv_input_test.py index ae0793fa5..68048cce6 100644 --- a/easy_rec/python/test/csv_input_test.py +++ b/easy_rec/python/test/csv_input_test.py @@ -3,17 +3,15 @@ """Define cv_input, the base class for cv tasks.""" import os -import unittest - import tensorflow as tf +import unittest from google.protobuf import text_format from easy_rec.python.input.csv_input import CSVInput from easy_rec.python.input.csv_input_ex import CSVInputEx from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.protos.feature_config_pb2 import FeatureConfig -from easy_rec.python.utils import config_util -from easy_rec.python.utils import constant +from easy_rec.python.utils import config_util, constant from easy_rec.python.utils.test_utils import RunAsSubprocess if tf.__version__ >= '2.0': @@ -72,8 +70,9 @@ def test_csv_data(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput(dataset_config, feature_configs, - self._input_path).create_input() + train_input_fn = CSVInput( + dataset_config, feature_configs, self._input_path + ).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -81,9 +80,10 @@ def test_csv_data(self): init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=False) + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=False + ) with self.test_session(config=session_config) as sess: sess.run(init_op) feature_dict, label_dict = sess.run([features, labels]) @@ -135,8 +135,9 @@ def test_csv_data_flt_to_str_exception(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput(dataset_config, feature_configs, - self._input_path).create_input() + train_input_fn = CSVInput( + dataset_config, feature_configs, self._input_path + ).create_input() try: dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) # noqa: F841 passed = True @@ -192,8 +193,9 @@ def test_csv_data_flt_to_str(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput(dataset_config, feature_configs, - self._input_path).create_input() + train_input_fn = CSVInput( + dataset_config, feature_configs, self._input_path + ).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) @@ -203,9 +205,10 @@ def test_csv_data_flt_to_str(self): init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=False) + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=False + ) with self.test_session(config=session_config) as sess: sess.run(init_op) feature_dict, label_dict = sess.run([features, labels]) @@ -252,8 +255,9 @@ def test_csv_input_ex(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInputEx(dataset_config, feature_configs, - self._input_path_with_quote).create_input() + train_input_fn = CSVInputEx( + dataset_config, feature_configs, self._input_path_with_quote + ).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -261,15 +265,18 @@ def test_csv_input_ex(self): init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=False) + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=False + ) with self.test_session(config=session_config) as sess: sess.run(init_op) feature_dict, label_dict = sess.run([features, labels]) - @unittest.skipIf('AVX_TEST' not in os.environ, - 'Only execute when avx512 instructions are supported') + @unittest.skipIf( + 'AVX_TEST' not in os.environ, + 'Only execute when avx512 instructions are supported' + ) @RunAsSubprocess def test_csv_input_ex_avx(self): constant.enable_avx_str_split() @@ -319,8 +326,9 @@ def test_csv_data_ignore_error(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput(dataset_config, feature_configs, - self._input_path_with_quote).create_input() + train_input_fn = CSVInput( + dataset_config, feature_configs, self._input_path_with_quote + ).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -328,9 +336,10 @@ def test_csv_data_ignore_error(self): init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=False) + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=False + ) with self.test_session(config=session_config) as sess: sess.run(init_op) feature_dict, label_dict = sess.run([features, labels]) diff --git a/easy_rec/python/test/custom_early_stop_func.py b/easy_rec/python/test/custom_early_stop_func.py index fbc2f3cba..68de6a4e3 100644 --- a/easy_rec/python/test/custom_early_stop_func.py +++ b/easy_rec/python/test/custom_early_stop_func.py @@ -13,7 +13,8 @@ def custom_early_stop_func(eval_results, func_param): val = metrics[metric_name] if val > tmp_thre: logging.info( - 'At step %s, metric "%s" has value %s, which is larger than %s, causing early stop.' - % (step, metric_name, val, tmp_thre)) + 'At step %s, metric "%s" has value %s, which is larger than %s, causing early stop.' + % (step, metric_name, val, tmp_thre) + ) return True return False diff --git a/easy_rec/python/test/dh_local_run.py b/easy_rec/python/test/dh_local_run.py index be514d382..28f2b7a51 100644 --- a/easy_rec/python/test/dh_local_run.py +++ b/easy_rec/python/test/dh_local_run.py @@ -3,18 +3,16 @@ import os import shutil import sys - import tensorflow as tf from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig -from easy_rec.python.test.odps_test_util import delete_oss_path -from easy_rec.python.test.odps_test_util import get_oss_bucket +from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA from easy_rec.python.utils import test_utils logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) odps_oss_config = OdpsOSSConfig(script_path='./samples/dh_script') @@ -37,8 +35,9 @@ def test_datahub_train_eval(self): odps_cmd = OdpsCommand(odps_oss_config) self._success = test_utils.test_datahub_train_eval( - '%s/configs/deepfm.config' % odps_oss_config.temp_dir, odps_oss_config, - self._test_dir) + '%s/configs/deepfm.config' % odps_oss_config.temp_dir, odps_oss_config, + self._test_dir + ) odps_cmd.run_list(end) self.assertTrue(self._success) @@ -46,23 +45,30 @@ def test_datahub_train_eval(self): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--odps_config', type=str, default=None, help='odps config path') + '--odps_config', type=str, default=None, help='odps config path' + ) parser.add_argument( - '--oss_config', type=str, default=None, help='ossutilconfig path') + '--oss_config', type=str, default=None, help='ossutilconfig path' + ) parser.add_argument( - '--bucket_name', type=str, default=None, help='test oss bucket name') + '--bucket_name', type=str, default=None, help='test oss bucket name' + ) parser.add_argument('--arn', type=str, default=None, help='oss rolearn') parser.add_argument( - '--odpscmd', type=str, default='odpscmd', help='odpscmd path') + '--odpscmd', type=str, default='odpscmd', help='odpscmd path' + ) parser.add_argument( - '--algo_project', type=str, default=None, help='algo project name') + '--algo_project', type=str, default=None, help='algo project name' + ) parser.add_argument( - '--algo_res_project', - type=str, - default=None, - help='algo resource project name') + '--algo_res_project', + type=str, + default=None, + help='algo resource project name' + ) parser.add_argument( - '--algo_version', type=str, default=None, help='algo version') + '--algo_version', type=str, default=None, help='algo version' + ) args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] @@ -88,8 +94,8 @@ def test_datahub_train_eval(self): odps_oss_config.bucket_name = args.bucket_name prepare(odps_oss_config) start = [ - 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql' + 'deep_fm/create_external_deepfm_table.sql', + 'deep_fm/create_inner_deepfm_table.sql' ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) @@ -97,8 +103,10 @@ def test_datahub_train_eval(self): odps_oss_config.init_dh_and_odps() tf.test.main() # delete oss path - bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name) + bucket = get_oss_bucket( + odps_oss_config.oss_key, odps_oss_config.oss_secret, + odps_oss_config.endpoint, odps_oss_config.bucket_name + ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp shutil.rmtree(odps_oss_config.temp_dir) diff --git a/easy_rec/python/test/embed_test.py b/easy_rec/python/test/embed_test.py index e499b29d0..3d2990006 100644 --- a/easy_rec/python/test/embed_test.py +++ b/easy_rec/python/test/embed_test.py @@ -2,7 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import numpy as np import tensorflow as tf from google.protobuf import text_format @@ -11,8 +10,7 @@ from easy_rec.python.feature_column.feature_column import FeatureColumnParser from easy_rec.python.input.dummy_input import DummyInput from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.protos.feature_config_pb2 import FeatureConfig -from easy_rec.python.protos.feature_config_pb2 import WideOrDeep +from easy_rec.python.protos.feature_config_pb2 import FeatureConfig, WideOrDeep if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -64,7 +62,8 @@ def test_raw_embed(self): feature_configs = [feature_config] features = {'field1': tf.constant(['0.1,0.2,0.3,0.4,0.5'])} dummy_input = DummyInput( - data_config, feature_configs, '', input_vals=features) + data_config, feature_configs, '', input_vals=features + ) field_dict, _ = dummy_input._build(tf.estimator.ModeKeys.TRAIN, {}) wide_and_deep_dict = {'field1': WideOrDeep.WIDE_AND_DEEP} @@ -130,7 +129,8 @@ def test_seq_multi_embed(self): feature_configs = [feature_config] features = {'field1': tf.constant(['0112', '132430'])} dummy_input = DummyInput( - data_config, feature_configs, '', input_vals=features) + data_config, feature_configs, '', input_vals=features + ) field_dict, _ = dummy_input._build(tf.estimator.ModeKeys.TRAIN, {}) wide_and_deep_dict = {'field1': WideOrDeep.DEEP} diff --git a/easy_rec/python/test/emr_run.py b/easy_rec/python/test/emr_run.py index 305190544..ab3bb5a24 100644 --- a/easy_rec/python/test/emr_run.py +++ b/easy_rec/python/test/emr_run.py @@ -6,18 +6,16 @@ import os import shutil import sys - import tensorflow as tf from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig -from easy_rec.python.test.odps_test_util import delete_oss_path -from easy_rec.python.test.odps_test_util import get_oss_bucket +from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA from easy_rec.python.utils import test_utils logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) odps_oss_config = OdpsOSSConfig(script_path='./samples/emr_script') @@ -28,7 +26,8 @@ class TestPipelineOnEmr(tf.test.TestCase): def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) self._test_hdfs_dir = test_utils.get_hdfs_tmp_dir( - 'hdfs://emr-header-1:9000/user/easy_rec/emr_test') + 'hdfs://emr-header-1:9000/user/easy_rec/emr_test' + ) self._success = True logging.info('test hdfs dir: %s' % self._test_hdfs_dir) @@ -39,28 +38,31 @@ def tearDown(self): def test_deepfm_train_eval_export(self): start = [ - 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql' + 'deep_fm/create_external_deepfm_table.sql', + 'deep_fm/create_inner_deepfm_table.sql' ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) odps_cmd.run_list(start) self._success = test_utils.test_hdfs_train_eval( - '%s/configs/deepfm.config' % odps_oss_config.temp_dir, - '%s/yaml_config/train.paitf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir) + '%s/configs/deepfm.config' % odps_oss_config.temp_dir, + '%s/yaml_config/train.paitf.yaml' % odps_oss_config.temp_dir, + self._test_hdfs_dir + ) self.assertTrue(self._success) self._success = test_utils.test_hdfs_eval( - '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, - '%s/yaml_config/eval.tf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir) + '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, + '%s/yaml_config/eval.tf.yaml' % odps_oss_config.temp_dir, + self._test_hdfs_dir + ) self.assertTrue(self._success) self._success = test_utils.test_hdfs_export( - '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, - '%s/yaml_config/export.tf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir) + '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, + '%s/yaml_config/export.tf.yaml' % odps_oss_config.temp_dir, + self._test_hdfs_dir + ) self.assertTrue(self._success) odps_cmd.run_list(end) @@ -69,23 +71,30 @@ def test_deepfm_train_eval_export(self): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--odps_config', type=str, default=None, help='odps config path') + '--odps_config', type=str, default=None, help='odps config path' + ) parser.add_argument( - '--oss_config', type=str, default=None, help='ossutilconfig path') + '--oss_config', type=str, default=None, help='ossutilconfig path' + ) parser.add_argument( - '--bucket_name', type=str, default=None, help='test oss bucket name') + '--bucket_name', type=str, default=None, help='test oss bucket name' + ) parser.add_argument('--arn', type=str, default=None, help='oss rolearn') parser.add_argument( - '--odpscmd', type=str, default='odpscmd', help='odpscmd path') + '--odpscmd', type=str, default='odpscmd', help='odpscmd path' + ) parser.add_argument( - '--algo_project', type=str, default=None, help='algo project name') + '--algo_project', type=str, default=None, help='algo project name' + ) parser.add_argument( - '--algo_res_project', - type=str, - default=None, - help='algo resource project name') + '--algo_res_project', + type=str, + default=None, + help='algo resource project name' + ) parser.add_argument( - '--algo_version', type=str, default=None, help='algo version') + '--algo_version', type=str, default=None, help='algo version' + ) args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] for unk_arg in unknown_args: @@ -112,8 +121,10 @@ def test_deepfm_train_eval_export(self): prepare(odps_oss_config) tf.test.main() # delete oss path - bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name) + bucket = get_oss_bucket( + odps_oss_config.oss_key, odps_oss_config.oss_secret, + odps_oss_config.endpoint, odps_oss_config.bucket_name + ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp shutil.rmtree(odps_oss_config.temp_dir) diff --git a/easy_rec/python/test/eval_metric_test.py b/easy_rec/python/test/eval_metric_test.py index 22e019b6a..86fda46e2 100644 --- a/easy_rec/python/test/eval_metric_test.py +++ b/easy_rec/python/test/eval_metric_test.py @@ -3,7 +3,6 @@ from __future__ import division import logging - import tensorflow as tf from absl.testing import parameterized @@ -44,61 +43,72 @@ def test_gauc_all_negative_label(self): self.assertAlmostEqual(score, 0.0) @parameterized.named_parameters( - [['_reduction_mean', 'mean', 0.5833333], - ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], - ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6]]) + [ + ['_reduction_mean', 'mean', 0.5833333], + ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], + ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6] + ] + ) @RunAsSubprocess def test_gauc(self, reduction, expected): from easy_rec.python.core.metrics import gauc - labels = tf.placeholder(dtype=tf.int32, shape=(None,)) - probs = tf.placeholder(dtype=tf.float32, shape=(None,)) - uids = tf.placeholder(dtype=tf.int32, shape=(None,)) + labels = tf.placeholder(dtype=tf.int32, shape=(None, )) + probs = tf.placeholder(dtype=tf.float32, shape=(None, )) + uids = tf.placeholder(dtype=tf.int32, shape=(None, )) value_op, update_op = gauc(labels, probs, uids, reduction=reduction) with tf.Session() as sess: sess.run( - update_op, - feed_dict={ - labels: [1, 0, 1, 1, 0], - probs: [0.9, 0.8, 0.7, 0.6, 0.5], - uids: [1, 1, 1, 1, 1] - }) + update_op, + feed_dict={ + labels: [1, 0, 1, 1, 0], + probs: [0.9, 0.8, 0.7, 0.6, 0.5], + uids: [1, 1, 1, 1, 1] + } + ) sess.run( - update_op, - feed_dict={ - labels: [1, 0, 0, 1], - probs: [0.9, 0.8, 0.7, 0.6], - uids: [2, 2, 2, 2] - }) + update_op, + feed_dict={ + labels: [1, 0, 0, 1], + probs: [0.9, 0.8, 0.7, 0.6], + uids: [2, 2, 2, 2] + } + ) score = sess.run(value_op) self.assertAlmostEqual(score, expected) @parameterized.named_parameters( - [['_reduction_mean', 'mean', 0.5833333], - ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], - ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6]]) + [ + ['_reduction_mean', 'mean', 0.5833333], + ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], + ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6] + ] + ) @RunAsSubprocess def test_session_auc(self, reduction, expected): from easy_rec.python.core.metrics import session_auc - labels = tf.placeholder(dtype=tf.int32, shape=(None,)) - probs = tf.placeholder(dtype=tf.float32, shape=(None,)) - session_ids = tf.placeholder(dtype=tf.int32, shape=(None,)) + labels = tf.placeholder(dtype=tf.int32, shape=(None, )) + probs = tf.placeholder(dtype=tf.float32, shape=(None, )) + session_ids = tf.placeholder(dtype=tf.int32, shape=(None, )) value_op, update_op = session_auc( - labels, probs, session_ids, reduction=reduction) + labels, probs, session_ids, reduction=reduction + ) with tf.Session() as sess: sess.run( - update_op, - feed_dict={ - labels: [1, 0, 1, 1, 0], - probs: [0.9, 0.8, 0.7, 0.6, 0.5], - session_ids: [1, 1, 1, 1, 1] - }) + update_op, + feed_dict={ + labels: [1, 0, 1, 1, 0], + probs: [0.9, 0.8, 0.7, 0.6, 0.5], + session_ids: [1, 1, 1, 1, 1] + } + ) sess.run( - update_op, - feed_dict={ - labels: [1, 0, 0, 1], - probs: [0.9, 0.8, 0.7, 0.6], - session_ids: [2, 2, 2, 2] - }) + update_op, + feed_dict={ + labels: [1, 0, 0, 1], + probs: [0.9, 0.8, 0.7, 0.6], + session_ids: [2, 2, 2, 2] + } + ) score = sess.run(value_op) self.assertAlmostEqual(score, expected) diff --git a/easy_rec/python/test/excel_convert_test.py b/easy_rec/python/test/excel_convert_test.py index a860d3277..6f5f7d515 100644 --- a/easy_rec/python/test/excel_convert_test.py +++ b/easy_rec/python/test/excel_convert_test.py @@ -3,7 +3,6 @@ import logging import os - import tensorflow as tf from easy_rec.python.utils import test_utils @@ -29,13 +28,16 @@ def test_deepfm_convert(self): --train_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv --eval_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv """ % pipeline_config_path - proc = test_utils.run_cmd(convert_cmd, - '%s/log_%s.txt' % (test_dir, 'convert')) + proc = test_utils.run_cmd( + convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + ) proc.wait() self.assertTrue(proc.returncode == 0) self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir)) + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir + ) + ) test_utils.clean_up(test_dir) def test_multi_tower_convert(self): @@ -50,13 +52,16 @@ def test_multi_tower_convert(self): --train_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv --eval_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv """ % pipeline_config_path - proc = test_utils.run_cmd(convert_cmd, - '%s/log_%s.txt' % (test_dir, 'convert')) + proc = test_utils.run_cmd( + convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + ) proc.wait() self.assertTrue(proc.returncode == 0) self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir)) + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir + ) + ) test_utils.clean_up(test_dir) diff --git a/easy_rec/python/test/export_test.py b/easy_rec/python/test/export_test.py index 23bff2890..b41409b80 100644 --- a/easy_rec/python/test/export_test.py +++ b/easy_rec/python/test/export_test.py @@ -5,17 +5,15 @@ import functools import json import logging -import os -import unittest - import numpy as np +import os import tensorflow as tf +import unittest from tensorflow.python.platform import gfile import easy_rec from easy_rec.python.inference.predictor import Predictor -from easy_rec.python.utils import config_util -from easy_rec.python.utils import test_utils +from easy_rec.python.utils import config_util, test_utils from easy_rec.python.utils.test_utils import RunAsSubprocess @@ -28,13 +26,15 @@ def tearDown(self): test_utils.set_gpu_id(None) @RunAsSubprocess - def _predict_and_check(self, - data_path, - saved_model_dir, - cmp_result, - keys=['probs'], - separator=',', - tol=1e-4): + def _predict_and_check( + self, + data_path, + saved_model_dir, + cmp_result, + keys=['probs'], + separator=',', + tol=1e-4 + ): predictor = Predictor(saved_model_dir) with open(data_path, 'r') as fin: inputs = [] @@ -74,39 +74,46 @@ def _extract_rtp_data(self, input_path, output_path, separator=';'): fout.write('%s\n' % line_toks[-1]) def test_multi_tower(self): - self._export_test('samples/model_config/multi_tower_export.config', - self._extract_data) + self._export_test( + 'samples/model_config/multi_tower_export.config', self._extract_data + ) def test_filter_input(self): - self._export_test('samples/model_config/export_filter_input.config', - self._extract_data) + self._export_test( + 'samples/model_config/export_filter_input.config', self._extract_data + ) def test_mmoe(self): self._export_test( - 'samples/model_config/mmoe_on_taobao.config', - functools.partial(self._extract_data, offset=2), - keys=['probs_ctr', 'probs_cvr']) + 'samples/model_config/mmoe_on_taobao.config', + functools.partial(self._extract_data, offset=2), + keys=['probs_ctr', 'probs_cvr'] + ) def test_fg(self): self._export_test( - 'samples/model_config/taobao_fg.config', - self._extract_rtp_data, - separator='') + 'samples/model_config/taobao_fg.config', + self._extract_rtp_data, + separator='' + ) def test_fg_export(self): self._export_test( - 'samples/model_config/taobao_fg_export.config', - self._extract_rtp_data, - separator='', - test_multi=False) + 'samples/model_config/taobao_fg_export.config', + self._extract_rtp_data, + separator='', + test_multi=False + ) def test_export_with_asset(self): pipeline_config_path = 'samples/model_config/taobao_fg.config' test_dir = test_utils.get_tmp_dir() # prepare model self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir)) + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir + ) + ) test_utils.set_gpu_id(None) config_path = os.path.join(test_dir, 'pipeline.config') export_dir = os.path.join(test_dir, 'export/') @@ -117,11 +124,12 @@ def test_export_with_asset(self): --asset_files fg.json:samples/model_config/taobao_fg.json --export_done_file ExportDone """ % ( - config_path, - export_dir, + config_path, + export_dir, + ) + proc = test_utils.run_cmd( + export_cmd, '%s/log_%s.txt' % (test_dir, 'export') ) - proc = test_utils.run_cmd(export_cmd, - '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() self.assertTrue(proc.returncode == 0) files = gfile.Glob(export_dir + '*') @@ -145,50 +153,60 @@ def _post_check_func(pipeline_config): --checkpoint_path %s --export_dir %s """ % (pipeline_config_path, ckpt_path, export_dir) - proc = test_utils.run_cmd(export_cmd, - '%s/log_%s.txt' % (test_dir, 'export')) + proc = test_utils.run_cmd( + export_cmd, '%s/log_%s.txt' % (test_dir, 'export') + ) proc.wait() return proc.returncode == 0 # prepare model self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, - test_dir=test_dir, - post_check_func=_post_check_func)) + test_utils.test_single_train_eval( + pipeline_config_path, + test_dir=test_dir, + post_check_func=_post_check_func + ) + ) def test_multi_class_predict(self): self._export_test( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - extract_data_func=self._extract_data, - keys=['probs', 'logits', 'probs_y', 'logits_y', 'y']) - - def _export_test(self, - pipeline_config_path, - extract_data_func=None, - separator=',', - keys=['probs'], - test_multi=True): + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + extract_data_func=self._extract_data, + keys=['probs', 'logits', 'probs_y', 'logits_y', 'y'] + ) + + def _export_test( + self, + pipeline_config_path, + extract_data_func=None, + separator=',', + keys=['probs'], + test_multi=True + ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) # prepare model self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir)) + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir + ) + ) test_utils.set_gpu_id(None) # prepare two version config config_path_single = os.path.join(test_dir, 'pipeline.config') config_path_multi = os.path.join(test_dir, 'pipeline_v2.config') pipeline_config = config_util.get_configs_from_pipeline_file( - config_path_single) + config_path_single + ) if pipeline_config.export_config.multi_placeholder: config_path_single, config_path_multi = config_path_multi, config_path_single pipeline_config.export_config.multi_placeholder =\ not pipeline_config.export_config.multi_placeholder - config_util.save_pipeline_config(pipeline_config, test_dir, - 'pipeline_v2.config') + config_util.save_pipeline_config( + pipeline_config, test_dir, 'pipeline_v2.config' + ) # prepare two version export dir export_dir_single = os.path.join(test_dir, 'train/export/final') @@ -198,8 +216,9 @@ def _export_test(self, --pipeline_config_path %s --export_dir %s """ % (config_path_multi, export_dir_multi) - proc = test_utils.run_cmd(export_cmd, - '%s/log_%s.txt' % (test_dir, 'export')) + proc = test_utils.run_cmd( + export_cmd, '%s/log_%s.txt' % (test_dir, 'export') + ) proc.wait() self.assertTrue(proc.returncode == 0) @@ -210,8 +229,9 @@ def _export_test(self, --pipeline_config_path %s --output_path %s """ % (config_path_single, result_path) - proc = test_utils.run_cmd(predict_cmd % (), - '%s/log_%s.txt' % (test_dir, 'predict')) + proc = test_utils.run_cmd( + predict_cmd % (), '%s/log_%s.txt' % (test_dir, 'predict') + ) proc.wait() self.assertTrue(proc.returncode == 0) with open(result_path, 'r') as fin: @@ -226,25 +246,29 @@ def _export_test(self, extract_data_func(test_data_path, tmp_data_path) test_data_path = tmp_data_path self._predict_and_check( + test_data_path, + export_dir_single, + cmp_result, + keys=keys, + separator=separator + ) + if test_multi: + self._predict_and_check( test_data_path, - export_dir_single, + export_dir_multi, cmp_result, keys=keys, - separator=separator) - if test_multi: - self._predict_and_check( - test_data_path, - export_dir_multi, - cmp_result, - keys=keys, - separator=separator) + separator=separator + ) test_utils.clean_up(test_dir) - def _test_big_model_export(self, - pipeline_config_path, - test_data_path, - extract_data_func=None, - total_steps=50): + def _test_big_model_export( + self, + pipeline_config_path, + test_data_path, + extract_data_func=None, + total_steps=50 + ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) @@ -253,8 +277,10 @@ def _test_big_model_export(self, # prepare model self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir, total_steps=total_steps)) + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir, total_steps=total_steps + ) + ) test_utils.set_gpu_id(None) # the pipeline.config is produced by the prepare model cmd @@ -270,10 +296,13 @@ def _test_big_model_export(self, --redis_threads 1 --redis_write_kv 1 --verbose 1 - """ % (config_path, export_dir, test_data_path, os.environ['redis_url'], - os.environ['redis_passwd']) - proc = test_utils.run_cmd(export_cmd, - '%s/log_%s.txt' % (test_dir, 'export')) + """ % ( + config_path, export_dir, test_data_path, os.environ['redis_url'], + os.environ['redis_passwd'] + ) + proc = test_utils.run_cmd( + export_cmd, '%s/log_%s.txt' % (test_dir, 'export') + ) proc.wait() self.assertTrue(proc.returncode == 0) @@ -289,8 +318,9 @@ def _test_big_model_export(self, --input_path %s --output_path %s """ % (config_path, test_data_path, result_path) - proc = test_utils.run_cmd(predict_cmd % (), - '%s/log_%s.txt' % (test_dir, 'predict')) + proc = test_utils.run_cmd( + predict_cmd % (), '%s/log_%s.txt' % (test_dir, 'predict') + ) proc.wait() self.assertTrue(proc.returncode == 0) with open(result_path, 'r') as fin: @@ -306,97 +336,112 @@ def _test_big_model_export(self, self._predict_and_check(test_data_path, export_dir, cmp_result) @unittest.skipIf( - 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd') + 'redis_url' not in os.environ, + 'Only execute when redis is available: redis_url, redis_passwd' + ) def test_big_model_export(self): pipeline_config_path = 'samples/model_config/multi_tower_export.config' test_data_path = 'data/test/export/data.csv' self._test_big_model_export( - pipeline_config_path, - test_data_path, - extract_data_func=self._extract_data) + pipeline_config_path, + test_data_path, + extract_data_func=self._extract_data + ) @unittest.skipIf( - 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd') + 'redis_url' not in os.environ, + 'Only execute when redis is available: redis_url, redis_passwd' + ) def test_big_model_deepfm_export(self): pipeline_config_path = 'samples/model_config/deepfm_combo_on_avazu_ctr.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' self._test_big_model_export( - pipeline_config_path, - test_data_path, - extract_data_func=self._extract_data) + pipeline_config_path, + test_data_path, + extract_data_func=self._extract_data + ) @unittest.skipIf( - 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd') + 'redis_url' not in os.environ, + 'Only execute when redis is available: redis_url, redis_passwd' + ) def test_big_model_din_export(self): pipeline_config_path = 'samples/model_config/din_on_taobao.config' test_data_path = 'data/test/tb_data/taobao_test_data' self._test_big_model_export( - pipeline_config_path, - test_data_path, - extract_data_func=functools.partial(self._extract_data, offset=2)) + pipeline_config_path, + test_data_path, + extract_data_func=functools.partial(self._extract_data, offset=2) + ) @unittest.skipIf( - 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd') + 'redis_url' not in os.environ, + 'Only execute when redis is available: redis_url, redis_passwd' + ) def test_big_model_wide_and_deep_export(self): pipeline_config_path = 'samples/model_config/wide_and_deep_two_opti.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' self._test_big_model_export( - pipeline_config_path, - test_data_path, - extract_data_func=functools.partial(self._extract_data)) + pipeline_config_path, + test_data_path, + extract_data_func=functools.partial(self._extract_data) + ) @unittest.skipIf( - 'redis_url' not in os.environ or '-PAI' not in tf.__version__, - 'Only execute when pai-tf and redis is available: redis_url, redis_passwd' + 'redis_url' not in os.environ or '-PAI' not in tf.__version__, + 'Only execute when pai-tf and redis is available: redis_url, redis_passwd' ) def test_big_model_embedding_variable_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev.config' test_data_path = 'data/test/rtp/taobao_valid_feature.txt' self._test_big_model_export( - pipeline_config_path, - test_data_path, - self._extract_rtp_data, - total_steps=1000) + pipeline_config_path, + test_data_path, + self._extract_rtp_data, + total_steps=1000 + ) @unittest.skipIf( - 'oss_endpoint' not in os.environ or 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ or 'oss_path' not in os.environ or - '-PAI' not in tf.__version__, - 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' - 'and pai-tf is available.') + 'oss_endpoint' not in os.environ or 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ or 'oss_path' not in os.environ + or '-PAI' not in tf.__version__, + 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' + 'and pai-tf is available.' + ) def test_big_model_embedding_variable_oss_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev.config' test_data_path = 'data/test/rtp/taobao_valid_feature.txt' self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - self._extract_rtp_data, - total_steps=100) + pipeline_config_path, + test_data_path, + self._extract_rtp_data, + total_steps=100 + ) @unittest.skipIf( - 'oss_endpoint' not in os.environ or 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ or 'oss_path' not in os.environ or - '-PAI' not in tf.__version__, - 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' - 'and pai-tf is available.') + 'oss_endpoint' not in os.environ or 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ or 'oss_path' not in os.environ + or '-PAI' not in tf.__version__, + 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' + 'and pai-tf is available.' + ) def test_big_model_embedding_variable_v2_oss_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev_v2.config' test_data_path = 'data/test/rtp/taobao_valid_feature.txt' self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - self._extract_rtp_data, - total_steps=100) - - def _test_big_model_export_to_oss(self, - pipeline_config_path, - test_data_path, - extract_data_func=None, - total_steps=50): + pipeline_config_path, + test_data_path, + self._extract_rtp_data, + total_steps=100 + ) + + def _test_big_model_export_to_oss( + self, + pipeline_config_path, + test_data_path, + extract_data_func=None, + total_steps=50 + ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) @@ -405,8 +450,10 @@ def _test_big_model_export_to_oss(self, # prepare model self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir, total_steps=total_steps)) + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir, total_steps=total_steps + ) + ) test_utils.set_gpu_id(None) # the pipeline.config is produced by the prepare model cmd @@ -424,11 +471,13 @@ def _test_big_model_export_to_oss(self, --oss_timeout 10 --oss_write_kv 1 --verbose 1 - """ % (config_path, export_dir, test_data_path, os.environ['oss_path'], - os.environ['oss_endpoint'], os.environ['oss_ak'], - os.environ['oss_sk']) - proc = test_utils.run_cmd(export_cmd, - '%s/log_%s.txt' % (test_dir, 'export')) + """ % ( + config_path, export_dir, test_data_path, os.environ['oss_path'], + os.environ['oss_endpoint'], os.environ['oss_ak'], os.environ['oss_sk'] + ) + proc = test_utils.run_cmd( + export_cmd, '%s/log_%s.txt' % (test_dir, 'export') + ) proc.wait() self.assertTrue(proc.returncode == 0) @@ -444,8 +493,9 @@ def _test_big_model_export_to_oss(self, --input_path %s --output_path %s """ % (config_path, test_data_path, result_path) - proc = test_utils.run_cmd(predict_cmd, - '%s/log_%s.txt' % (test_dir, 'predict')) + proc = test_utils.run_cmd( + predict_cmd, '%s/log_%s.txt' % (test_dir, 'predict') + ) proc.wait() self.assertTrue(proc.returncode == 0) with open(result_path, 'r') as fin: @@ -461,52 +511,56 @@ def _test_big_model_export_to_oss(self, self._predict_and_check(test_data_path, export_dir, cmp_result) @unittest.skipIf( - 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'oss_path' not in os.environ, + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' ) def test_big_model_export_to_oss(self): pipeline_config_path = 'samples/model_config/multi_tower_export.config' test_data_path = 'data/test/export/data.csv' self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - extract_data_func=self._extract_data) + pipeline_config_path, + test_data_path, + extract_data_func=self._extract_data + ) @unittest.skipIf( - 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'oss_path' not in os.environ, + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' ) def test_big_model_deepfm_export_to_oss(self): pipeline_config_path = 'samples/model_config/deepfm_combo_on_avazu_ctr.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - extract_data_func=self._extract_data) + pipeline_config_path, + test_data_path, + extract_data_func=self._extract_data + ) @unittest.skipIf( - 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'oss_path' not in os.environ, + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' ) def test_big_model_din_export_to_oss(self): pipeline_config_path = 'samples/model_config/din_on_taobao.config' test_data_path = 'data/test/tb_data/taobao_test_data' self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - extract_data_func=functools.partial(self._extract_data, offset=2)) + pipeline_config_path, + test_data_path, + extract_data_func=functools.partial(self._extract_data, offset=2) + ) @unittest.skipIf( - 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'oss_path' not in os.environ, + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' ) def test_big_model_wide_and_deep_export_to_oss(self): pipeline_config_path = 'samples/model_config/wide_and_deep_two_opti.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - extract_data_func=functools.partial(self._extract_data)) + pipeline_config_path, + test_data_path, + extract_data_func=functools.partial(self._extract_data) + ) if __name__ == '__main__': diff --git a/easy_rec/python/test/fg_test.py b/easy_rec/python/test/fg_test.py index efbce46ea..65981500e 100644 --- a/easy_rec/python/test/fg_test.py +++ b/easy_rec/python/test/fg_test.py @@ -1,14 +1,11 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging -import unittest - import tensorflow as tf +import unittest from google.protobuf import text_format -from easy_rec.python.utils import config_util -from easy_rec.python.utils import fg_util -from easy_rec.python.utils import test_utils +from easy_rec.python.utils import config_util, fg_util, test_utils if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -36,33 +33,41 @@ def test_fg_json_to_config(self): fg_path = 'samples/rtp_fg/fg_test_extensions.json' pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path) + pipeline_config_path + ) pipeline_config.fg_json_path = fg_path fg_util.load_fg_json_to_config(pipeline_config) pipeline_config_str = text_format.MessageToString( - pipeline_config, as_utf8=True) + pipeline_config, as_utf8=True + ) final_pipeline_config = config_util.get_configs_from_pipeline_file( - final_pipeline_config_path) + final_pipeline_config_path + ) final_pipeline_config_str = text_format.MessageToString( - final_pipeline_config, as_utf8=True) + final_pipeline_config, as_utf8=True + ) self.assertEqual(pipeline_config_str, final_pipeline_config_str) def test_fg_dtype(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg_test_dtype.config', self._test_dir) + 'samples/model_config/taobao_fg_test_dtype.config', self._test_dir + ) self.assertTrue(self._success) def test_fg_train(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/fg_train.config', self._test_dir) + 'samples/model_config/fg_train.config', self._test_dir + ) self.assertTrue(self._success) - @unittest.skipIf('-PAI' not in tf.__version__, - 'Only test when pai-tf is used.') + @unittest.skipIf( + '-PAI' not in tf.__version__, 'Only test when pai-tf is used.' + ) def test_fg_train_ev(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/fg_train_ev.config', self._test_dir) + 'samples/model_config/fg_train_ev.config', self._test_dir + ) self.assertTrue(self._success) diff --git a/easy_rec/python/test/hive_input_test.py b/easy_rec/python/test/hive_input_test.py index 71aeafd4b..993b7fb98 100644 --- a/easy_rec/python/test/hive_input_test.py +++ b/easy_rec/python/test/hive_input_test.py @@ -3,9 +3,8 @@ """Define cv_input, the base class for cv tasks.""" import logging import os -import unittest - import tensorflow as tf +import unittest from google.protobuf import text_format from easy_rec.python.input.hive_input import HiveInput @@ -13,8 +12,7 @@ from easy_rec.python.protos.feature_config_pb2 import FeatureConfig from easy_rec.python.protos.hive_config_pb2 import HiveConfig from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig -from easy_rec.python.utils import config_util -from easy_rec.python.utils import test_utils +from easy_rec.python.utils import config_util, test_utils if tf.__version__ >= '2.0': import tensorflow.compat.v1 as tf @@ -59,12 +57,13 @@ def _init_config(self): def __init__(self, methodName='HiveInputTest'): super(HiveInputTest, self).__init__(methodName=methodName) - @unittest.skipIf('hive_host' not in os.environ or - 'hive_username' not in os.environ or - 'hive_table_name' not in os.environ or - 'hive_hash_fields' not in os.environ, - """Only execute hive_config var are specified,hive_host、 - hive_username、hive_table_name、hive_hash_fields is available.""") + @unittest.skipIf( + 'hive_host' not in os.environ or 'hive_username' not in os.environ + or 'hive_table_name' not in os.environ + or 'hive_hash_fields' not in os.environ, + """Only execute hive_config var are specified,hive_host、 + hive_username、hive_table_name、hive_hash_fields is available.""" + ) def test_hive_input(self): self._init_config() data_config_str = """ @@ -231,8 +230,9 @@ def test_hive_input(self): empty_config.input_names.pop() while len(empty_config.shared_names) > 0: empty_config.shared_names.pop() - train_input_fn = HiveInput(dataset_config, feature_configs, - self.hive_train_input_config).create_input() + train_input_fn = HiveInput( + dataset_config, feature_configs, self.hive_train_input_config + ).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -240,9 +240,10 @@ def test_hive_input(self): init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=False) + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=False + ) with self.test_session(config=session_config) as sess: sess.run(init_op) feature_dict, label_dict = sess.run([features, labels]) @@ -253,12 +254,13 @@ def test_hive_input(self): print(key, label_dict[key][:5]) return 0 - @unittest.skipIf('hive_host' not in os.environ or - 'hive_username' not in os.environ or - 'hive_table_name' not in os.environ or - 'hive_hash_fields' not in os.environ, - """Only execute hive_config var are specified,hive_host、 - hive_username、hive_table_name、hive_hash_fields is available.""") + @unittest.skipIf( + 'hive_host' not in os.environ or 'hive_username' not in os.environ + or 'hive_table_name' not in os.environ + or 'hive_hash_fields' not in os.environ, + """Only execute hive_config var are specified,hive_host、 + hive_username、hive_table_name、hive_hash_fields is available.""" + ) def test_mmoe(self): pipeline_config_path = 'samples/emr_script/mmoe/mmoe_census_income.config' gpus = test_utils.get_available_gpus() @@ -276,7 +278,8 @@ def test_mmoe(self): pipeline_config = pipeline_config_path else: pipeline_config = test_utils._load_config_for_test( - pipeline_config_path, self._test_dir) + pipeline_config_path, self._test_dir + ) pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -286,9 +289,11 @@ def test_mmoe(self): test_pipeline_config_path = os.path.join(self._test_dir, 'pipeline.config') hyperparam_str = '' train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path %s %s' % ( - test_pipeline_config_path, hyperparam_str) - proc = test_utils.run_cmd(train_cmd, - '%s/log_%s.txt' % (self._test_dir, 'master')) + test_pipeline_config_path, hyperparam_str + ) + proc = test_utils.run_cmd( + train_cmd, '%s/log_%s.txt' % (self._test_dir, 'master') + ) proc.wait() if proc.returncode != 0: logging.error('train %s failed' % test_pipeline_config_path) diff --git a/easy_rec/python/test/hpo_test.py b/easy_rec/python/test/hpo_test.py index a570d87d0..e2a2d52c4 100644 --- a/easy_rec/python/test/hpo_test.py +++ b/easy_rec/python/test/hpo_test.py @@ -3,16 +3,13 @@ import json import logging -import os -import time - import numpy as np +import os import tensorflow as tf +import time from easy_rec.python.protos.feature_config_pb2 import FeatureConfig -from easy_rec.python.utils import config_util -from easy_rec.python.utils import hpo_util -from easy_rec.python.utils import test_utils +from easy_rec.python.utils import config_util, hpo_util, test_utils if tf.__version__ >= '2.0': import tensorflow.io.gfile as gfile @@ -43,8 +40,9 @@ def load_config(self, config_path): def test_save_eval_metrics(self): test_dir = test_utils.get_tmp_dir() - tmp_file = os.path.join(test_dir, - 'easy_rec_hpo_test_%d.metric' % time.time()) + tmp_file = os.path.join( + test_dir, 'easy_rec_hpo_test_%d.metric' % time.time() + ) hpo_util.save_eval_metrics('data/test/hpo_test/', tmp_file, False) test_utils.clean_up(test_dir) @@ -52,14 +50,18 @@ def test_edit_config(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) assert tmp_config.feature_config.features[0].embedding_dim == 120 def test_edit_config_v2(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v2.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for tmp_fea in tmp_config.feature_configs: if tmp_fea.input_names[0] == 'site_id': assert tmp_fea.embedding_dim == 32 @@ -70,7 +72,9 @@ def test_edit_config_v3(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v3.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if i >= 10 and i < 20: assert tmp_fea.embedding_dim == 37 @@ -81,7 +85,9 @@ def test_edit_config_v4(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v4.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if i < 15: assert tmp_fea.embedding_dim == 37 @@ -92,7 +98,9 @@ def test_edit_config_v5(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v5.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if i >= 5: assert tmp_fea.embedding_dim == 37 @@ -103,7 +111,9 @@ def test_edit_config_v51(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v51.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if i == 5: assert tmp_fea.embedding_dim == 37 @@ -112,11 +122,14 @@ def test_edit_config_v6(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v6.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] >= 'site': assert tmp_fea.embedding_dim == 32, 'input_name = %s %d' % ( - tmp_fea.input_names[0], tmp_fea.embedding_dim) + tmp_fea.input_names[0], tmp_fea.embedding_dim + ) else: assert tmp_fea.embedding_dim == 16 @@ -124,38 +137,46 @@ def test_edit_config_v7(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v7.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - - 10.0) < 1e-5 + assert len(tmp_fea.boundaries + ) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 def test_edit_config_v71(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v71.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - - 10.0) < 1e-5 + assert len(tmp_fea.boundaries + ) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 def test_edit_config_v8(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v8.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - - 4.0) < 1e-5 + assert len(tmp_fea.boundaries + ) == 4 and np.abs(tmp_fea.boundaries[0] - 4.0) < 1e-5 assert tmp_fea.embedding_dim == 32 def test_edit_config_v81(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v81.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.feature_type == tmp_fea.RawFeature: assert tmp_fea.embedding_dim == 24 @@ -164,7 +185,9 @@ def test_edit_config_v9(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v9.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) assert tmp_config.train_config.fine_tune_checkpoint == \ 'oss://easy-rec/test/experiment/ctr_v93/model.ckpt-1000' @@ -172,28 +195,34 @@ def test_edit_config_v10(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v10.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - - 4.0) < 1e-5 + assert len(tmp_fea.boundaries + ) == 4 and np.abs(tmp_fea.boundaries[0] - 4.0) < 1e-5 assert tmp_fea.embedding_dim == 32 def test_edit_config_v11(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v11.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - - 10.0) < 1e-5 + assert len(tmp_fea.boundaries + ) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 def test_edit_config_v12(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v12.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': assert len(tmp_fea.boundaries) == 25 @@ -203,14 +232,18 @@ def test_edit_config_v13(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v13.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) assert not tmp_config.export_config.multi_placeholder def test_edit_config_v14(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v14.json' - tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + tmp_config = config_util.edit_config( + tmp_config, self.load_config(tmp_file) + ) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'hour': assert len(tmp_fea.feature_type) == FeatureConfig.RawFeature @@ -225,8 +258,9 @@ def test_save_eval_metrics_with_env(self): } """ test_dir = test_utils.get_tmp_dir() - tmp_file = os.path.join(test_dir, - 'easy_rec_hpo_test_%d.metric' % time.time()) + tmp_file = os.path.join( + test_dir, 'easy_rec_hpo_test_%d.metric' % time.time() + ) hpo_util.save_eval_metrics('data/test/hpo_test/', tmp_file, False) test_utils.clean_up(test_dir) diff --git a/easy_rec/python/test/kafka_test.py b/easy_rec/python/test/kafka_test.py index f0da2d5d5..00a5b00a9 100644 --- a/easy_rec/python/test/kafka_test.py +++ b/easy_rec/python/test/kafka_test.py @@ -3,26 +3,24 @@ import json import logging +import numpy as np import os +import six +import tensorflow as tf import threading import time import traceback import unittest - -import numpy as np -import six -import tensorflow as tf from tensorflow.python.data.ops import iterator_ops from tensorflow.python.platform import gfile from easy_rec.python.inference.predictor import Predictor from easy_rec.python.input.kafka_dataset import KafkaDataset -from easy_rec.python.utils import numpy_utils -from easy_rec.python.utils import test_utils +from easy_rec.python.utils import numpy_utils, test_utils try: import kafka - from kafka import KafkaProducer, KafkaAdminClient + from kafka import KafkaAdminClient, KafkaProducer from kafka.admin import NewTopic except ImportError: logging.warning('kafka-python is not installed: %s' % traceback.format_exc()) @@ -38,8 +36,10 @@ def setUp(self): self._zookeeper_proc = None return - logging.info('Testing %s.%s, test_dir=%s' % - (type(self).__name__, self._testMethodName, self._test_dir)) + logging.info( + 'Testing %s.%s, test_dir=%s' % + (type(self).__name__, self._testMethodName, self._test_dir) + ) self._log_dir = os.path.join(self._test_dir, 'logs') if not gfile.IsDirectory(self._log_dir): gfile.MakeDirs(self._log_dir) @@ -59,8 +59,9 @@ def setUp(self): fout.write('dataDir=%s/zookeeper\n' % self._test_dir) else: fout.write(line_str) - cmd = 'bash %s/bin/zookeeper-server-start.sh %s' % (kafka_install_dir, - zookeeper_config) + cmd = 'bash %s/bin/zookeeper-server-start.sh %s' % ( + kafka_install_dir, zookeeper_config + ) log_file = os.path.join(self._log_dir, 'zookeeper.log') self._zookeeper_proc = test_utils.run_cmd(cmd, log_file) @@ -73,8 +74,9 @@ def setUp(self): fout.write('log.dirs=%s/kafka\n' % self._test_dir) else: fout.write(line_str) - cmd = 'bash %s/bin/kafka-server-start.sh %s' % (kafka_install_dir, - kafka_config) + cmd = 'bash %s/bin/kafka-server-start.sh %s' % ( + kafka_install_dir, kafka_config + ) log_file = os.path.join(self._log_dir, 'kafka_server.log') self._kafka_server_proc = test_utils.run_cmd(cmd, log_file) @@ -89,7 +91,9 @@ def setUp(self): else: try: admin_clt = KafkaAdminClient(bootstrap_servers=self._kafka_servers) - logging.info('old topics: %s' % (','.join(admin_clt.list_topics()))) + logging.info( + 'old topics: %s' % (','.join(admin_clt.list_topics())) + ) admin_clt.close() started = True except kafka.errors.NoBrokersAvailable: @@ -106,10 +110,11 @@ def _create_topic(self, num_partitions=2): logging.info('create topic: %s' % self._test_topic) topic_list = [ - NewTopic( - name=self._test_topic, - num_partitions=num_partitions, - replication_factor=1) + NewTopic( + name=self._test_topic, + num_partitions=num_partitions, + replication_factor=1 + ) ] admin_clt.create_topics(new_topics=topic_list, validate_only=False) @@ -146,15 +151,18 @@ def tearDown(self): if self._success: test_utils.clean_up(self._test_dir) - @unittest.skipIf('kafka_install_dir' not in os.environ, - 'Only execute when kafka is available') + @unittest.skipIf( + 'kafka_install_dir' not in os.environ, + 'Only execute when kafka is available' + ) def test_kafka_ops(self): try: test_utils.set_gpu_id(None) def _generate(): producer = KafkaProducer( - bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1)) + bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1) + ) i = 0 while not self._should_stop: msg = 'user_id_%d' % i @@ -165,19 +173,21 @@ def _generate(): group = 'dataset_consumer' k = KafkaDataset( - servers=self._kafka_servers[0], - topics=[self._test_topic + ':0', self._test_topic + ':1'], - group=group, - eof=True, - # control the maximal read of each partition - config_global=['max.partition.fetch.bytes=1048576'], - message_key=True, - message_offset=True) + servers=self._kafka_servers[0], + topics=[self._test_topic + ':0', self._test_topic + ':1'], + group=group, + eof=True, + # control the maximal read of each partition + config_global=['max.partition.fetch.bytes=1048576'], + message_key=True, + message_offset=True + ) batch_dataset = k.batch(5) iterator = iterator_ops.Iterator.from_structure( - batch_dataset.output_types) + batch_dataset.output_types + ) init_batch_op = iterator.make_initializer(batch_dataset) get_next = iterator.get_next() @@ -206,8 +216,10 @@ def _generate(): self._success = False raise ex - @unittest.skipIf('kafka_install_dir' not in os.environ, - 'Only execute when kafka is available') + @unittest.skipIf( + 'kafka_install_dir' not in os.environ, + 'Only execute when kafka is available' + ) def test_kafka_train(self): try: # start produce thread @@ -216,8 +228,8 @@ def test_kafka_train(self): test_utils.set_gpu_id(None) self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_avazu_kafka.config', - self._test_dir) + 'samples/model_config/deepfm_combo_avazu_kafka.config', self._test_dir + ) self.assertTrue(self._success) except Exception as ex: self._success = False @@ -225,7 +237,8 @@ def test_kafka_train(self): def _generate(self): producer = KafkaProducer( - bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1)) + bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1) + ) while not self._should_stop: with open('data/test/dwd_avazu_ctr_deepmodel_10w.csv', 'r') as fin: for line_str in fin: @@ -238,8 +251,10 @@ def _generate(self): producer.close() logging.info('data generation thread done.') - @unittest.skipIf('kafka_install_dir' not in os.environ, - 'Only execute when kafka is available') + @unittest.skipIf( + 'kafka_install_dir' not in os.environ, + 'Only execute when kafka is available' + ) def test_kafka_train_chief_redundant(self): try: # start produce thread @@ -248,16 +263,19 @@ def test_kafka_train_chief_redundant(self): test_utils.set_gpu_id(None) self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_combo_avazu_kafka_chief_redundant.config', - self._test_dir, - num_evaluator=1) + 'samples/model_config/deepfm_combo_avazu_kafka_chief_redundant.config', + self._test_dir, + num_evaluator=1 + ) self.assertTrue(self._success) except Exception as ex: self._success = False raise ex - @unittest.skipIf('kafka_install_dir' not in os.environ, - 'Only execute when kafka is available') + @unittest.skipIf( + 'kafka_install_dir' not in os.environ, + 'Only execute when kafka is available' + ) def test_kafka_train_v2(self): try: # start produce thread @@ -266,8 +284,9 @@ def test_kafka_train_v2(self): test_utils.set_gpu_id(None) self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_avazu_kafka_time_offset.config', - self._test_dir) + 'samples/model_config/deepfm_combo_avazu_kafka_time_offset.config', + self._test_dir + ) self.assertTrue(self._success) except Exception as ex: @@ -275,36 +294,44 @@ def test_kafka_train_v2(self): raise ex @unittest.skipIf( - 'kafka_install_dir' not in os.environ or 'oss_path' not in os.environ or - 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ, 'Only execute when kafka is available') + 'kafka_install_dir' not in os.environ or 'oss_path' not in os.environ + or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ, 'Only execute when kafka is available' + ) def test_kafka_processor(self): self._test_kafka_processor( - 'samples/model_config/taobao_fg_incr_save.config') + 'samples/model_config/taobao_fg_incr_save.config' + ) @unittest.skipIf( - 'kafka_install_dir' not in os.environ or 'oss_path' not in os.environ or - 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ, 'Only execute when kafka is available') + 'kafka_install_dir' not in os.environ or 'oss_path' not in os.environ + or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ, 'Only execute when kafka is available' + ) def test_kafka_processor_ev(self): self._test_kafka_processor( - 'samples/model_config/taobao_fg_incr_save_ev.config') + 'samples/model_config/taobao_fg_incr_save_ev.config' + ) def _test_kafka_processor(self, config_path): self._success = False success = test_utils.test_distributed_train_eval( - config_path, self._test_dir, total_steps=500) + config_path, self._test_dir, total_steps=500 + ) self.assertTrue(success) export_cmd = """ python -m easy_rec.python.export --pipeline_config_path %s/pipeline.config --export_dir %s/export/sep/ --oss_path=%s --oss_ak=%s --oss_sk=%s --oss_endpoint=%s --asset_files ./samples/rtp_fg/fg.json --checkpoint_path %s/train/model.ckpt-0 - """ % (self._test_dir, self._test_dir, os.environ['oss_path'], - os.environ['oss_ak'], os.environ['oss_sk'], - os.environ['oss_endpoint'], self._test_dir) - proc = test_utils.run_cmd(export_cmd, - '%s/log_export_sep.txt' % self._test_dir) + """ % ( + self._test_dir, self._test_dir, os.environ['oss_path'], + os.environ['oss_ak'], os.environ['oss_sk'], os.environ['oss_endpoint'], + self._test_dir + ) + proc = test_utils.run_cmd( + export_cmd, '%s/log_export_sep.txt' % self._test_dir + ) proc.wait() self.assertTrue(proc.returncode == 0) files = gfile.Glob(os.path.join(self._test_dir, 'export/sep/[1-9][0-9]*')) @@ -318,7 +345,8 @@ def _test_kafka_processor(self, config_path): envs = dict(os.environ) envs['PROCESSOR_TEST'] = '1' proc = test_utils.run_cmd( - predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs) + predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs + ) proc.wait() self.assertTrue(proc.returncode == 0) @@ -341,7 +369,8 @@ def _test_kafka_processor(self, config_path): with open('%s/predictor.out' % self._test_dir, 'w') as fout: for i in range(len(output_res)): fout.write( - json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n') + json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n' + ) for i in range(len(output_res)): val0 = output_res[i]['probs'] @@ -350,8 +379,10 @@ def _test_kafka_processor(self, config_path): assert diff < 1e-4, 'too much difference[%.6f] >= 1e-4' % diff self._success = True - @unittest.skipIf('kafka_install_dir' not in os.environ, - 'Only execute when kafka is available') + @unittest.skipIf( + 'kafka_install_dir' not in os.environ, + 'Only execute when kafka is available' + ) def test_kafka_train_v3(self): try: # start produce thread @@ -360,8 +391,9 @@ def test_kafka_train_v3(self): test_utils.set_gpu_id(None) self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_avazu_kafka_time_offset2.config', - self._test_dir) + 'samples/model_config/deepfm_combo_avazu_kafka_time_offset2.config', + self._test_dir + ) self.assertTrue(self._success) except Exception as ex: diff --git a/easy_rec/python/test/local_incr_test.py b/easy_rec/python/test/local_incr_test.py index ad2d657f3..c505e1cde 100644 --- a/easy_rec/python/test/local_incr_test.py +++ b/easy_rec/python/test/local_incr_test.py @@ -3,16 +3,14 @@ import json import logging -import os -import unittest - import numpy as np +import os import tensorflow as tf +import unittest from tensorflow.python.platform import gfile from easy_rec.python.inference.predictor import Predictor -from easy_rec.python.utils import numpy_utils -from easy_rec.python.utils import test_utils +from easy_rec.python.utils import numpy_utils, test_utils class LocalIncrTest(tf.test.TestCase): @@ -21,57 +19,69 @@ def setUp(self): self._success = True self._test_dir = test_utils.get_tmp_dir() - logging.info('Testing %s.%s, test_dir=%s' % - (type(self).__name__, self._testMethodName, self._test_dir)) + logging.info( + 'Testing %s.%s, test_dir=%s' % + (type(self).__name__, self._testMethodName, self._test_dir) + ) self._log_dir = os.path.join(self._test_dir, 'logs') if not gfile.IsDirectory(self._log_dir): gfile.MakeDirs(self._log_dir) @unittest.skipIf( - 'oss_path' not in os.environ or - 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ, 'Only execute when kafka is available') + 'oss_path' not in os.environ + or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ, 'Only execute when kafka is available' + ) def test_incr_save(self): self._test_incr_save( - 'samples/model_config/taobao_fg_incr_save_local.config') + 'samples/model_config/taobao_fg_incr_save_local.config' + ) @unittest.skipIf( - 'oss_path' not in os.environ or - 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ, 'Only execute when kafka is available') + 'oss_path' not in os.environ + or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ, 'Only execute when kafka is available' + ) def test_incr_save_ev(self): self._test_incr_save( - 'samples/model_config/taobao_fg_incr_save_ev_local.config') + 'samples/model_config/taobao_fg_incr_save_ev_local.config' + ) @unittest.skipIf( - 'oss_path' not in os.environ or - 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ, 'Only execute when kafka is available') + 'oss_path' not in os.environ + or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ, 'Only execute when kafka is available' + ) def test_incr_save_share_ev(self): self._test_incr_save( - 'samples/model_config/taobao_fg_incr_save_share_ev_local.config') + 'samples/model_config/taobao_fg_incr_save_share_ev_local.config' + ) def _test_incr_save(self, config_path): self._success = False success = test_utils.test_distributed_train_eval( - config_path, - self._test_dir, - total_steps=100, - edit_config_json={ - 'train_config.incr_save_config.fs.mount_path': - os.path.join(self._test_dir, 'train/incr_save/') - }) + config_path, + self._test_dir, + total_steps=100, + edit_config_json={ + 'train_config.incr_save_config.fs.mount_path': + os.path.join(self._test_dir, 'train/incr_save/') + } + ) self.assertTrue(success) export_cmd = """ python -m easy_rec.python.export --pipeline_config_path %s/pipeline.config --export_dir %s/export/sep/ --oss_path=%s --oss_ak=%s --oss_sk=%s --oss_endpoint=%s --asset_files ./samples/rtp_fg/fg.json --checkpoint_path %s/train/model.ckpt-0 - """ % (self._test_dir, self._test_dir, os.environ['oss_path'], - os.environ['oss_ak'], os.environ['oss_sk'], - os.environ['oss_endpoint'], self._test_dir) - proc = test_utils.run_cmd(export_cmd, - '%s/log_export_sep.txt' % self._test_dir) + """ % ( + self._test_dir, self._test_dir, os.environ['oss_path'], + os.environ['oss_ak'], os.environ['oss_sk'], os.environ['oss_endpoint'], + self._test_dir + ) + proc = test_utils.run_cmd( + export_cmd, '%s/log_export_sep.txt' % self._test_dir + ) proc.wait() self.assertTrue(proc.returncode == 0) files = gfile.Glob(os.path.join(self._test_dir, 'export/sep/[1-9][0-9]*')) @@ -85,7 +95,8 @@ def _test_incr_save(self, config_path): envs = dict(os.environ) envs['PROCESSOR_TEST'] = '1' proc = test_utils.run_cmd( - predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs) + predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs + ) proc.wait() self.assertTrue(proc.returncode == 0) @@ -108,7 +119,8 @@ def _test_incr_save(self, config_path): with open('%s/predictor.out' % self._test_dir, 'w') as fout: for i in range(len(output_res)): fout.write( - json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n') + json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n' + ) for i in range(len(output_res)): val0 = output_res[i]['probs'] diff --git a/easy_rec/python/test/loss_test.py b/easy_rec/python/test/loss_test.py index f78b74ce6..798538c57 100644 --- a/easy_rec/python/test/loss_test.py +++ b/easy_rec/python/test/loss_test.py @@ -1,11 +1,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.loss.circle_loss import circle_loss -from easy_rec.python.loss.circle_loss import get_anchor_positive_triplet_mask - +from easy_rec.python.loss.circle_loss import circle_loss, get_anchor_positive_triplet_mask # NOQA from easy_rec.python.loss.f1_reweight_loss import f1_reweight_sigmoid_cross_entropy # NOQA - from easy_rec.python.loss.softmax_loss_with_negative_mining import softmax_loss_with_negative_mining # NOQA if tf.__version__ >= '2.0': @@ -19,33 +16,44 @@ def test_f1_reweighted_loss(self): logits = tf.constant([0.1, 0.5, 0.3, 0.8, -0.1, 0.3]) labels = tf.constant([1, 1, 0, 0, 1, 1]) loss = f1_reweight_sigmoid_cross_entropy( - labels=labels, logits=logits, beta_square=4) + labels=labels, logits=logits, beta_square=4 + ) with self.test_session() as sess: loss_val = sess.run(loss) self.assertAlmostEqual(loss_val, 0.47844395, delta=1e-5) def test_softmax_loss_with_negative_mining(self): print('test_softmax_loss_with_negative_mining') - user_emb = tf.constant([[0.1, 0.5, 0.3], [0.8, -0.1, 0.3], [0.28, 0.3, 0.9], - [0.37, 0.45, 0.93], [-0.7, 0.15, 0.03], - [0.18, 0.9, -0.3]]) - item_emb = tf.constant([[0.1, -0.5, 0.3], [0.8, -0.31, 0.3], - [0.7, -0.45, 0.15], [0.08, -0.31, -0.9], - [-0.7, 0.85, 0.03], [0.18, 0.89, -0.3]]) + user_emb = tf.constant( + [ + [0.1, 0.5, 0.3], [0.8, -0.1, 0.3], [0.28, 0.3, 0.9], + [0.37, 0.45, 0.93], [-0.7, 0.15, 0.03], [0.18, 0.9, -0.3] + ] + ) + item_emb = tf.constant( + [ + [0.1, -0.5, 0.3], [0.8, -0.31, 0.3], [0.7, -0.45, 0.15], + [0.08, -0.31, -0.9], [-0.7, 0.85, 0.03], [0.18, 0.89, -0.3] + ] + ) label = tf.constant([1, 1, 0, 0, 1, 1]) loss = softmax_loss_with_negative_mining( - user_emb, item_emb, label, num_negative_samples=2, seed=1) + user_emb, item_emb, label, num_negative_samples=2, seed=1 + ) with self.test_session() as sess: loss_val = sess.run(loss) self.assertAlmostEqual(loss_val, 0.48577175, delta=1e-5) def test_circle_loss(self): print('test_circle_loss') - emb = tf.constant([[0.1, 0.2, 0.15, 0.1], [0.3, 0.6, 0.45, 0.3], - [0.13, 0.6, 0.45, 0.3], [0.3, 0.26, 0.45, 0.3], - [0.3, 0.6, 0.5, 0.13], [0.08, 0.43, 0.21, 0.6]], - dtype=tf.float32) + emb = tf.constant( + [ + [0.1, 0.2, 0.15, 0.1], [0.3, 0.6, 0.45, 0.3], [0.13, 0.6, 0.45, 0.3], + [0.3, 0.26, 0.45, 0.3], [0.3, 0.6, 0.5, 0.13], [0.08, 0.43, 0.21, 0.6] + ], + dtype=tf.float32 + ) label = tf.constant([1, 1, 2, 2, 3, 3]) loss = circle_loss(emb, label, label, margin=0.25, gamma=64) with self.test_session() as sess: @@ -56,17 +64,23 @@ def test_triplet_mask(self): print('test_triplet_mask') label = tf.constant([1, 1, 2, 2, 3, 3, 4, 5]) positive_mask = tf.constant( - [[0., 1., 0., 0., 0., 0., 0., 0.], [1., 0., 0., 0., 0., 0., 0., 0.], - [0., 0., 0., 1., 0., 0., 0., 0.], [0., 0., 1., 0., 0., 0., 0., 0.], - [0., 0., 0., 0., 0., 1., 0., 0.], [0., 0., 0., 0., 1., 0., 0., 0.], - [0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0.]], - dtype=tf.float32) + [ + [0., 1., 0., 0., 0., 0., 0., 0.], [1., 0., 0., 0., 0., 0., 0., 0.], + [0., 0., 0., 1., 0., 0., 0., 0.], [0., 0., 1., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 1., 0., 0.], [0., 0., 0., 0., 1., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0.] + ], + dtype=tf.float32 + ) negative_mask = tf.constant( - [[0., 0., 1., 1., 1., 1., 1., 1.], [0., 0., 1., 1., 1., 1., 1., 1.], - [1., 1., 0., 0., 1., 1., 1., 1.], [1., 1., 0., 0., 1., 1., 1., 1.], - [1., 1., 1., 1., 0., 0., 1., 1.], [1., 1., 1., 1., 0., 0., 1., 1.], - [1., 1., 1., 1., 1., 1., 0., 1.], [1., 1., 1., 1., 1., 1., 1., 0.]], - dtype=tf.float32) + [ + [0., 0., 1., 1., 1., 1., 1., 1.], [0., 0., 1., 1., 1., 1., 1., 1.], + [1., 1., 0., 0., 1., 1., 1., 1.], [1., 1., 0., 0., 1., 1., 1., 1.], + [1., 1., 1., 1., 0., 0., 1., 1.], [1., 1., 1., 1., 0., 0., 1., 1.], + [1., 1., 1., 1., 1., 1., 0., 1.], [1., 1., 1., 1., 1., 1., 1., 0.] + ], + dtype=tf.float32 + ) with self.test_session(): pos_mask = get_anchor_positive_triplet_mask(label, label) self.assertAllEqual(positive_mask, pos_mask) @@ -92,7 +106,8 @@ def _get_anchor_negative_triplet_mask(labels, sessions): # Check if sessions[i] != sessions[k] # Uses broadcasting where the 1st argument has shape (1, batch_size) and the 2nd (batch_size, 1) session_not_equal = tf.not_equal( - tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1)) + tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1) + ) if labels is sessions: return tf.cast(session_not_equal, tf.float32) @@ -100,7 +115,8 @@ def _get_anchor_negative_triplet_mask(labels, sessions): # Check if labels[i] != labels[k] # Uses broadcasting where the 1st argument has shape (1, batch_size) and the 2nd (batch_size, 1) label_not_equal = tf.not_equal( - tf.expand_dims(labels, 0), tf.expand_dims(labels, 1)) + tf.expand_dims(labels, 0), tf.expand_dims(labels, 1) + ) mask = tf.logical_or(session_not_equal, label_not_equal) return tf.cast(mask, tf.float32) diff --git a/easy_rec/python/test/odps_command.py b/easy_rec/python/test/odps_command.py index 33298f727..123944c0b 100644 --- a/easy_rec/python/test/odps_command.py +++ b/easy_rec/python/test/odps_command.py @@ -16,10 +16,10 @@ def __init__(self, odps_oss_config): Args: odps_oss_config: instance of easy_rec.python.utils.odps_test_util.OdpsOSSConfig """ - self.bucket = get_oss_bucket(odps_oss_config.oss_key, - odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name) + self.bucket = get_oss_bucket( + odps_oss_config.oss_key, odps_oss_config.oss_secret, + odps_oss_config.endpoint, odps_oss_config.bucket_name + ) self.bucket_name = odps_oss_config.bucket_name self.temp_dir = odps_oss_config.temp_dir self.log_path = odps_oss_config.log_dir @@ -42,19 +42,23 @@ def run_odps_cmd(self, script_file): log_file = os.path.join(self.log_path, file_name) if self.odps_config_path is None: - cmd = 'nohup %s -f %s > %s.log 2>&1' % (self.odpscmd, exec_file_path, - log_file) + cmd = 'nohup %s -f %s > %s.log 2>&1' % ( + self.odpscmd, exec_file_path, log_file + ) else: cmd = 'nohup %s --config=%s -f %s > %s.log 2>&1' % ( - self.odpscmd, self.odps_config_path, exec_file_path, log_file) + self.odpscmd, self.odps_config_path, exec_file_path, log_file + ) logging.info('will run cmd: %s' % (cmd)) proc = subprocess.Popen(cmd, shell=True) proc.wait() if (proc.returncode == 0): logging.info('%s run succeed' % script_file) else: - raise ValueError('%s run FAILED: please check log file:%s.log' % - (exec_file_path, log_file)) + raise ValueError( + '%s run FAILED: please check log file:%s.log' % + (exec_file_path, log_file) + ) def run_list(self, files): for f in files: diff --git a/easy_rec/python/test/odps_local_run.py b/easy_rec/python/test/odps_local_run.py index 8c1d15274..1d8b9befc 100644 --- a/easy_rec/python/test/odps_local_run.py +++ b/easy_rec/python/test/odps_local_run.py @@ -6,18 +6,16 @@ import os import shutil import sys - import tensorflow as tf from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig -from easy_rec.python.test.odps_test_util import delete_oss_path -from easy_rec.python.test.odps_test_util import get_oss_bucket +from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA from easy_rec.python.utils import test_utils logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) odps_oss_config = OdpsOSSConfig(script_path='./samples/emr_script') @@ -36,14 +34,15 @@ def tearDown(self): def test_deepfm_local_with_common_io(self): start = [ - 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql' + 'deep_fm/create_external_deepfm_table.sql', + 'deep_fm/create_inner_deepfm_table.sql' ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) odps_cmd.run_list(start) self._success = test_utils.test_single_train_eval( - '%s/configs/deepfm.config' % odps_oss_config.temp_dir, self._test_dir) + '%s/configs/deepfm.config' % odps_oss_config.temp_dir, self._test_dir + ) odps_cmd.run_list(end) self.assertTrue(self._success) @@ -51,14 +50,18 @@ def test_deepfm_local_with_common_io(self): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--odps_config', type=str, default=None, help='odps config path') + '--odps_config', type=str, default=None, help='odps config path' + ) parser.add_argument( - '--oss_config', type=str, default=None, help='ossutilconfig path') + '--oss_config', type=str, default=None, help='ossutilconfig path' + ) parser.add_argument( - '--bucket_name', type=str, default=None, help='test oss bucket name') + '--bucket_name', type=str, default=None, help='test oss bucket name' + ) parser.add_argument('--arn', type=str, default=None, help='oss rolearn') parser.add_argument( - '--odpscmd', type=str, default='odpscmd', help='odpscmd path') + '--odpscmd', type=str, default='odpscmd', help='odpscmd path' + ) args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] for unk_arg in unknown_args: @@ -79,8 +82,10 @@ def test_deepfm_local_with_common_io(self): prepare(odps_oss_config) tf.test.main() # delete oss path - bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name) + bucket = get_oss_bucket( + odps_oss_config.oss_key, odps_oss_config.oss_secret, + odps_oss_config.endpoint, odps_oss_config.bucket_name + ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp shutil.rmtree(odps_oss_config.temp_dir) diff --git a/easy_rec/python/test/odps_run.py b/easy_rec/python/test/odps_run.py index 1eb77c926..96b64d9dd 100644 --- a/easy_rec/python/test/odps_run.py +++ b/easy_rec/python/test/odps_run.py @@ -4,21 +4,19 @@ import argparse import logging import os +import oss2 import shutil import sys - -import oss2 import tensorflow as tf from easy_rec.python.test.odps_test_cls import OdpsTest from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig -from easy_rec.python.test.odps_test_util import delete_oss_path -from easy_rec.python.test.odps_test_util import get_oss_bucket +from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA from easy_rec.python.utils import config_util logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) odps_oss_config = OdpsOSSConfig() @@ -29,9 +27,9 @@ class TestPipelineOnOdps(tf.test.TestCase): def test_deepfm(self): start_files = ['deep_fm/create_inner_deepfm_table.sql'] test_files = [ - 'deep_fm/train_deepfm_model.sql', 'deep_fm/eval_deepfm.sql', - 'deep_fm/export_deepfm.sql', 'deep_fm/predict_deepfm.sql', - 'deep_fm/export_rtp_ckpt.sql' + 'deep_fm/train_deepfm_model.sql', 'deep_fm/eval_deepfm.sql', + 'deep_fm/export_deepfm.sql', 'deep_fm/predict_deepfm.sql', + 'deep_fm/export_rtp_ckpt.sql' ] end_file = ['deep_fm/drop_table.sql'] @@ -42,10 +40,10 @@ def test_deepfm(self): def test_mmoe(self): start_files = ['mmoe/create_inner_mmoe_table.sql'] test_files = [ - 'mmoe/train_mmoe_model.sql', - 'mmoe/eval_mmoe.sql', - 'mmoe/export_mmoe.sql', - 'mmoe/predict_mmoe.sql', + 'mmoe/train_mmoe_model.sql', + 'mmoe/eval_mmoe.sql', + 'mmoe/export_mmoe.sql', + 'mmoe/predict_mmoe.sql', ] end_file = ['mmoe/drop_mmoe_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -54,16 +52,16 @@ def test_mmoe(self): def test_dssm(self): start_files = [ - 'dssm/create_inner_dssm_table.sql', + 'dssm/create_inner_dssm_table.sql', ] test_files = [ - 'dssm/train_dssm_model.sql', - 'dssm/eval_dssm.sql', - 'dssm/export_dssm.sql', - 'dssm/predict_dssm.sql', + 'dssm/train_dssm_model.sql', + 'dssm/eval_dssm.sql', + 'dssm/export_dssm.sql', + 'dssm/predict_dssm.sql', ] end_file = [ - 'dssm/drop_dssm_table.sql', + 'dssm/drop_dssm_table.sql', ] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) tot.start_test() @@ -72,12 +70,12 @@ def test_dssm(self): def test_multi_tower(self): start_files = ['multi_tower/create_inner_multi_tower_table.sql'] test_files = [ - 'multi_tower/train_multil_tower_din_model.sql', - 'multi_tower/train_multil_tower_bst_model.sql', - 'multi_tower/eval_multil_tower.sql', - 'multi_tower/export_multil_tower.sql', - 'multi_tower/export_again_multi_tower.sql', - 'multi_tower/predict_multil_tower.sql', + 'multi_tower/train_multil_tower_din_model.sql', + 'multi_tower/train_multil_tower_bst_model.sql', + 'multi_tower/eval_multil_tower.sql', + 'multi_tower/export_multil_tower.sql', + 'multi_tower/export_again_multi_tower.sql', + 'multi_tower/predict_multil_tower.sql', ] end_file = ['multi_tower/drop_multil_tower_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -87,16 +85,16 @@ def test_multi_tower(self): def test_other(self): start_files = ['deep_fm/create_inner_deepfm_table.sql'] test_files = [ - # 'other_test/test_train_gpuRequired_mirrored', # 线上报错， - # 'other_test/test_train_distribute_strategy_collective', # 线上报错， - 'other_test/test_train_hpo_with_evaluator.sql', - # 'other_test/test_train_version.sql', - # 'other_test/test_train_distribute_strategy_ess.sql', - 'other_test/test_train_before_export.sql', - 'other_test/test_eval_checkpoint_path.sql', - 'other_test/test_export_checkpoint_path.sql', - 'other_test/test_export_update_model_dir.sql', - 'other_test/test_predict_selected_cols.sql', + # 'other_test/test_train_gpuRequired_mirrored', # 线上报错， + # 'other_test/test_train_distribute_strategy_collective', # 线上报错， + 'other_test/test_train_hpo_with_evaluator.sql', + # 'other_test/test_train_version.sql', + # 'other_test/test_train_distribute_strategy_ess.sql', + 'other_test/test_train_before_export.sql', + 'other_test/test_eval_checkpoint_path.sql', + 'other_test/test_export_checkpoint_path.sql', + 'other_test/test_export_update_model_dir.sql', + 'other_test/test_predict_selected_cols.sql', ] end_file = ['other_test/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -106,14 +104,15 @@ def test_other(self): def test_best_exporter(self): start_files = ['deep_fm/create_inner_deepfm_table.sql'] test_files = [ - 'other_test/test_train_best_export.sql', + 'other_test/test_train_best_export.sql', ] end_file = ['other_test/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) tot.start_test() config_path = os.path.join( - odps_oss_config.temp_dir, - 'configs/dwd_avazu_ctr_deepmodel_ext_best_export.config') + odps_oss_config.temp_dir, + 'configs/dwd_avazu_ctr_deepmodel_ext_best_export.config' + ) config = config_util.get_configs_from_pipeline_file(config_path) model_dir = config.model_dir logging.info('raw model_dir = %s' % model_dir) @@ -122,35 +121,34 @@ def test_best_exporter(self): model_dir = model_dir[spos:] logging.info('stripped model_dir = %s' % model_dir) - bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name) + bucket = get_oss_bucket( + odps_oss_config.oss_key, odps_oss_config.oss_secret, + odps_oss_config.endpoint, odps_oss_config.bucket_name + ) best_ckpt_prefix = os.path.join(model_dir, 'best_ckpt/model.ckpt') best_ckpts = [ - x.key - for x in oss2.ObjectIterator(bucket, prefix=best_ckpt_prefix) - if x.key.endswith('.meta') + x.key for x in oss2.ObjectIterator(bucket, prefix=best_ckpt_prefix) + if x.key.endswith('.meta') ] logging.info('best ckpts: %s' % str(best_ckpts)) assert len(best_ckpts) <= 2, 'too many best ckpts: %s' % str(best_ckpts) best_export_prefix = os.path.join(model_dir, 'export/best/') best_exports = [ - x.key - for x in oss2.ObjectIterator(bucket, prefix=best_export_prefix) - if x.key.endswith('/saved_model.pb') + x.key for x in oss2.ObjectIterator(bucket, prefix=best_export_prefix) + if x.key.endswith('/saved_model.pb') ] logging.info('best exports: %s' % str(best_exports)) - assert len( - best_exports) <= 2, 'too many best exports: %s' % str(best_exports) + assert len(best_exports + ) <= 2, 'too many best exports: %s' % str(best_exports) return True def test_embedding_variable(self): start_files = [ - 'embedding_variable/create_table.sql', + 'embedding_variable/create_table.sql', ] test_files = [ - 'embedding_variable/train.sql', 'embedding_variable/train_work_que.sql', - 'embedding_variable/export.sql' + 'embedding_variable/train.sql', 'embedding_variable/train_work_que.sql', + 'embedding_variable/export.sql' ] end_file = ['embedding_variable/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -167,12 +165,12 @@ def test_multi_value_export(self): def test_boundary_test(self): start_files = [ - 'boundary/create_inner_boundary_table.sql', + 'boundary/create_inner_boundary_table.sql', ] test_files = [ - 'boundary/train_multi_tower_model.sql', - 'boundary/finetune_multi_tower_model.sql', - 'boundary/finetune_multi_tower_conti.sql', 'boundary/train_compat.sql' + 'boundary/train_multi_tower_model.sql', + 'boundary/finetune_multi_tower_model.sql', + 'boundary/finetune_multi_tower_conti.sql', 'boundary/train_compat.sql' ] end_file = ['boundary/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -191,33 +189,42 @@ def test_vector_retrieve(self): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--odps_config', type=str, default=None, help='odps config path') + '--odps_config', type=str, default=None, help='odps config path' + ) parser.add_argument( - '--oss_config', type=str, default=None, help='ossutilconfig path') + '--oss_config', type=str, default=None, help='ossutilconfig path' + ) parser.add_argument( - '--bucket_name', type=str, default=None, help='test oss bucket name') + '--bucket_name', type=str, default=None, help='test oss bucket name' + ) parser.add_argument('--arn', type=str, default=None, help='oss rolearn') parser.add_argument( - '--odpscmd', type=str, default='odpscmd', help='odpscmd path') + '--odpscmd', type=str, default='odpscmd', help='odpscmd path' + ) parser.add_argument( - '--algo_name', - type=str, - default='easy_rec_ext', - help='whether use pai-tf 1.15') + '--algo_name', + type=str, + default='easy_rec_ext', + help='whether use pai-tf 1.15' + ) parser.add_argument( - '--algo_project', type=str, default=None, help='algo project name') + '--algo_project', type=str, default=None, help='algo project name' + ) parser.add_argument( - '--algo_res_project', - type=str, - default=None, - help='algo resource project name') + '--algo_res_project', + type=str, + default=None, + help='algo resource project name' + ) parser.add_argument( - '--algo_version', type=str, default=None, help='algo version') + '--algo_version', type=str, default=None, help='algo version' + ) parser.add_argument( - '--is_outer', - type=int, - default=1, - help='is outer pai or inner pai, the arguments are differed slightly due to history reasons' + '--is_outer', + type=int, + default=1, + help= + 'is outer pai or inner pai, the arguments are differed slightly due to history reasons' ) args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] @@ -238,7 +245,8 @@ def test_vector_retrieve(self): odps_oss_config.algo_version = args.algo_version algo_names = ['easy_rec_ext15', 'easy_rec_ext'] assert args.algo_name in algo_names, 'algo_name must be oneof: %s' % ( - ','.join(algo_names)) + ','.join(algo_names) + ) odps_oss_config.algo_name = args.algo_name if args.arn: odps_oss_config.arn = args.arn @@ -248,7 +256,9 @@ def test_vector_retrieve(self): prepare(odps_oss_config) tf.test.main() - bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name) + bucket = get_oss_bucket( + odps_oss_config.oss_key, odps_oss_config.oss_secret, + odps_oss_config.endpoint, odps_oss_config.bucket_name + ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) shutil.rmtree(odps_oss_config.temp_dir) diff --git a/easy_rec/python/test/odps_test_prepare.py b/easy_rec/python/test/odps_test_prepare.py index e4b0c23d2..60e6c3493 100644 --- a/easy_rec/python/test/odps_test_prepare.py +++ b/easy_rec/python/test/odps_test_prepare.py @@ -4,13 +4,11 @@ import glob import logging import os +import oss2 import shutil import sys -import oss2 - -from easy_rec.python.test.odps_test_util import OdpsOSSConfig -from easy_rec.python.test.odps_test_util import get_oss_bucket +from easy_rec.python.test.odps_test_util import OdpsOSSConfig, get_oss_bucket def download_data(ali_bucket, script_path): @@ -45,8 +43,10 @@ def download_data(ali_bucket, script_path): if not os.path.exists(dst_dir): os.makedirs(dst_dir) ali_bucket.get_object_to_file(obj_key, dst_path) - logging.info('down file oss://%s/%s to %s completed' % - (ali_bucket.bucket_name, obj_key, dst_path)) + logging.info( + 'down file oss://%s/%s to %s completed' % + (ali_bucket.bucket_name, obj_key, dst_path) + ) def merge_files(merge_dir, merge_out): @@ -107,7 +107,8 @@ def change_files(odps_oss_config, file_path): # tmp_e = tmp_e.replace('.aliyuncs.com', '.oss-internal.aliyun-inc.com') if '-Dbuckets=' in line: line = '-Dbuckets=oss://%s/?role_arn=%s&host=%s\n' % ( - odps_oss_config.bucket_name, odps_oss_config.arn, tmp_e) + odps_oss_config.bucket_name, odps_oss_config.arn, tmp_e + ) elif '-Darn=' in line or '-DossHost' in line: continue line = line.replace('{OSS_BUCKET_NAME}', odps_oss_config.bucket_name) @@ -136,28 +137,31 @@ def put_data_to_bucket(odps_oss_config): Args: odps_oss_config: odps oss config obj """ - test_bucket = get_oss_bucket(odps_oss_config.oss_key, - odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name) + test_bucket = get_oss_bucket( + odps_oss_config.oss_key, odps_oss_config.oss_secret, + odps_oss_config.endpoint, odps_oss_config.bucket_name + ) for sub_dir in ['configs']: for root, dirs, files in os.walk( - os.path.join(odps_oss_config.temp_dir, sub_dir)): + os.path.join(odps_oss_config.temp_dir, sub_dir) + ): for one_file in files: file_path = os.path.join(root, one_file) obj_path = file_path.split(sub_dir + '/')[1] dst_path = os.path.join(odps_oss_config.exp_dir, sub_dir, obj_path) test_bucket.put_object_from_file(dst_path, file_path) - logging.info('put %s to oss://%s/%s' % - (file_path, odps_oss_config.bucket_name, dst_path)) + logging.info( + 'put %s to oss://%s/%s' % + (file_path, odps_oss_config.bucket_name, dst_path) + ) def prepare(odps_oss_config): logging.info('temp_dir = %s' % odps_oss_config.temp_dir) - ali_bucket = get_oss_bucket(odps_oss_config.ali_oss_key, - odps_oss_config.ali_oss_secret, - odps_oss_config.ali_bucket_endpoint, - odps_oss_config.ali_bucket_name) + ali_bucket = get_oss_bucket( + odps_oss_config.ali_oss_key, odps_oss_config.ali_oss_secret, + odps_oss_config.ali_bucket_endpoint, odps_oss_config.ali_bucket_name + ) shutil.copytree(odps_oss_config.script_path, odps_oss_config.temp_dir) logging.info('start down data') download_data(ali_bucket, odps_oss_config.temp_dir) @@ -186,8 +190,9 @@ def prepare(odps_oss_config): if __name__ == '__main__': if len(sys.argv) < 5: - print('usage: %s ossutilconfig bucket_name rolearn odpsconfig' % - sys.argv[0]) + print( + 'usage: %s ossutilconfig bucket_name rolearn odpsconfig' % sys.argv[0] + ) sys.exit(1) odps_oss_config = OdpsOSSConfig() diff --git a/easy_rec/python/test/odps_test_util.py b/easy_rec/python/test/odps_test_util.py index 35dc7f743..54946edbf 100644 --- a/easy_rec/python/test/odps_test_util.py +++ b/easy_rec/python/test/odps_test_util.py @@ -3,26 +3,22 @@ import logging import os +import oss2 import time import traceback -import oss2 - try: from datahub import DataHub - from datahub.exceptions import InvalidOperationException - from datahub.exceptions import ResourceExistException + from datahub.exceptions import InvalidOperationException, ResourceExistException # NOQA # from datahub.exceptions import LimitExceededException # from datahub.exceptions import ResourceNotFoundException # from datahub.models import BlobRecord # from datahub.models import CursorType - from datahub.models import FieldType - from datahub.models import RecordSchema - from datahub.models import RecordType - from datahub.models import TupleRecord + from datahub.models import FieldType, RecordSchema, RecordType, TupleRecord except Exception: logging.error( - 'DataHub is not installed, please installed it by: pip install pydatahub') + 'DataHub is not installed, please installed it by: pip install pydatahub' + ) DataHub = None try: @@ -133,27 +129,29 @@ def clean_project(self): pass def clean_subscription(self, topic_name): - subscriptions = self.dh.list_subscription(self.dh_project, topic_name, '', - 1, 100).subscriptions + subscriptions = self.dh.list_subscription( + self.dh_project, topic_name, '', 1, 100 + ).subscriptions for subscription in subscriptions: self.dh.delete_subscription(self.dh_project, topic_name, subscription) def get_input_type(self, input_type): DhDict = { - 'INT64': FieldType.BIGINT, - 'INT32': FieldType.BIGINT, - 'STRING': FieldType.STRING, - 'BOOLEAN': FieldType.BOOLEAN, - 'FLOAT32': FieldType.DOUBLE, - 'FLOAT64': FieldType.DOUBLE + 'INT64': FieldType.BIGINT, + 'INT32': FieldType.BIGINT, + 'STRING': FieldType.STRING, + 'BOOLEAN': FieldType.BOOLEAN, + 'FLOAT32': FieldType.DOUBLE, + 'FLOAT64': FieldType.DOUBLE } return DhDict.get(input_type) def init_dh_and_odps(self): self.dh = DataHub(self.dh_id, self.dh_key, self.dh_endpoint) - self.odps = ODPS(self.dh_id, self.dh_key, self.project_name, - self.odps_endpoint) + self.odps = ODPS( + self.dh_id, self.dh_key, self.project_name, self.odps_endpoint + ) self.odpsTable = 'deepfm_train_%s' % self.time_stamp self.clean_project() read_odps = DataFrame(self.odps.get_table(self.odpsTable)) @@ -170,12 +168,13 @@ def init_dh_and_odps(self): try: # project_name, topic_name, shard_count, life_cycle, record_schema, comment self.dh.create_tuple_topic( - self.dh_project, - self.dh_topic, - 7, - 3, - record_schema, - comment='EasyRecTest') + self.dh_project, + self.dh_topic, + 7, + 3, + record_schema, + comment='EasyRecTest' + ) logging.info('create tuple topic %s success!' % self.dh_topic) except ResourceExistException: logging.info('topic %s already exist!' % self.dh_topic) @@ -184,8 +183,9 @@ def init_dh_and_odps(self): logging.error(traceback.format_exc()) try: self.dh.wait_shards_ready(self.dh_project, self.dh_topic) - logging.info('datahub[%s,%s] shards all ready' % - (self.dh_project, self.dh_topic)) + logging.info( + 'datahub[%s,%s] shards all ready' % (self.dh_project, self.dh_topic) + ) topic_result = self.dh.get_topic(self.dh_project, self.dh_topic) if topic_result.record_type != RecordType.TUPLE: logging.error('invalid topic type: %s' % str(topic_result.record_type)) diff --git a/easy_rec/python/test/pre_check_test.py b/easy_rec/python/test/pre_check_test.py index 58b295157..5a63f0948 100644 --- a/easy_rec/python/test/pre_check_test.py +++ b/easy_rec/python/test/pre_check_test.py @@ -2,7 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging - import tensorflow as tf from easy_rec.python.utils import test_utils @@ -27,26 +26,28 @@ def tearDown(self): def test_csv_input_train_with_check(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_taobao.config', - self._test_dir, - check_mode=True) + 'samples/model_config/dbmtl_on_taobao.config', + self._test_dir, + check_mode=True + ) self.assertTrue(self._success) def test_rtp_input_train_with_check(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.config', - self._test_dir, - check_mode=True) + 'samples/model_config/taobao_fg.config', self._test_dir, check_mode=True + ) self.assertTrue(self._success) def test_csv_input_with_pre_check(self): self._success = test_utils.test_single_pre_check( - 'samples/model_config/dbmtl_on_taobao.config', self._test_dir) + 'samples/model_config/dbmtl_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_rtp_input_with_pre_check(self): self._success = test_utils.test_single_pre_check( - 'samples/model_config/dbmtl_on_taobao.config', self._test_dir) + 'samples/model_config/dbmtl_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) diff --git a/easy_rec/python/test/predictor_test.py b/easy_rec/python/test/predictor_test.py index 7ad8ae36e..e68ed4a59 100644 --- a/easy_rec/python/test/predictor_test.py +++ b/easy_rec/python/test/predictor_test.py @@ -3,16 +3,14 @@ import csv import json import logging +import numpy as np import os import shutil - -import numpy as np import tensorflow as tf from easy_rec.python.inference.csv_predictor import CSVPredictor from easy_rec.python.inference.predictor import Predictor -from easy_rec.python.utils import config_util -from easy_rec.python.utils import test_utils +from easy_rec.python.utils import config_util, test_utils from easy_rec.python.utils.test_utils import RunAsSubprocess @@ -56,10 +54,10 @@ def test_lookup_pred(self): def test_pred_dict(self): predictor = Predictor('data/test/inference/tb_multitower_export/') field_keys = [ - 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', - 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', - 'age_level', 'pvalue_level', 'shopping_level', 'occupation', - 'new_user_class_level', 'tag_category_list', 'tag_brand_list', 'price' + 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', + 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', 'age_level', + 'pvalue_level', 'shopping_level', 'occupation', 'new_user_class_level', + 'tag_category_list', 'tag_brand_list', 'price' ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -72,12 +70,13 @@ def test_pred_dict(self): @RunAsSubprocess def test_pred_placeholder_named_by_input(self): predictor = Predictor( - 'data/test/inference/tb_multitower_placeholder_rename_export/') + 'data/test/inference/tb_multitower_placeholder_rename_export/' + ) field_keys = [ - 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', - 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', - 'age_level', 'pvalue_level', 'shopping_level', 'occupation', - 'new_user_class_level', 'tag_category_list', 'tag_brand_list', 'price' + 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', + 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', 'age_level', + 'pvalue_level', 'shopping_level', 'occupation', 'new_user_class_level', + 'tag_category_list', 'tag_brand_list', 'price' ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -108,10 +107,10 @@ def test_fm_pred_list(self): def test_fm_pred_dict(self): predictor = Predictor('data/test/inference/fm_export/') field_keys = [ - 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', - 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', - 'age_level', 'pvalue_level', 'shopping_level', 'occupation', - 'new_user_class_level', 'tag_category_list', 'tag_brand_list', 'price' + 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', + 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', 'age_level', + 'pvalue_level', 'shopping_level', 'occupation', 'new_user_class_level', + 'tag_category_list', 'tag_brand_list', 'price' ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -138,25 +137,31 @@ def tearDown(self): @RunAsSubprocess def test_local_pred(self): test_input_path = 'data/test/inference/taobao_infer_data.txt' - self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') + self._test_output_path = os.path.join( + self._test_dir, 'taobao_infer_result' + ) saved_model_dir = 'data/test/inference/tb_multitower_export/' - pipeline_config_path = os.path.join(saved_model_dir, - 'assets/pipeline.config') + pipeline_config_path = os.path.join( + saved_model_dir, 'assets/pipeline.config' + ) pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False) + pipeline_config_path, False + ) predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - output_sep=';', - selected_cols='') + saved_model_dir, + pipeline_config.data_config, + output_sep=';', + selected_cols='' + ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='ALL_COLUMNS', - output_cols='ALL_COLUMNS', - slice_id=0, - slice_num=1) + test_input_path, + self._test_output_path, + reserved_cols='ALL_COLUMNS', + output_cols='ALL_COLUMNS', + slice_id=0, + slice_num=1 + ) header_truth = 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;'\ 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' \ 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' @@ -169,28 +174,34 @@ def test_local_pred(self): @RunAsSubprocess def test_local_pred_with_header(self): test_input_path = 'data/test/inference/taobao_infer_data_with_header.txt' - self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') + self._test_output_path = os.path.join( + self._test_dir, 'taobao_infer_result' + ) saved_model_dir = 'data/test/inference/tb_multitower_export/' - pipeline_config_path = os.path.join(saved_model_dir, - 'assets/pipeline.config') + pipeline_config_path = os.path.join( + saved_model_dir, 'assets/pipeline.config' + ) pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False) + pipeline_config_path, False + ) pipeline_config.data_config.with_header = True predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - with_header=True, - output_sep=';', - selected_cols='') + saved_model_dir, + pipeline_config.data_config, + with_header=True, + output_sep=';', + selected_cols='' + ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='ALL_COLUMNS', - output_cols='ALL_COLUMNS', - slice_id=0, - slice_num=1) + test_input_path, + self._test_output_path, + reserved_cols='ALL_COLUMNS', + output_cols='ALL_COLUMNS', + slice_id=0, + slice_num=1 + ) header_truth = 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;'\ 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' \ 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' @@ -203,12 +214,13 @@ def test_local_pred_with_header(self): @RunAsSubprocess def test_local_pred_without_config(self): test_input_path = 'data/test/inference/taobao_infer_data.txt' - self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') + self._test_output_path = os.path.join( + self._test_dir, 'taobao_infer_result' + ) saved_model_dir = 'data/test/inference/tb_multitower_export/' - self._success = test_utils.test_single_predict(self._test_dir, - test_input_path, - self._test_output_path, - saved_model_dir) + self._success = test_utils.test_single_predict( + self._test_dir, test_input_path, self._test_output_path, saved_model_dir + ) self.assertTrue(self._success) with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() @@ -217,26 +229,32 @@ def test_local_pred_without_config(self): @RunAsSubprocess def test_local_pred_with_part_col(self): test_input_path = 'data/test/inference/taobao_infer_data.txt' - self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') + self._test_output_path = os.path.join( + self._test_dir, 'taobao_infer_result' + ) saved_model_dir = 'data/test/inference/tb_multitower_export/' - pipeline_config_path = os.path.join(saved_model_dir, - 'assets/pipeline.config') + pipeline_config_path = os.path.join( + saved_model_dir, 'assets/pipeline.config' + ) pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False) + pipeline_config_path, False + ) predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - output_sep=';', - selected_cols='') + saved_model_dir, + pipeline_config.data_config, + output_sep=';', + selected_cols='' + ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='clk,buy,user_id,adgroup_id', - output_cols='probs', - slice_id=0, - slice_num=1) + test_input_path, + self._test_output_path, + reserved_cols='clk,buy,user_id,adgroup_id', + output_cols='probs', + slice_id=0, + slice_num=1 + ) header_truth = 'probs;clk;buy;user_id;adgroup_id' with open(self._test_output_path + '/part-0.csv', 'r') as f: @@ -247,26 +265,31 @@ def test_local_pred_with_part_col(self): @RunAsSubprocess def test_local_pred_rtp(self): test_input_path = 'data/test/inference/taobao_infer_rtp_data.txt' - self._test_output_path = os.path.join(self._test_dir, - 'taobao_test_feature_result') + self._test_output_path = os.path.join( + self._test_dir, 'taobao_test_feature_result' + ) saved_model_dir = 'data/test/inference/tb_multitower_rtp_export/' - pipeline_config_path = os.path.join(saved_model_dir, - 'assets/pipeline.config') + pipeline_config_path = os.path.join( + saved_model_dir, 'assets/pipeline.config' + ) pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False) + pipeline_config_path, False + ) predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - output_sep=';', - selected_cols='0,3') + saved_model_dir, + pipeline_config.data_config, + output_sep=';', + selected_cols='0,3' + ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='ALL_COLUMNS', - output_cols='ALL_COLUMNS', - slice_id=0, - slice_num=1) + test_input_path, + self._test_output_path, + reserved_cols='ALL_COLUMNS', + output_cols='ALL_COLUMNS', + slice_id=0, + slice_num=1 + ) header_truth = 'logits;probs;clk;no_used_1;no_used_2;features' with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() @@ -276,26 +299,31 @@ def test_local_pred_rtp(self): @RunAsSubprocess def test_local_pred_rtp_with_part_col(self): test_input_path = 'data/test/inference/taobao_infer_rtp_data.txt' - self._test_output_path = os.path.join(self._test_dir, - 'taobao_test_feature_result') + self._test_output_path = os.path.join( + self._test_dir, 'taobao_test_feature_result' + ) saved_model_dir = 'data/test/inference/tb_multitower_rtp_export/' - pipeline_config_path = os.path.join(saved_model_dir, - 'assets/pipeline.config') + pipeline_config_path = os.path.join( + saved_model_dir, 'assets/pipeline.config' + ) pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False) + pipeline_config_path, False + ) predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - output_sep=';', - selected_cols='0,3') + saved_model_dir, + pipeline_config.data_config, + output_sep=';', + selected_cols='0,3' + ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='clk,features,no_used_1', - output_cols='ALL_COLUMNS', - slice_id=0, - slice_num=1) + test_input_path, + self._test_output_path, + reserved_cols='clk,features,no_used_1', + output_cols='ALL_COLUMNS', + slice_id=0, + slice_num=1 + ) header_truth = 'logits;probs;clk;features;no_used_1' with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() @@ -305,36 +333,43 @@ def test_local_pred_rtp_with_part_col(self): @RunAsSubprocess def test_local_pred_embedding(self): test_input_path = 'data/test/inference/taobao_item_feature_data.csv' - self._test_output_path = os.path.join(self._test_dir, 'taobao_item_feature') + self._test_output_path = os.path.join( + self._test_dir, 'taobao_item_feature' + ) saved_model_dir = 'data/test/inference/dssm_item_model/' - pipeline_config_path = os.path.join(saved_model_dir, - 'assets/pipeline.config') + pipeline_config_path = os.path.join( + saved_model_dir, 'assets/pipeline.config' + ) pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False) + pipeline_config_path, False + ) predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - ds_vector_recall=True, - output_sep=';', - selected_cols='pid,adgroup_id,cate_id,campaign_id,customer,brand,price') + saved_model_dir, + pipeline_config.data_config, + ds_vector_recall=True, + output_sep=';', + selected_cols='pid,adgroup_id,cate_id,campaign_id,customer,brand,price' + ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='adgroup_id', - output_cols='item_emb', - slice_id=0, - slice_num=1) + test_input_path, + self._test_output_path, + reserved_cols='adgroup_id', + output_cols='item_emb', + slice_id=0, + slice_num=1 + ) with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() self.assertTrue( - output_res[1] == - '-0.187066,-0.027638,-0.117294,0.115318,-0.273561,0.035698,-0.055832,' - '0.226849,-0.105808,-0.152751,0.081528,-0.183329,0.134619,0.185392,' - '0.096774,0.104428,0.161868,0.269710,-0.268538,0.138760,-0.170105,' - '0.232625,-0.121130,0.198466,-0.078941,0.017774,0.268834,-0.238553,0.084058,' - '-0.269466,-0.289651,0.179517;620392\n') + output_res[1] == + '-0.187066,-0.027638,-0.117294,0.115318,-0.273561,0.035698,-0.055832,' + '0.226849,-0.105808,-0.152751,0.081528,-0.183329,0.134619,0.185392,' + '0.096774,0.104428,0.161868,0.269710,-0.268538,0.138760,-0.170105,' + '0.232625,-0.121130,0.198466,-0.078941,0.017774,0.268834,-0.238553,0.084058,' + '-0.269466,-0.289651,0.179517;620392\n' + ) class PredictorTestV2(tf.test.TestCase): @@ -368,7 +403,8 @@ def test_pred_multi(self): line_str = line_str.strip() line_pred = json.loads(line_str) self.assertTrue( - np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-6) + np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-6 + ) @RunAsSubprocess def test_pred_single(self): @@ -387,7 +423,8 @@ def test_pred_single(self): line_str = line_str.strip() line_pred = json.loads(line_str) self.assertTrue( - np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-5) + np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-5 + ) if __name__ == '__main__': diff --git a/easy_rec/python/test/rtp_convert_test.py b/easy_rec/python/test/rtp_convert_test.py index 4210c9dc2..718300355 100644 --- a/easy_rec/python/test/rtp_convert_test.py +++ b/easy_rec/python/test/rtp_convert_test.py @@ -3,11 +3,9 @@ import logging import os - import tensorflow as tf -from easy_rec.python.utils import config_util -from easy_rec.python.utils import test_utils +from easy_rec.python.utils import config_util, test_utils class RTPConvertTest(tf.test.TestCase): @@ -33,13 +31,16 @@ def test_rtp_convert(self): --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 """ % pipeline_config_path - proc = test_utils.run_cmd(convert_cmd, - '%s/log_%s.txt' % (test_dir, 'convert')) + proc = test_utils.run_cmd( + convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + ) proc.wait() self.assertTrue(proc.returncode == 0) self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir)) + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir + ) + ) test_utils.clean_up(test_dir) def test_rtp_convert_bucketize(self): @@ -57,13 +58,16 @@ def test_rtp_convert_bucketize(self): --eval_input_path data/test/rtp/taobao_test_bucketize_feature.txt --selected_cols 0,3 --num_steps 400 """ % pipeline_config_path - proc = test_utils.run_cmd(convert_cmd, - '%s/log_%s.txt' % (test_dir, 'convert')) + proc = test_utils.run_cmd( + convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + ) proc.wait() self.assertTrue(proc.returncode == 0) self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir)) + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir + ) + ) test_utils.clean_up(test_dir) def test_rtp_convert_bucketize_v2(self): @@ -81,20 +85,24 @@ def test_rtp_convert_bucketize_v2(self): --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 """ % pipeline_config_path - proc = test_utils.run_cmd(convert_cmd, - '%s/log_%s.txt' % (test_dir, 'convert')) + proc = test_utils.run_cmd( + convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + ) proc.wait() self.assertTrue(proc.returncode == 0) tmp_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path) + pipeline_config_path + ) for feature_config in tmp_config.feature_configs: if feature_config.input_names[0] == 'price': assert len(feature_config.boundaries) == 6 self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir)) + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir + ) + ) test_utils.clean_up(test_dir) def test_rtp_convert_test_model_config(self): @@ -111,21 +119,25 @@ def test_rtp_convert_test_model_config(self): --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 """ % pipeline_config_path - proc = test_utils.run_cmd(convert_cmd, - '%s/log_%s.txt' % (test_dir, 'convert')) + proc = test_utils.run_cmd( + convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + ) proc.wait() self.assertTrue(proc.returncode == 0) tmp_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path) + pipeline_config_path + ) assert len(tmp_config.model_config.wide_and_deep.dnn.hidden_units) == 2 assert tmp_config.model_config.wide_and_deep.dnn.hidden_units[0] == 48 assert tmp_config.model_config.wide_and_deep.dnn.hidden_units[1] == 24 assert tmp_config.model_dir == 'experiments/rtp_fg/wide_and_deep_update_model' self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir)) + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir + ) + ) test_utils.clean_up(test_dir) diff --git a/easy_rec/python/test/run.py b/easy_rec/python/test/run.py index 0c7ac4c79..6ea41714a 100644 --- a/easy_rec/python/test/run.py +++ b/easy_rec/python/test/run.py @@ -5,9 +5,8 @@ import logging import os import sys -import unittest - import tensorflow as tf +import unittest from easy_rec.python.utils import test_utils @@ -17,18 +16,22 @@ tf.app.flags.DEFINE_bool('list_tests', False, 'list all tests') tf.app.flags.DEFINE_string('list_test_to_file', None, 'list all tests') tf.app.flags.DEFINE_string('pattern', '*_test.py', 'test file pattern') -tf.app.flags.DEFINE_string('test_dir', 'easy_rec/python/test', - 'directory to be tested') -tf.app.flags.DEFINE_integer('num_parallel', 10, - 'number of parallel executed cases.') -tf.app.flags.DEFINE_integer('timeout', 3600, - 'maximal execute time in seconds for each case.') +tf.app.flags.DEFINE_string( + 'test_dir', 'easy_rec/python/test', 'directory to be tested' +) +tf.app.flags.DEFINE_integer( + 'num_parallel', 10, 'number of parallel executed cases.' +) +tf.app.flags.DEFINE_integer( + 'timeout', 3600, 'maximal execute time in seconds for each case.' +) FLAGS = tf.flags.FLAGS def gather_test_cases(test_dir, pattern): discover = unittest.defaultTestLoader.discover( - test_dir, pattern=pattern, top_level_dir=None) + test_dir, pattern=pattern, top_level_dir=None + ) all_tests = [] for suite_discovered in discover: for test_case in suite_discovered: @@ -76,8 +79,9 @@ def main(argv): test_log_dir = os.path.join(test_dir, 'logs') if not os.path.exists(test_log_dir): os.makedirs(test_log_dir) - logging.info('Total number of cases: %d test_dir: %s' % - (len(all_tests), test_dir)) + logging.info( + 'Total number of cases: %d test_dir: %s' % (len(all_tests), test_dir) + ) max_num_port_per_proc = 3 total_port_num = (max_num_port_per_proc + 2) * FLAGS.num_parallel * 10 @@ -101,7 +105,8 @@ def main(argv): cmd = 'python -m easy_rec.python.test.%s %s' % (case_file, case_name) log_file = '%s/%s.%s.log' % (test_log_dir, case_file, case_name) tmp_ports = ','.join( - [str(x) for x in all_available_ports[:max_num_port_per_proc]]) + [str(x) for x in all_available_ports[:max_num_port_per_proc]] + ) all_available_ports = all_available_ports[max_num_port_per_proc:] logging.info('Run %s.%s Log: %s' % (case_file, case_name, log_file)) @@ -113,10 +118,13 @@ def main(argv): for proc in procs: try: test_utils.proc_wait( - proc, timeout=int(os.environ.get('TEST_TIME_OUT', 1200))) + proc, timeout=int(os.environ.get('TEST_TIME_OUT', 1200)) + ) except Exception as ex: fail_file, fail_name = procs[proc] - logging.info('Case Exception: %s.%s %s' % (fail_file, fail_name, str(ex))) + logging.info( + 'Case Exception: %s.%s %s' % (fail_file, fail_name, str(ex)) + ) proc.kill() if proc.returncode != 0: @@ -126,8 +134,10 @@ def main(argv): if len(failed_cases) > 0: logging.info('Number Cases Failed: %d' % len(failed_cases)) for fail_file, fail_name, exit_code in failed_cases: - logging.info('\t%s.%s failed, exit_code:%d log: %s.%s.log' % - (fail_file, fail_name, exit_code, fail_file, fail_name)) + logging.info( + '\t%s.%s failed, exit_code:%d log: %s.%s.log' % + (fail_file, fail_name, exit_code, fail_file, fail_name) + ) return 1 else: logging.info('TestSucceed.') diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 14b511a63..f7bc77acc 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -3,22 +3,18 @@ import glob import logging +import numpy as np import os +import six +import tensorflow as tf import threading import time import unittest - -import numpy as np -import six -import tensorflow as tf from distutils.version import LooseVersion from tensorflow.python.platform import gfile from easy_rec.python.main import predict -from easy_rec.python.utils import config_util -from easy_rec.python.utils import constant -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import test_utils +from easy_rec.python.utils import config_util, constant, estimator_utils, test_utils # NOQA try: import graphlearn as gl @@ -55,88 +51,102 @@ def tearDown(self): def test_deepfm_with_lookup_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_lookup.config', self._test_dir) + 'samples/model_config/deepfm_lookup.config', self._test_dir + ) self.assertTrue(self._success) def test_deepfm_with_combo_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_ctr.config', self._test_dir) + 'samples/model_config/deepfm_combo_on_avazu_ctr.config', self._test_dir + ) self.assertTrue(self._success) def test_deepfm_with_combo_v2_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_v2_on_avazu_ctr.config', - self._test_dir) + 'samples/model_config/deepfm_combo_v2_on_avazu_ctr.config', + self._test_dir + ) self.assertTrue(self._success) def test_deepfm_with_combo_v3_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_v3_on_avazu_ctr.config', - self._test_dir) + 'samples/model_config/deepfm_combo_v3_on_avazu_ctr.config', + self._test_dir + ) self.assertTrue(self._success) def test_deepfm_freeze_gradient(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_freeze_gradient.config', self._test_dir) + 'samples/model_config/deepfm_freeze_gradient.config', self._test_dir + ) self.assertTrue(self._success) def test_deepfm_with_vocab_list(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_vocab_list_on_avazu_ctr.config', - self._test_dir) + 'samples/model_config/deepfm_vocab_list_on_avazu_ctr.config', + self._test_dir + ) self.assertTrue(self._success) def test_deepfm_with_multi_class(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - self._test_dir) + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + self._test_dir + ) self.assertTrue(self._success) def test_wide_and_deep_no_final(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/wide_and_deep_no_final_on_avazau_ctr.config', - self._test_dir) + 'samples/model_config/wide_and_deep_no_final_on_avazau_ctr.config', + self._test_dir + ) self.assertTrue(self._success) def test_wide_and_deep(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/wide_and_deep_on_avazau_ctr.config', - self._test_dir) + 'samples/model_config/wide_and_deep_on_avazau_ctr.config', self._test_dir + ) self.assertTrue(self._success) def test_wide_and_deep_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/wide_and_deep_backbone_on_avazau.config', - self._test_dir) + 'samples/model_config/wide_and_deep_backbone_on_avazau.config', + self._test_dir + ) self.assertTrue(self._success) def test_dlrm(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dlrm_on_taobao.config', self._test_dir) + 'samples/model_config/dlrm_on_taobao.config', self._test_dir + ) def test_dlrm_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dlrm_backbone_on_taobao.config', self._test_dir) + 'samples/model_config/dlrm_backbone_on_taobao.config', self._test_dir + ) def test_adamw_optimizer(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_adamw_ctr.config', - self._test_dir) + 'samples/model_config/deepfm_combo_on_avazu_adamw_ctr.config', + self._test_dir + ) self.assertTrue(self._success) def test_momentumw_optimizer(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_momentumw_ctr.config', - self._test_dir) + 'samples/model_config/deepfm_combo_on_avazu_momentumw_ctr.config', + self._test_dir + ) self.assertTrue(self._success) def test_deepfm_with_param_edit(self): model_dir = os.path.join(self._test_dir, 'train_new') self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - self._test_dir, - hyperparam_str='{"model_dir":"%s", ' - '"model_config.deepfm.wide_output_dim": 32}' % model_dir) + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + self._test_dir, + hyperparam_str='{"model_dir":"%s", ' + '"model_config.deepfm.wide_output_dim": 32}' % model_dir + ) self.assertTrue(self._success) config_path = os.path.join(model_dir, 'pipeline.config') pipeline_config = config_util.get_configs_from_pipeline_file(config_path) @@ -145,32 +155,36 @@ def test_deepfm_with_param_edit(self): def test_multi_tower(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_on_taobao.config', self._test_dir) + 'samples/model_config/multi_tower_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_multi_tower_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_backbone_on_taobao.config', - self._test_dir) + 'samples/model_config/multi_tower_backbone_on_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_multi_tower_gauc(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_on_taobao_gauc.config', - self._test_dir) + 'samples/model_config/multi_tower_on_taobao_gauc.config', self._test_dir + ) self.assertTrue(self._success) def test_multi_tower_session_auc(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_on_taobao_session_auc.config', - self._test_dir) + 'samples/model_config/multi_tower_on_taobao_session_auc.config', + self._test_dir + ) self.assertTrue(self._success) def test_multi_tower_save_checkpoint_secs(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_save_secs_on_taobao.config', - self._test_dir, - total_steps=100) + 'samples/model_config/multi_tower_save_secs_on_taobao.config', + self._test_dir, + total_steps=100 + ) ckpts_times = [] ckpt_dir = os.path.join(self._test_dir, 'train') for filepath in os.listdir(ckpt_dir): @@ -182,22 +196,26 @@ def test_multi_tower_save_checkpoint_secs(self): diffs = list(ckpts_times[1:] - ckpts_times[:-1]) logging.info('nearby ckpts_times diff = %s' % diffs) self.assertAllClose( - ckpts_times[1:] - ckpts_times[:-1], [20] * (len(ckpts_times) - 1), - atol=20) + ckpts_times[1:] - ckpts_times[:-1], [20] * (len(ckpts_times) - 1), + atol=20 + ) self.assertTrue(self._success) def test_keep_ckpt_max(self): def _post_check_func(pipeline_config): - ckpt_prefix = os.path.join(pipeline_config.model_dir, 'model.ckpt-*.meta') + ckpt_prefix = os.path.join( + pipeline_config.model_dir, 'model.ckpt-*.meta' + ) ckpts = gfile.Glob(ckpt_prefix) assert len(ckpts) == 3, 'invalid number of checkpoints: %d' % len(ckpts) self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_ckpt_keep_3_on_taobao.config', - self._test_dir, - total_steps=500, - post_check_func=_post_check_func) + 'samples/model_config/multi_tower_ckpt_keep_3_on_taobao.config', + self._test_dir, + total_steps=500, + post_check_func=_post_check_func + ) def test_multi_tower_with_best_exporter(self): @@ -208,16 +226,17 @@ def _post_check_func(pipeline_config): assert len(best_ckpts) <= 2, 'too many best ckpts: %s' % str(best_ckpts) best_exports = os.path.join(model_dir, 'export/best/*') best_exports = gfile.Glob(best_exports) - assert len( - best_exports) <= 2, 'too many best exports: %s' % str(best_exports) + assert len(best_exports + ) <= 2, 'too many best exports: %s' % str(best_exports) return True self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_best_export_on_taobao.config', - self._test_dir, - total_steps=800, - post_check_func=_post_check_func, - timeout=3000) + 'samples/model_config/multi_tower_best_export_on_taobao.config', + self._test_dir, + total_steps=800, + post_check_func=_post_check_func, + timeout=3000 + ) self.assertTrue(self._success) def test_latest_ckpt(self): @@ -230,15 +249,18 @@ def test_latest_ckpt_v2(self): def _post_check_func(pipeline_config): logging.info('model_dir: %s' % pipeline_config.model_dir) - logging.info('latest_checkpoint: %s' % - estimator_utils.latest_checkpoint(pipeline_config.model_dir)) + logging.info( + 'latest_checkpoint: %s' % + estimator_utils.latest_checkpoint(pipeline_config.model_dir) + ) return tf.train.latest_checkpoint(pipeline_config.model_dir) == \ estimator_utils.latest_checkpoint(pipeline_config.model_dir) self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.config', - self._test_dir, - post_check_func=_post_check_func) + 'samples/model_config/taobao_fg.config', + self._test_dir, + post_check_func=_post_check_func + ) self.assertTrue(self._success) def test_oss_stop_signal(self): @@ -260,9 +282,10 @@ def _watch_func(): watch_th.start() self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/taobao_fg_signal_stop.config', - self._test_dir, - total_steps=1000) + 'samples/model_config/taobao_fg_signal_stop.config', + self._test_dir, + total_steps=1000 + ) self.assertTrue(self._success) watch_th.join() final_ckpt = estimator_utils.latest_checkpoint(train_dir) @@ -274,9 +297,10 @@ def _watch_func(): def test_dead_line_stop_signal(self): train_dir = os.path.join(self._test_dir, 'train/') self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/dead_line_stop.config', - self._test_dir, - total_steps=1000) + 'samples/model_config/dead_line_stop.config', + self._test_dir, + total_steps=1000 + ) self.assertTrue(self._success) final_ckpt = estimator_utils.latest_checkpoint(train_dir) ckpt_version = estimator_utils.get_ckpt_version(final_ckpt) @@ -289,17 +313,21 @@ def test_fine_tune_latest_ckpt_path(self): def _post_check_func(pipeline_config): logging.info('model_dir: %s' % pipeline_config.model_dir) pipeline_config = config_util.get_configs_from_pipeline_file( - os.path.join(pipeline_config.model_dir, 'pipeline.config'), False) - logging.info('fine_tune_checkpoint: %s' % - pipeline_config.train_config.fine_tune_checkpoint) + os.path.join(pipeline_config.model_dir, 'pipeline.config'), False + ) + logging.info( + 'fine_tune_checkpoint: %s' % + pipeline_config.train_config.fine_tune_checkpoint + ) return pipeline_config.train_config.fine_tune_checkpoint == \ 'data/test/mt_ckpt/model.ckpt-100' self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_on_taobao.config', - self._test_dir, - fine_tune_checkpoint='data/test/mt_ckpt', - post_check_func=_post_check_func) + 'samples/model_config/multi_tower_on_taobao.config', + self._test_dir, + fine_tune_checkpoint='data/test/mt_ckpt', + post_check_func=_post_check_func + ) self.assertTrue(self._success) def test_fine_tune_ckpt(self): @@ -312,107 +340,127 @@ def _post_check_func(pipeline_config): return test_utils.test_single_train_eval(pipeline_config, test_dir) self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.config', - self._test_dir, - post_check_func=_post_check_func) + 'samples/model_config/taobao_fg.config', + self._test_dir, + post_check_func=_post_check_func + ) self.assertTrue(self._success) def test_multi_tower_multi_value_export(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_multi_value_export_on_taobao.config', - self._test_dir) + 'samples/model_config/multi_tower_multi_value_export_on_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_multi_tower_fg_input(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.config', self._test_dir) + 'samples/model_config/taobao_fg.config', self._test_dir + ) self.assertTrue(self._success) def test_multi_tower_fg_json_config(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.json', self._test_dir) + 'samples/model_config/taobao_fg.json', self._test_dir + ) self.assertTrue(self._success) def test_fm(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/fm_on_taobao.config', self._test_dir) + 'samples/model_config/fm_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_place_embed_on_cpu(self): os.environ['place_embedding_on_cpu'] = 'True' self._success = test_utils.test_single_train_eval( - 'samples/model_config/fm_on_taobao.config', self._test_dir) + 'samples/model_config/fm_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_din(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/din_on_taobao.config', self._test_dir) + 'samples/model_config/din_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_din_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/din_backbone_on_taobao.config', self._test_dir) + 'samples/model_config/din_backbone_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_bst(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/bst_on_taobao.config', self._test_dir) + 'samples/model_config/bst_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_bst_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/bst_backbone_on_taobao.config', self._test_dir) + 'samples/model_config/bst_backbone_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_cl4srec(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/cl4srec_on_taobao.config', self._test_dir) + 'samples/model_config/cl4srec_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_dcn(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dcn_on_taobao.config', self._test_dir) + 'samples/model_config/dcn_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_ziln_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mlp_on_taobao_with_ziln_loss.config', - self._test_dir) + 'samples/model_config/mlp_on_taobao_with_ziln_loss.config', + self._test_dir + ) self.assertTrue(self._success) def test_fibinet(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/fibinet_on_taobao.config', self._test_dir) + 'samples/model_config/fibinet_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_masknet(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/masknet_on_taobao.config', self._test_dir) + 'samples/model_config/masknet_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_dcn_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dcn_backbone_on_taobao.config', self._test_dir) + 'samples/model_config/dcn_backbone_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_dcn_with_f1(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dcn_f1_on_taobao.config', self._test_dir) + 'samples/model_config/dcn_f1_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_autoint(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/autoint_on_taobao.config', self._test_dir) + 'samples/model_config/autoint_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_uniter(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/uniter_on_movielens.config', self._test_dir) + 'samples/model_config/uniter_on_movielens.config', self._test_dir + ) self.assertTrue(self._success) def test_highway(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/highway_on_movielens.config', self._test_dir) + 'samples/model_config/highway_on_movielens.config', self._test_dir + ) self.assertTrue(self._success) # @unittest.skipIf( @@ -426,107 +474,124 @@ def test_highway(self): def test_cdn(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/cdn_on_taobao.config', self._test_dir) + 'samples/model_config/cdn_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_ppnet(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/ppnet_on_taobao.config', self._test_dir) + 'samples/model_config/ppnet_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_uniter_only_text_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/uniter_on_movielens_only_text_feature.config', - self._test_dir) + 'samples/model_config/uniter_on_movielens_only_text_feature.config', + self._test_dir + ) self.assertTrue(self._success) def test_uniter_only_image_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/uniter_on_movielens_only_image_feature.config', - self._test_dir) + 'samples/model_config/uniter_on_movielens_only_image_feature.config', + self._test_dir + ) self.assertTrue(self._success) def test_cmbf(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/cmbf_on_movielens.config', self._test_dir) + 'samples/model_config/cmbf_on_movielens.config', self._test_dir + ) self.assertTrue(self._success) def test_cmbf_with_multi_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/cmbf_with_multi_loss.config', self._test_dir) + 'samples/model_config/cmbf_with_multi_loss.config', self._test_dir + ) self.assertTrue(self._success) def test_cmbf_has_other_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/cmbf_on_movielens_has_other_feature.config', - self._test_dir) + 'samples/model_config/cmbf_on_movielens_has_other_feature.config', + self._test_dir + ) self.assertTrue(self._success) def test_cmbf_only_text_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/cmbf_on_movielens_only_text_feature.config', - self._test_dir) + 'samples/model_config/cmbf_on_movielens_only_text_feature.config', + self._test_dir + ) self.assertTrue(self._success) def test_cmbf_only_image_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/cmbf_on_movielens_only_image_feature.config', - self._test_dir) + 'samples/model_config/cmbf_on_movielens_only_image_feature.config', + self._test_dir + ) self.assertTrue(self._success) def test_dssm(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_on_taobao.config', self._test_dir) + 'samples/model_config/dssm_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_dropoutnet(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dropoutnet_on_taobao.config', self._test_dir) + 'samples/model_config/dropoutnet_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_metric_learning(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/metric_learning_on_taobao.config', self._test_dir) + 'samples/model_config/metric_learning_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_neg_sampler_on_taobao.config', - self._test_dir) + 'samples/model_config/dssm_neg_sampler_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler_v2(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_neg_sampler_v2_on_taobao.config', - self._test_dir) + 'samples/model_config/dssm_neg_sampler_v2_on_taobao.config', + self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_hard_neg_sampler(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_hard_neg_sampler_on_taobao.config', - self._test_dir) + 'samples/model_config/dssm_hard_neg_sampler_on_taobao.config', + self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_hard_neg_regular_sampler(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_hard_neg_sampler_regular_on_taobao.config', - self._test_dir) + 'samples/model_config/dssm_hard_neg_sampler_regular_on_taobao.config', + self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_hard_neg_sampler_v2(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_hard_neg_sampler_v2_on_taobao.config', - self._test_dir) + 'samples/model_config/dssm_hard_neg_sampler_v2_on_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_dssm_no_norm(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_inner_prod_on_taobao.config', self._test_dir) + 'samples/model_config/dssm_inner_prod_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) # def test_dssm_with_regression(self): @@ -545,7 +610,9 @@ def _test_kd(self, config0, config1): @test_utils.RunAsSubprocess def _gen_kd_data(train_path, eval_path): - pred_result = predict(config_path, None, pipeline_config.train_input_path) + pred_result = predict( + config_path, None, pipeline_config.train_input_path + ) with gfile.GFile(pipeline_config.train_input_path, 'r') as fin: with gfile.GFile(train_path, 'w') as fout: for line, pred in zip(fin, pred_result): @@ -568,156 +635,184 @@ def _gen_kd_data(train_path, eval_path): pipeline_config = config_util.get_configs_from_pipeline_file(config1) pipeline_config.train_input_path = train_path pipeline_config.eval_input_path = eval_path - config_util.save_pipeline_config(pipeline_config, self._test_dir, - 'kd_pipeline.config') + config_util.save_pipeline_config( + pipeline_config, self._test_dir, 'kd_pipeline.config' + ) self._success = test_utils.test_single_train_eval( - os.path.join(self._test_dir, 'kd_pipeline.config'), - os.path.join(self._test_dir, 'kd')) + os.path.join(self._test_dir, 'kd_pipeline.config'), + os.path.join(self._test_dir, 'kd') + ) self.assertTrue(self._success) def test_dssm_with_kd(self): - self._test_kd('samples/model_config/multi_tower_on_taobao.config', - 'samples/model_config/dssm_kd_on_taobao.config') + self._test_kd( + 'samples/model_config/multi_tower_on_taobao.config', + 'samples/model_config/dssm_kd_on_taobao.config' + ) def test_deepfm_multi_class_with_kd(self): - self._test_kd('samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - 'samples/model_config/deepfm_multi_cls_small.config') + self._test_kd( + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + 'samples/model_config/deepfm_multi_cls_small.config' + ) def test_mind(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mind_on_taobao.config', self._test_dir) + 'samples/model_config/mind_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_mind_with_time_id(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mind_on_taobao_with_time.config', self._test_dir) + 'samples/model_config/mind_on_taobao_with_time.config', self._test_dir + ) self.assertTrue(self._success) def test_deepfm_with_regression(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_reg.config', self._test_dir) + 'samples/model_config/deepfm_combo_on_avazu_reg.config', self._test_dir + ) self.assertTrue(self._success) def test_deepfm_with_sigmoid_l2_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_sigmoid_l2.config', - self._test_dir) + 'samples/model_config/deepfm_combo_on_avazu_sigmoid_l2.config', + self._test_dir + ) self.assertTrue(self._success) def test_deepfm_with_embedding_learning_rate(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_emblr_ctr.config', - self._test_dir) + 'samples/model_config/deepfm_combo_on_avazu_emblr_ctr.config', + self._test_dir + ) self.assertTrue(self._success) def test_deepfm_with_eval_online(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_eval_online_ctr.config', - self._test_dir) + 'samples/model_config/deepfm_combo_on_avazu_eval_online_ctr.config', + self._test_dir + ) self.assertTrue(self._success) def test_deepfm_with_eval_online_gauc(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_eval_online_gauc_ctr.config', - self._test_dir) + 'samples/model_config/deepfm_combo_on_avazu_eval_online_gauc_ctr.config', + self._test_dir + ) self.assertTrue(self._success) def test_mmoe(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mmoe_on_taobao.config', self._test_dir) + 'samples/model_config/mmoe_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_mmoe_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mmoe_backbone_on_taobao.config', self._test_dir) + 'samples/model_config/mmoe_backbone_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_mmoe_with_multi_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mmoe_on_taobao_with_multi_loss.config', - self._test_dir) + 'samples/model_config/mmoe_on_taobao_with_multi_loss.config', + self._test_dir + ) self.assertTrue(self._success) def test_mmoe_deprecated(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mmoe_on_taobao_deprecated.config', self._test_dir) + 'samples/model_config/mmoe_on_taobao_deprecated.config', self._test_dir + ) self.assertTrue(self._success) def test_simple_multi_task(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/simple_multi_task_on_taobao.config', - self._test_dir) + 'samples/model_config/simple_multi_task_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_simple_multi_task_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/simple_multi_task_backbone_on_taobao.config', - self._test_dir) + 'samples/model_config/simple_multi_task_backbone_on_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_esmm(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/esmm_on_taobao.config', self._test_dir) + 'samples/model_config/esmm_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_tag_kv_input(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/kv_tag.config', self._test_dir) + 'samples/model_config/kv_tag.config', self._test_dir + ) self.assertTrue(self._success) def test_aitm(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/aitm_on_taobao.config', self._test_dir) + 'samples/model_config/aitm_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_taobao.config', self._test_dir) + 'samples/model_config/dbmtl_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_dbmtl_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_backbone_on_taobao.config', self._test_dir) + 'samples/model_config/dbmtl_backbone_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_dbmtl_cmbf(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_cmbf_on_movielens.config', self._test_dir) + 'samples/model_config/dbmtl_cmbf_on_movielens.config', self._test_dir + ) self.assertTrue(self._success) def test_dbmtl_uniter(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_uniter_on_movielens.config', self._test_dir) + 'samples/model_config/dbmtl_uniter_on_movielens.config', self._test_dir + ) self.assertTrue(self._success) def test_dbmtl_with_multi_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_taobao_with_multi_loss.config', - self._test_dir) + 'samples/model_config/dbmtl_on_taobao_with_multi_loss.config', + self._test_dir + ) self.assertTrue(self._success) def test_early_stop(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_early_stop_on_taobao.config', - self._test_dir) + 'samples/model_config/multi_tower_early_stop_on_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_early_stop_custom(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/custom_early_stop_on_taobao.config', - self._test_dir) + 'samples/model_config/custom_early_stop_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_early_stop_dis(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_early_stop_on_taobao.config', - self._test_dir) + 'samples/model_config/multi_tower_early_stop_on_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_latest_export_with_asset(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/din_on_taobao_latest_export.config', - self._test_dir) + 'samples/model_config/din_on_taobao_latest_export.config', self._test_dir + ) self.assertTrue(self._success) def test_incompatible_restore(self): @@ -729,569 +824,663 @@ def _post_check_func(config): config.model_dir += '_finetune' config.train_config.force_restore_shape_compatible = True return test_utils.test_single_train_eval( - config, os.path.join(self._test_dir, 'finetune')) + config, os.path.join(self._test_dir, 'finetune') + ) self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.config', - self._test_dir, - post_check_func=_post_check_func) + 'samples/model_config/taobao_fg.config', + self._test_dir, + post_check_func=_post_check_func + ) self.assertTrue(self._success) def test_dbmtl_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_variational_dropout.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection) + 'samples/model_config/dbmtl_variational_dropout.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection + ) self.assertTrue(self._success) def test_dbmtl_variational_dropout_feature_num(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_variational_dropout_feature_num.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection) + 'samples/model_config/dbmtl_variational_dropout_feature_num.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection + ) self.assertTrue(self._success) def test_essm_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/esmm_variational_dropout_on_taobao.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection) + 'samples/model_config/esmm_variational_dropout_on_taobao.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection + ) self.assertTrue(self._success) def test_fm_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/fm_variational_dropout_on_taobao.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection) + 'samples/model_config/fm_variational_dropout_on_taobao.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection + ) self.assertTrue(self._success) def test_deepfm_with_combo_feature_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_variational_dropout_on_avazu_ctr.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection) + 'samples/model_config/deepfm_combo_variational_dropout_on_avazu_ctr.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection + ) self.assertTrue(self._success) def test_dbmtl_sequence_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_variational_dropout_on_sequence_feature_taobao.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection) + 'samples/model_config/dbmtl_variational_dropout_on_sequence_feature_taobao.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection + ) self.assertTrue(self._success) def test_din_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/din_varitional_dropout_on_taobao.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection) + 'samples/model_config/din_varitional_dropout_on_taobao.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection + ) self.assertTrue(self._success) def test_rocket_launching(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/rocket_launching.config', self._test_dir) + 'samples/model_config/rocket_launching.config', self._test_dir + ) self.assertTrue(self._success) def test_rocket_launching_feature_based(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/rocket_launching_feature_based.config', - self._test_dir) + 'samples/model_config/rocket_launching_feature_based.config', + self._test_dir + ) self.assertTrue(self._success) def test_rocket_launching_with_rtp_input(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/rocket_launching_with_rtp_input.config', - self._test_dir) + 'samples/model_config/rocket_launching_with_rtp_input.config', + self._test_dir + ) self.assertTrue(self._success) def test_dbmtl_mmoe(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_mmoe_on_taobao.config', self._test_dir) + 'samples/model_config/dbmtl_mmoe_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) def test_train_with_ps_worker(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao.config', self._test_dir) + 'samples/model_config/multi_tower_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) @unittest.skip('Timeout on CI machine') def test_fit_on_eval(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao.config', - self._test_dir, - total_steps=10, - num_evaluator=1, - fit_on_eval=True) + 'samples/model_config/multi_tower_on_taobao.config', + self._test_dir, + total_steps=10, + num_evaluator=1, + fit_on_eval=True + ) self.assertTrue(self._success) def test_unbalance_data(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao_unblanace.config', - self._test_dir, - total_steps=0, - num_epoch=1, - num_evaluator=1) + 'samples/model_config/multi_tower_on_taobao_unblanace.config', + self._test_dir, + total_steps=0, + num_epoch=1, + num_evaluator=1 + ) self.assertTrue(self._success) def test_train_with_ps_worker_with_evaluator(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao.config', - self._test_dir, - num_evaluator=1) + 'samples/model_config/multi_tower_on_taobao.config', + self._test_dir, + num_evaluator=1 + ) self.assertTrue(self._success) final_export_dir = os.path.join(self._test_dir, 'train/export/final') all_saved_files = glob.glob(final_export_dir + '/*/saved_model.pb') - logging.info('final_export_dir=%s all_saved_files=%s' % - (final_export_dir, ','.join(all_saved_files))) + logging.info( + 'final_export_dir=%s all_saved_files=%s' % + (final_export_dir, ','.join(all_saved_files)) + ) self.assertTrue(len(all_saved_files) == 1) def test_train_with_ps_worker_chief_redundant(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao_chief_redundant.config', - self._test_dir) + 'samples/model_config/multi_tower_on_taobao_chief_redundant.config', + self._test_dir + ) self.assertTrue(self._success) def test_deepfm_embed_input(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_with_embed.config', self._test_dir) + 'samples/model_config/deepfm_with_embed.config', self._test_dir + ) self.assertTrue(self._success) def test_multi_tower_embed_input(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_with_embed.config', self._test_dir) + 'samples/model_config/multi_tower_with_embed.config', self._test_dir + ) self.assertTrue(self._success) def test_tfrecord_input(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_on_criteo_tfrecord.config', self._test_dir) + 'samples/model_config/deepfm_on_criteo_tfrecord.config', self._test_dir + ) self.assertTrue(self._success) def test_batch_tfrecord_input(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_on_criteo_batch_tfrecord.config', - self._test_dir) + 'samples/model_config/deepfm_on_criteo_batch_tfrecord.config', + self._test_dir + ) self.assertTrue(self._success) def test_autodis_embedding(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_on_criteo_with_autodis.config', - self._test_dir) + 'samples/model_config/deepfm_on_criteo_with_autodis.config', + self._test_dir + ) self.assertTrue(self._success) def test_periodic_embedding(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_on_criteo_with_periodic.config', - self._test_dir) + 'samples/model_config/deepfm_on_criteo_with_periodic.config', + self._test_dir + ) self.assertTrue(self._success) def test_sample_weight(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_with_sample_weight.config', self._test_dir) + 'samples/model_config/deepfm_with_sample_weight.config', self._test_dir + ) self.assertTrue(self._success) def test_dssm_sample_weight(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_with_sample_weight.config', self._test_dir) + 'samples/model_config/dssm_with_sample_weight.config', self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler_with_sample_weight(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_neg_sampler_with_sample_weight.config', - self._test_dir) + 'samples/model_config/dssm_neg_sampler_with_sample_weight.config', + self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf( - LooseVersion(tf.__version__) != LooseVersion('2.3.0'), - 'MultiWorkerMirroredStrategy need tf version == 2.3') + LooseVersion(tf.__version__) != LooseVersion('2.3.0'), + 'MultiWorkerMirroredStrategy need tf version == 2.3' + ) def test_train_with_multi_worker_mirror(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_multi_worker_mirrored_strategy_on_taobao.config', - self._test_dir) + 'samples/model_config/multi_tower_multi_worker_mirrored_strategy_on_taobao.config', + self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf( - LooseVersion(tf.__version__) != LooseVersion('2.3.0'), - 'MultiWorkerMirroredStrategy need tf version == 2.3') + LooseVersion(tf.__version__) != LooseVersion('2.3.0'), + 'MultiWorkerMirroredStrategy need tf version == 2.3' + ) def test_train_mmoe_with_multi_worker_mirror(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/mmoe_mirrored_strategy_on_taobao.config', - self._test_dir) + 'samples/model_config/mmoe_mirrored_strategy_on_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_fg_dtype(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg_test_dtype.config', self._test_dir) + 'samples/model_config/taobao_fg_test_dtype.config', self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(six.PY2, 'Only run in python3') def test_share_not_used(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/share_not_used.config', self._test_dir) + 'samples/model_config/share_not_used.config', self._test_dir + ) self.assertTrue(self._success) def test_sequence_autoint(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/autoint_on_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/autoint_on_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_sequence_dcn(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dcn_on_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dcn_on_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_sequence_dssm(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_on_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dssm_on_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_sequence_esmm(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/esmm_on_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/esmm_on_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_sequence_mmoe(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mmoe_on_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/mmoe_on_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_sequence_ple(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/ple_on_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/ple_on_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_sequence_rocket_launching(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/rocket_launching_on_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/rocket_launching_on_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_sequence_simple_multi_task(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/simple_multi_task_on_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/simple_multi_task_on_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_sequence_wide_and_deep(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/wide_and_deep_on_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/wide_and_deep_on_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_numeric_boundary_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_numeric_hash_bucket_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_numeric_hash_bucket_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_numeric_hash_bucket_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_numeric_raw_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_numeric_raw_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_numeric_raw_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_numeric_num_buckets_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_numeric_num_buckets_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_numeric_num_buckets_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_multi_numeric_boundary_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_boundary_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_multi_numeric_boundary_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_multi_numeric_hash_bucket_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_hash_bucket_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_multi_numeric_hash_bucket_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_multi_numeric_raw_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_raw_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_multi_numeric_raw_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_multi_numeric_num_buckets_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_num_buckets_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_multi_numeric_num_buckets_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_multi_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_sequence_feature_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_multi_sequence_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_multi_optimizer(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/wide_and_deep_two_opti.config', self._test_dir) + 'samples/model_config/wide_and_deep_two_opti.config', self._test_dir + ) self.assertTrue(self._success) def test_embedding_separate_optimizer(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_embed_adagrad.config', - self._test_dir) + 'samples/model_config/deepfm_combo_on_avazu_embed_adagrad.config', + self._test_dir + ) self.assertTrue(self._success) def test_expr_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_on_taobao_for_expr.config', - self._test_dir) + 'samples/model_config/multi_tower_on_taobao_for_expr.config', + self._test_dir + ) self.assertTrue(self._success) def test_gzip_data(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/din_on_gzip_data.config', self._test_dir) + 'samples/model_config/din_on_gzip_data.config', self._test_dir + ) self.assertTrue(self._success) def test_cmd_config_param(self): def _post_check_config(pipeline_config): - train_saved_config_path = os.path.join(self._test_dir, - 'train/pipeline.config') + train_saved_config_path = os.path.join( + self._test_dir, 'train/pipeline.config' + ) pipeline_config = config_util.get_configs_from_pipeline_file( - train_saved_config_path) + train_saved_config_path + ) assert pipeline_config.model_config.deepfm.wide_output_dim == 8,\ 'invalid model_config.deepfm.wide_output_dim=%d' % \ pipeline_config.model_config.deepfm.wide_output_dim self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - self._test_dir, - post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim 8') + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + self._test_dir, + post_check_func=_post_check_config, + extra_cmd_args='--model_config.deepfm.wide_output_dim 8' + ) def test_cmd_config_param_v2(self): def _post_check_config(pipeline_config): - train_saved_config_path = os.path.join(self._test_dir, - 'train/pipeline.config') + train_saved_config_path = os.path.join( + self._test_dir, 'train/pipeline.config' + ) pipeline_config = config_util.get_configs_from_pipeline_file( - train_saved_config_path) + train_saved_config_path + ) assert pipeline_config.model_config.deepfm.wide_output_dim == 1,\ 'invalid model_config.deepfm.wide_output_dim=%d' % \ pipeline_config.model_config.deepfm.wide_output_dim self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - self._test_dir, - post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim=1') + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + self._test_dir, + post_check_func=_post_check_config, + extra_cmd_args='--model_config.deepfm.wide_output_dim=1' + ) def test_cmd_config_param_v3(self): def _post_check_config(pipeline_config): - train_saved_config_path = os.path.join(self._test_dir, - 'train/pipeline.config') + train_saved_config_path = os.path.join( + self._test_dir, 'train/pipeline.config' + ) pipeline_config = config_util.get_configs_from_pipeline_file( - train_saved_config_path) + train_saved_config_path + ) assert pipeline_config.model_config.deepfm.wide_output_dim == 3,\ 'invalid model_config.deepfm.wide_output_dim=%d' % \ pipeline_config.model_config.deepfm.wide_output_dim self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - self._test_dir, - post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim="3"') + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + self._test_dir, + post_check_func=_post_check_config, + extra_cmd_args='--model_config.deepfm.wide_output_dim="3"' + ) def test_distribute_eval_deepfm_multi_cls(self): cur_eval_path = 'data/test/distribute_eval_test/deepfm_distribute_eval_dwd_avazu_out_multi_cls' self._success = test_utils.test_distributed_eval( - 'samples/model_config/deepfm_distribute_eval_multi_cls_on_avazu_ctr.config', - cur_eval_path, self._test_dir) + 'samples/model_config/deepfm_distribute_eval_multi_cls_on_avazu_ctr.config', + cur_eval_path, self._test_dir + ) self.assertTrue(self._success) def test_distribute_eval_deepfm_single_cls(self): cur_eval_path = 'data/test/distribute_eval_test/dwd_distribute_eval_avazu_out_test_combo' self._success = test_utils.test_distributed_eval( - 'samples/model_config/deepfm_distribute_eval_combo_on_avazu_ctr.config', - cur_eval_path, self._test_dir) + 'samples/model_config/deepfm_distribute_eval_combo_on_avazu_ctr.config', + cur_eval_path, self._test_dir + ) self.assertTrue(self._success) def test_distribute_eval_dssm_pointwise_classification(self): cur_eval_path = 'data/test/distribute_eval_test/dssm_distribute_eval_pointwise_classification_taobao_ckpt' self._success = test_utils.test_distributed_eval( - 'samples/model_config/dssm_distribute_eval_pointwise_classification_on_taobao.config', - cur_eval_path, self._test_dir) + 'samples/model_config/dssm_distribute_eval_pointwise_classification_on_taobao.config', + cur_eval_path, self._test_dir + ) self.assertTrue(self._success) def test_distribute_eval_dssm_reg(self): cur_eval_path = 'data/test/distribute_eval_test/dssm_distribute_eval_reg_taobao_ckpt' self._success = test_utils.test_distributed_eval( - 'samples/model_config/dssm_distribute_eval_reg_on_taobao.config', - cur_eval_path, self._test_dir) + 'samples/model_config/dssm_distribute_eval_reg_on_taobao.config', + cur_eval_path, self._test_dir + ) self.assertTrue(self._success) def test_distribute_eval_dropout(self): cur_eval_path = 'data/test/distribute_eval_test/dropoutnet_distribute_eval_taobao_ckpt' self._success = test_utils.test_distributed_eval( - 'samples/model_config/dropoutnet_distribute_eval_on_taobao.config', - cur_eval_path, self._test_dir) + 'samples/model_config/dropoutnet_distribute_eval_on_taobao.config', + cur_eval_path, self._test_dir + ) self.assertTrue(self._success) def test_distribute_eval_esmm(self): cur_eval_path = 'data/test/distribute_eval_test/esmm_distribute_eval_taobao_ckpt' self._success = test_utils.test_distributed_eval( - 'samples/model_config/esmm_distribute_eval_on_taobao.config', - cur_eval_path, self._test_dir) + 'samples/model_config/esmm_distribute_eval_on_taobao.config', + cur_eval_path, self._test_dir + ) self.assertTrue(self._success) def test_share_no_used(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/share_embedding_not_used.config', self._test_dir) + 'samples/model_config/share_embedding_not_used.config', self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler_sequence_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_neg_sampler_sequence_feature.config', - self._test_dir) + 'samples/model_config/dssm_neg_sampler_sequence_feature.config', + self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler_need_key_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_neg_sampler_need_key_feature.config', - self._test_dir) + 'samples/model_config/dssm_neg_sampler_need_key_feature.config', + self._test_dir + ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_need_key_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_boundary_need_key_feature_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_multi_numeric_boundary_need_key_feature_taobao.config', + self._test_dir + ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_allow_key_transform(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_boundary_allow_key_transform.config', - self._test_dir) + 'samples/model_config/dbmtl_on_multi_numeric_boundary_allow_key_transform.config', + self._test_dir + ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_aux_hist_seq(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_aux_hist_seq_taobao.config', - self._test_dir) + 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_aux_hist_seq_taobao.config', + self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_multi_tower_recall_neg_sampler_sequence_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_recall_neg_sampler_sequence_feature.config', - self._test_dir) + 'samples/model_config/multi_tower_recall_neg_sampler_sequence_feature.config', + self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_multi_tower_recall_neg_sampler_only_sequence_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_recall_neg_sampler_only_sequence_feature.config', - self._test_dir) + 'samples/model_config/multi_tower_recall_neg_sampler_only_sequence_feature.config', + self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(hvd is None, 'horovod is not installed') def test_horovod(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_ctr.config', - self._test_dir, - use_hvd=True) + 'samples/model_config/deepfm_combo_on_avazu_ctr.config', + self._test_dir, + use_hvd=True + ) self.assertTrue(self._success) - @unittest.skipIf(hvd is None or sok is None, - 'horovod and sok is not installed') + @unittest.skipIf( + hvd is None or sok is None, 'horovod and sok is not installed' + ) def test_sok(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao_sok.config', - self._test_dir, - use_hvd=True) + 'samples/model_config/multi_tower_on_taobao_sok.config', + self._test_dir, + use_hvd=True + ) self.assertTrue(self._success) @unittest.skipIf( - six.PY2 or tf_version.split('.')[0] != '2', - 'only run on python3 and tf 2.x') + six.PY2 or tf_version.split('.')[0] != '2', + 'only run on python3 and tf 2.x' + ) def test_train_parquet(self): os.environ[constant.NO_ARITHMETRIC_OPTI] = '1' self._success = test_utils.test_single_train_eval( - 'samples/model_config/dlrm_on_criteo_parquet.config', self._test_dir) + 'samples/model_config/dlrm_on_criteo_parquet.config', self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(hvd is None, 'horovod is not installed') def test_train_parquet_embedding_parallel(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/dlrm_on_criteo_parquet_ep.config', - self._test_dir, - use_hvd=True) + 'samples/model_config/dlrm_on_criteo_parquet_ep.config', + self._test_dir, + use_hvd=True + ) self.assertTrue(self._success) @unittest.skipIf(hvd is None, 'horovod is not installed') def test_train_parquet_embedding_parallel_v2(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/dlrm_on_criteo_parquet_ep_v2.config', - self._test_dir, - use_hvd=True) + 'samples/model_config/dlrm_on_criteo_parquet_ep_v2.config', + self._test_dir, + use_hvd=True + ) self.assertTrue(self._success) def test_pdn(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/pdn_on_taobao.config', self._test_dir) + 'samples/model_config/pdn_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_senet(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_senet_on_taobao.config', self._test_dir) + 'samples/model_config/dssm_senet_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_backbone_on_taobao(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_on_taobao_backbone.config', self._test_dir) + 'samples/model_config/dssm_on_taobao_backbone.config', self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_senet_backbone_on_taobao(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_senet_on_taobao_backbone.config', - self._test_dir) + 'samples/model_config/dssm_senet_on_taobao_backbone.config', + self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_parallel_dssm_backbone_on_taobao(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/parallel_dssm_on_taobao_backbone.config', - self._test_dir) + 'samples/model_config/parallel_dssm_on_taobao_backbone.config', + self._test_dir + ) self.assertTrue(self._success) def test_xdeefm_backbone_on_taobao(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/xdeepfm_on_taobao_backbone.config', - self._test_dir) + 'samples/model_config/xdeepfm_on_taobao_backbone.config', self._test_dir + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dat_on_taobao(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dat_on_taobao.config', self._test_dir) + 'samples/model_config/dat_on_taobao.config', self._test_dir + ) self.assertTrue(self._success) diff --git a/easy_rec/python/test/util_test.py b/easy_rec/python/test/util_test.py index c14524488..90788d4e8 100644 --- a/easy_rec/python/test/util_test.py +++ b/easy_rec/python/test/util_test.py @@ -16,46 +16,55 @@ class UtilTest(tf.test.TestCase): def test_get_ckpt_version(self): ver = estimator_utils.get_ckpt_version( - 'oss://easyrec/ckpts/model.ckpt-6500.meta') + 'oss://easyrec/ckpts/model.ckpt-6500.meta' + ) assert ver == 6500, 'invalid version: %s' % str(ver) ver = estimator_utils.get_ckpt_version( - 'oss://easyrec/ckpts/model.ckpt-6500') + 'oss://easyrec/ckpts/model.ckpt-6500' + ) assert ver == 6500, 'invalid version: %s' % str(ver) def test_get_expression_greater(self): - result = get_expression('age_level>item_age_level', - ['age_level', 'item_age_level']) + result = get_expression( + 'age_level>item_age_level', ['age_level', 'item_age_level'] + ) assert result == "tf.greater(parsed_dict['age_level'], parsed_dict['item_age_level'])" def test_get_expression_greater_equal(self): - result = get_expression('age_level>=item_age_level', - ['age_level', 'item_age_level']) + result = get_expression( + 'age_level>=item_age_level', ['age_level', 'item_age_level'] + ) assert result == "tf.greater_equal(parsed_dict['age_level'], parsed_dict['item_age_level'])" def test_get_expression_less(self): - result = get_expression('age_level3)&(item_age_level<1)', - ['age_level', 'item_age_level']) + result = get_expression( + '(age_level>3)&(item_age_level<1)', ['age_level', 'item_age_level'] + ) assert result == "tf.greater(parsed_dict['age_level'], 3) & tf.less(parsed_dict['item_age_level'], 1)" result = get_expression( - '(age_level>item_age_level) & (age_levelitem_age_level) & (age_level3)|(item_age_level<1)', - ['age_level', 'item_age_level']) + result = get_expression( + '(age_level>3)|(item_age_level<1)', ['age_level', 'item_age_level'] + ) assert result == "tf.greater(parsed_dict['age_level'], 3) | tf.less(parsed_dict['item_age_level'], 1)" def test_dag(self): diff --git a/easy_rec/python/test/zero_inflated_lognormal_test.py b/easy_rec/python/test/zero_inflated_lognormal_test.py index 627db542d..1a495231f 100644 --- a/easy_rec/python/test/zero_inflated_lognormal_test.py +++ b/easy_rec/python/test/zero_inflated_lognormal_test.py @@ -41,10 +41,11 @@ def zero_inflated_lognormal(self, labels, logits, max_sigma=5.0): if np.any(mask): # scipy 参数化：s=shape= sigma, scale=exp(mu), loc=0 logprob[mask, 0] = stats.lognorm.logpdf( - x=labels[mask, 0].astype(np.float64), - s=sigma[mask, 0].astype(np.float64), - loc=0.0, - scale=np.exp(mu[mask, 0].astype(np.float64))) + x=labels[mask, 0].astype(np.float64), + s=sigma[mask, 0].astype(np.float64), + loc=0.0, + scale=np.exp(mu[mask, 0].astype(np.float64)) + ) num_pos = np.sum(positive) + 1e-8 regression_loss = -(np.sum(positive * logprob) / num_pos) # 与实现一致的组合（此处测试会把正则项设为0） @@ -55,7 +56,8 @@ def test_loss_value(self): expected_loss = self.zero_inflated_lognormal(self.labels, self.logits) expected_loss = np.average(expected_loss) loss = zero_inflated_lognormal_loss( - self.labels, self.logits, mu_reg=0, sigma_reg=0) + self.labels, self.logits, mu_reg=0, sigma_reg=0 + ) # Absolute error tolerance in asserting array near. _ERR_TOL = 1e-6 self.assertNear(self.evaluate(loss), expected_loss, _ERR_TOL) diff --git a/easy_rec/python/tools/add_boundaries_to_config.py b/easy_rec/python/tools/add_boundaries_to_config.py index 18d5f6037..7e34300d9 100644 --- a/easy_rec/python/tools/add_boundaries_to_config.py +++ b/easy_rec/python/tools/add_boundaries_to_config.py @@ -1,28 +1,29 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. +import common_io import json import logging import os import sys - -import common_io import tensorflow as tf -from easy_rec.python.utils import config_util -from easy_rec.python.utils import io_util +from easy_rec.python.utils import config_util, io_util if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) -tf.app.flags.DEFINE_string('template_config_path', None, - 'Path to template pipeline config ' - 'file.') -tf.app.flags.DEFINE_string('output_config_path', None, - 'Path to output pipeline config ' - 'file.') + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) +tf.app.flags.DEFINE_string( + 'template_config_path', None, 'Path to template pipeline config ' + 'file.' +) +tf.app.flags.DEFINE_string( + 'output_config_path', None, 'Path to output pipeline config ' + 'file.' +) tf.app.flags.DEFINE_string('tables', '', 'quantile binning table') FLAGS = tf.app.flags.FLAGS @@ -30,11 +31,13 @@ def main(argv): pipeline_config = config_util.get_configs_from_pipeline_file( - FLAGS.template_config_path) + FLAGS.template_config_path + ) feature_boundaries_info = {} reader = common_io.table.TableReader( - FLAGS.tables, selected_cols='feature,json') + FLAGS.tables, selected_cols='feature,json' + ) while True: try: record = reader.read() diff --git a/easy_rec/python/tools/add_feature_info_to_config.py b/easy_rec/python/tools/add_feature_info_to_config.py index 7594d038b..d972da0c4 100644 --- a/easy_rec/python/tools/add_feature_info_to_config.py +++ b/easy_rec/python/tools/add_feature_info_to_config.py @@ -4,25 +4,26 @@ import logging import os import sys - import tensorflow as tf -from easy_rec.python.utils import config_util -from easy_rec.python.utils import io_util +from easy_rec.python.utils import config_util, io_util from easy_rec.python.utils.hive_utils import HiveUtils if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) -tf.app.flags.DEFINE_string('template_config_path', None, - 'Path to template pipeline config ' - 'file.') -tf.app.flags.DEFINE_string('output_config_path', None, - 'Path to output pipeline config ' - 'file.') + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) +tf.app.flags.DEFINE_string( + 'template_config_path', None, 'Path to template pipeline config ' + 'file.' +) +tf.app.flags.DEFINE_string( + 'output_config_path', None, 'Path to output pipeline config ' + 'file.' +) tf.app.flags.DEFINE_string('config_table', '', 'config table') FLAGS = tf.app.flags.FLAGS @@ -30,17 +31,19 @@ def main(argv): pipeline_config = config_util.get_configs_from_pipeline_file( - FLAGS.template_config_path) + FLAGS.template_config_path + ) sels = 'feature,feature_info,message' feature_info_map = {} drop_feature_names = [] if pipeline_config.WhichOneof('train_path') == 'hive_train_input': hive_util = HiveUtils( - data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input, - selected_cols=sels, - record_defaults=['', '', '']) + data_config=pipeline_config.data_config, + hive_config=pipeline_config.hive_train_input, + selected_cols=sels, + record_defaults=['', '', ''] + ) reader = hive_util.hive_read_line(FLAGS.config_table) for record in reader: feature_name = record[0][0] @@ -50,7 +53,9 @@ def main(argv): else: import common_io - reader = common_io.table.TableReader(FLAGS.config_table, selected_cols=sels) + reader = common_io.table.TableReader( + FLAGS.config_table, selected_cols=sels + ) while True: try: record = reader.read() @@ -74,23 +79,28 @@ def main(argv): if feature_name in feature_info_map: logging.info('edited %s' % feature_name) feature_config.embedding_dim = int( - feature_info_map[feature_name]['embedding_dim']) + feature_info_map[feature_name]['embedding_dim'] + ) logging.info('modify embedding_dim to %s' % feature_config.embedding_dim) if 'boundary' in feature_info_map[feature_name]: feature_config.ClearField('boundaries') feature_config.boundaries.extend( - [float(i) for i in feature_info_map[feature_name]['boundary']]) + [float(i) for i in feature_info_map[feature_name]['boundary']] + ) logging.info('modify boundaries to %s' % feature_config.boundaries) elif 'hash_bucket_size' in feature_info_map[feature_name]: feature_config.hash_bucket_size = int( - feature_info_map[feature_name]['hash_bucket_size']) - logging.info('modify hash_bucket_size to %s' % - feature_config.hash_bucket_size) + feature_info_map[feature_name]['hash_bucket_size'] + ) + logging.info( + 'modify hash_bucket_size to %s' % feature_config.hash_bucket_size + ) # modify num_steps pipeline_config.train_config.num_steps = feature_info_map['__NUM_STEPS__'][ - 'num_steps'] - logging.info('modify num_steps to %s' % - pipeline_config.train_config.num_steps) + 'num_steps'] + logging.info( + 'modify num_steps to %s' % pipeline_config.train_config.num_steps + ) # modify decay_steps optimizer_configs = pipeline_config.train_config.optimizer_config for optimizer_config in optimizer_configs: @@ -100,7 +110,7 @@ def main(argv): learning_rate = getattr(optimizer.learning_rate, learning_rate) if hasattr(learning_rate, 'decay_steps'): learning_rate.decay_steps = feature_info_map['__DECAY_STEPS__'][ - 'decay_steps'] + 'decay_steps'] logging.info('modify decay_steps to %s' % learning_rate.decay_steps) for feature_group in pipeline_config.model_config.feature_groups: diff --git a/easy_rec/python/tools/convert_config_format.py b/easy_rec/python/tools/convert_config_format.py index 5cfda1219..f8691c5df 100644 --- a/easy_rec/python/tools/convert_config_format.py +++ b/easy_rec/python/tools/convert_config_format.py @@ -1,9 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import os - -from google.protobuf import json_format -from google.protobuf import text_format +from google.protobuf import json_format, text_format from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig @@ -27,8 +25,10 @@ def save_config(pipeline_config, save_path): fout.write(text_format.MessageToString(pipeline_config, as_utf8=True)) elif save_path.endswith('.json'): fout.write( - json_format.MessageToJson( - pipeline_config, preserving_proto_field_name=True)) + json_format.MessageToJson( + pipeline_config, preserving_proto_field_name=True + ) + ) else: assert False, 'only .config/.json are supported(%s)' % save_path @@ -38,9 +38,11 @@ def save_config(pipeline_config, save_path): parser = argparse.ArgumentParser() parser.add_argument( - '--input_config', type=str, help='input_config path', default=None) + '--input_config', type=str, help='input_config path', default=None + ) parser.add_argument( - '--output_config', type=str, help='output_config path', default=None) + '--output_config', type=str, help='output_config path', default=None + ) args = parser.parse_args() assert os.path.exists(args.input_config) diff --git a/easy_rec/python/tools/convert_rtp_data.py b/easy_rec/python/tools/convert_rtp_data.py index 18455bd9c..a29f9752b 100644 --- a/easy_rec/python/tools/convert_rtp_data.py +++ b/easy_rec/python/tools/convert_rtp_data.py @@ -14,12 +14,12 @@ import json import logging import sys - import tensorflow as tf logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -27,9 +27,12 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--rtp_fg', type=str, default='', help='rtp fg path(.json)') + '--rtp_fg', type=str, default='', help='rtp fg path(.json)' + ) parser.add_argument('--input_path', type=str, default='', help='input path') - parser.add_argument('--output_path', type=str, default='', help='output path') + parser.add_argument( + '--output_path', type=str, default='', help='output path' + ) parser.add_argument('--label', type=str, default='', help='label for train') args = parser.parse_args() diff --git a/easy_rec/python/tools/convert_rtp_fg.py b/easy_rec/python/tools/convert_rtp_fg.py index bd090e0a3..91890f5f0 100644 --- a/easy_rec/python/tools/convert_rtp_fg.py +++ b/easy_rec/python/tools/convert_rtp_fg.py @@ -4,15 +4,15 @@ import argparse import logging import sys - import tensorflow as tf from easy_rec.python.utils.config_util import save_message from easy_rec.python.utils.convert_rtp_fg import convert_rtp_fg logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -21,60 +21,73 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--model_type', - type=str, - choices=model_types, - default='', - help='model type, currently support: %s' % ','.join(model_types)) + '--model_type', + type=str, + choices=model_types, + default='', + help='model type, currently support: %s' % ','.join(model_types) + ) parser.add_argument('--rtp_fg', type=str, help='rtp fg path') parser.add_argument( - '--embedding_dim', type=int, default=16, help='embedding_dimension') + '--embedding_dim', type=int, default=16, help='embedding_dimension' + ) parser.add_argument( - '--batch_size', type=int, default=1024, help='batch_size for train') + '--batch_size', type=int, default=1024, help='batch_size for train' + ) parser.add_argument( - '--label', - type=str, - default='', - nargs='+', - required=True, - help='label fields') + '--label', + type=str, + default='', + nargs='+', + required=True, + help='label fields' + ) parser.add_argument( - '--num_steps', - type=int, - default=1000, - help='number of train steps = num_samples * num_epochs / batch_size / num_workers' + '--num_steps', + type=int, + default=1000, + help= + 'number of train steps = num_samples * num_epochs / batch_size / num_workers' ) parser.add_argument('--output_path', type=str, help='generated config path') parser.add_argument( - '--incol_separator', - type=str, - default='\003', - help='separator for multi_value features') + '--incol_separator', + type=str, + default='\003', + help='separator for multi_value features' + ) parser.add_argument( - '--separator', - type=str, - default='\002', - help='separator between different features') + '--separator', + type=str, + default='\002', + help='separator between different features' + ) parser.add_argument( - '--train_input_path', type=str, default=None, help='train data path') + '--train_input_path', type=str, default=None, help='train data path' + ) parser.add_argument( - '--eval_input_path', type=str, default=None, help='eval data path') + '--eval_input_path', type=str, default=None, help='eval data path' + ) parser.add_argument( - '--selected_cols', - type=str, - default=None, - help='selected cols, for csv input, it is in the format of: label_col_id0,...,lable_cold_idn,feature_col_id ' - 'for odps table input, it is in the format of: label_col_name0,...,label_col_namen,feature_col_name ' + '--selected_cols', + type=str, + default=None, + help= + 'selected cols, for csv input, it is in the format of: label_col_id0,...,lable_cold_idn,feature_col_id ' + 'for odps table input, it is in the format of: label_col_name0,...,label_col_namen,feature_col_name ' ) parser.add_argument( - '--rtp_separator', type=str, default=';', help='separator') + '--rtp_separator', type=str, default=';', help='separator' + ) parser.add_argument( - '--input_type', - type=str, - default='OdpsRTPInput', - help='default to OdpsRTPInput, if test local, change it to RTPInput') + '--input_type', + type=str, + default='OdpsRTPInput', + help='default to OdpsRTPInput, if test local, change it to RTPInput' + ) parser.add_argument( - '--is_async', action='store_true', help='async mode, debug to false') + '--is_async', action='store_true', help='async mode, debug to false' + ) args = parser.parse_args() @@ -86,21 +99,24 @@ logging.error('output_path is not set') sys.exit(1) - pipeline_config = convert_rtp_fg(args.rtp_fg, args.embedding_dim, - args.batch_size, args.label, args.num_steps, - args.model_type, args.separator, - args.incol_separator, args.train_input_path, - args.eval_input_path, args.selected_cols, - args.input_type, args.is_async) + pipeline_config = convert_rtp_fg( + args.rtp_fg, args.embedding_dim, args.batch_size, args.label, + args.num_steps, args.model_type, args.separator, args.incol_separator, + args.train_input_path, args.eval_input_path, args.selected_cols, + args.input_type, args.is_async + ) save_message(pipeline_config, args.output_path) logging.info('Conversion done.') logging.info('Tips:') logging.info( - 'if run on local, please change data_config.input_type to RTPInput, ' - 'and model_dir/train_input_path/eval_input_path must also be set, ') + 'if run on local, please change data_config.input_type to RTPInput, ' + 'and model_dir/train_input_path/eval_input_path must also be set, ' + ) logging.info( - 'if run local, please set data_config.selected_cols in the format ' - 'label_col_id0,label_col_id1,...,label_col_idn,feature_col_id') + 'if run local, please set data_config.selected_cols in the format ' + 'label_col_id0,label_col_id1,...,label_col_idn,feature_col_id' + ) logging.info( - 'if run on odps, selected_cols must be set, which are label0_col,' - 'label1_col, ..., feature_col_name') + 'if run on odps, selected_cols must be set, which are label0_col,' + 'label1_col, ..., feature_col_name' + ) diff --git a/easy_rec/python/tools/create_config_from_excel.py b/easy_rec/python/tools/create_config_from_excel.py index c0f3e0a2b..b14d9c32a 100644 --- a/easy_rec/python/tools/create_config_from_excel.py +++ b/easy_rec/python/tools/create_config_from_excel.py @@ -2,21 +2,23 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import math -import sys - import numpy as np import pandas as pd +import sys from easy_rec.python.utils import config_util logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) class ModelConfigConverter: - def __init__(self, excel_path, output_path, model_type, column_separator, - incol_separator, train_input_path, eval_input_path, model_dir): + def __init__( + self, excel_path, output_path, model_type, column_separator, + incol_separator, train_input_path, eval_input_path, model_dir + ): self._excel_path = excel_path self._output_path = output_path self._model_type = model_type @@ -32,25 +34,27 @@ def __init__(self, excel_path, output_path, model_type, column_separator, self._model_dir = model_dir if not self._model_dir: self._model_dir = 'experiments/demo' - logging.warning('model_dir is not specified, set to %s' % self._model_dir) + logging.warning( + 'model_dir is not specified, set to %s' % self._model_dir + ) def _get_type_name(self, input_name): type_dict = { - 'bigint': 'INT64', - 'double': 'DOUBLE', - 'float': 'FLOAT', - 'string': 'STRING', - 'bool': 'BOOL' + 'bigint': 'INT64', + 'double': 'DOUBLE', + 'float': 'FLOAT', + 'string': 'STRING', + 'bool': 'BOOL' } return type_dict[input_name] def _get_type_default(self, input_name): type_dict = { - 'bigint': '0', - 'double': '0.0', - 'float': '0.0', - 'string': '', - 'bool': 'false' + 'bigint': '0', + 'double': '0.0', + 'float': '0.0', + 'string': '', + 'bool': 'false' } return type_dict[input_name] @@ -80,9 +84,9 @@ def _add_to_tower(self, tower_name, field): tower_names = ['wide', 'deep'] else: raise ValueError( - 'invalid tower_name[%s] for deepfm model, ' - 'only[label, deep, wide, wide_and_deep are supported]' % - tower_name) + 'invalid tower_name[%s] for deepfm model, ' + 'only[label, deep, wide, wide_and_deep are supported]' % tower_name + ) for tower_name in tower_names: if tower_name in self._tower_dicts: self._tower_dicts[tower_name].append(field) @@ -128,19 +132,19 @@ def _is_good(v): return str(v) not in ['nan', ''] if _is_good(self._dict_global[field['global']]['default_value']): - field['default_value'] = self._dict_global[ - field['global']]['default_value'] + field['default_value'] = self._dict_global[field['global'] + ]['default_value'] if _is_good(self._dict_global[field['global']]['hash_bucket_size']): - field['hash_bucket_size'] = self._dict_global[ - field['global']]['hash_bucket_size'] + field['hash_bucket_size'] = self._dict_global[field['global'] + ]['hash_bucket_size'] if _is_good(self._dict_global[field['global']]['embedding_dim']): - field['embedding_dim'] = self._dict_global[ - field['global']]['embedding_dim'] + field['embedding_dim'] = self._dict_global[field['global'] + ]['embedding_dim'] field['embedding_name'] = field['global'] for t in [ - 'type', 'global', 'hash_bucket_size', 'embedding_dim', - 'default_value', 'weights', 'boundaries' + 'type', 'global', 'hash_bucket_size', 'embedding_dim', 'default_value', + 'weights', 'boundaries' ]: if t not in row: continue @@ -179,14 +183,15 @@ def _is_good(v): # check that tag features weights are one of the fields for name, config in self._feature_details.items(): if config['type'] == 'tags': - if 'weights' in config and config[ - 'weights'] not in self._feature_details: + if 'weights' in config and config['weights' + ] not in self._feature_details: raise ValueError(config['weights'] + ' not in field names') def _write_train_eval_config(self, fout): fout.write('train_input_path: "%s"\n' % self._train_input_path) fout.write('eval_input_path: "%s"\n' % self._eval_input_path) - fout.write(""" + fout.write( + """ model_dir: "%s" train_config { @@ -212,7 +217,8 @@ def _write_train_eval_config(self, fout): metrics_set: { auc {} } - }""" % self._model_dir) + }""" % self._model_dir + ) def _write_deepfm_config(self, fout): # write model_config @@ -234,7 +240,8 @@ def _write_deepfm_config(self, fout): fout.write(' }\n') # write deepfm configs - fout.write(""" + fout.write( + """ deepfm { dnn { hidden_units: [128, 64, 32] @@ -247,7 +254,8 @@ def _write_deepfm_config(self, fout): } embedding_regularization: 1e-5 } - """) + """ + ) def _write_multi_tower_config(self, fout): # write model_config @@ -272,22 +280,26 @@ def _write_multi_tower_config(self, fout): fout.write('multi_tower { \n') for tower_name in tower_names: - fout.write(""" + fout.write( + """ towers { input: "%s" dnn { hidden_units: [256, 192, 128] } - }""" % tower_name) + }""" % tower_name + ) - fout.write(""" + fout.write( + """ final_dnn { hidden_units: [192, 128, 64] } l2_regularization: 1e-5 } embedding_regularization: 1e-5 - }""") + }""" + ) def _write_data_config(self, fout): fout.write('data_config {\n') @@ -295,19 +307,25 @@ def _write_data_config(self, fout): for name in self._feature_names: fout.write(' input_fields: {\n') fout.write(' input_name: "%s"\n' % name) - fout.write(' input_type: %s\n' % - self._get_type_name(self._feature_details[name]['data_type'])) + fout.write( + ' input_type: %s\n' % + self._get_type_name(self._feature_details[name]['data_type']) + ) if 'default_value' in self._feature_details[name]: - fout.write(' default_val:"%s"\n' % - self._feature_details[name]['default_value']) + fout.write( + ' default_val:"%s"\n' % + self._feature_details[name]['default_value'] + ) fout.write(' }\n') fout.write(' label_fields: "%s"\n' % self._label) - fout.write(""" + fout.write( + """ batch_size: 1024 prefetch_size: 32 input_type: CSVInput - }""") + }""" + ) def _write_feature_config(self, fout): for name in self._feature_names: @@ -328,10 +346,12 @@ def _write_feature_config(self, fout): fout.write(' feature_type: RawFeature\n') if self._model_type == 'deepfm': assert feature[ - 'boundaries'] != '', 'raw features must be discretized by specifying boundaries' + 'boundaries' + ] != '', 'raw features must be discretized by specifying boundaries' if 'boundaries' in feature and feature['boundaries'] != '': - fout.write(' boundaries: [%s]\n' % - str(feature['boundaries']).strip()) + fout.write( + ' boundaries: [%s]\n' % str(feature['boundaries']).strip() + ) fout.write(' embedding_dim: %d\n' % int(feature['embedding_dim'])) elif feature['type'] == 'tags': if 'weights' in feature: @@ -358,8 +378,9 @@ def _write_feature_config(self, fout): def convert(self): self._parse_features() logging.info( - 'TOWERS[%d]: %s' % - (len(self._tower_dicts), ','.join(list(self._tower_dicts.keys())))) + 'TOWERS[%d]: %s' % + (len(self._tower_dicts), ','.join(list(self._tower_dicts.keys()))) + ) with open(self._output_path, 'w') as fout: self._write_train_eval_config(fout) self._write_data_config(fout) @@ -370,11 +391,12 @@ def convert(self): self._write_multi_tower_config(fout) else: logging.warning( - 'the model_config could not be generated automatically, you have to write the model_config manually.' + 'the model_config could not be generated automatically, you have to write the model_config manually.' ) # reformat the config pipeline_config = config_util.get_configs_from_pipeline_file( - self._output_path) + self._output_path + ) config_util.save_message(pipeline_config, self._output_path) @@ -385,26 +407,31 @@ def convert(self): parser = argparse.ArgumentParser() parser.add_argument( - '--model_type', - type=str, - choices=model_types, - help='model type, currently support: %s' % ','.join(model_types)) + '--model_type', + type=str, + choices=model_types, + help='model type, currently support: %s' % ','.join(model_types) + ) parser.add_argument('--excel_path', type=str, help='excel config path') parser.add_argument('--output_path', type=str, help='generated config path') parser.add_argument( - '--column_separator', - type=str, - default=',', - help='column separator, separator betwen features') + '--column_separator', + type=str, + default=',', + help='column separator, separator betwen features' + ) parser.add_argument( - '--incol_separator', - type=str, - default='|', - help='separator within features, such as tag features') + '--incol_separator', + type=str, + default='|', + help='separator within features, such as tag features' + ) parser.add_argument( - '--train_input_path', type=str, default='', help='train input path') + '--train_input_path', type=str, default='', help='train input path' + ) parser.add_argument( - '--eval_input_path', type=str, default='', help='eval input path') + '--eval_input_path', type=str, default='', help='eval input path' + ) parser.add_argument('--model_dir', type=str, default='', help='model dir') args = parser.parse_args() @@ -412,13 +439,16 @@ def convert(self): parser.print_usage() sys.exit(1) - logging.info('column_separator = %s in_column_separator = %s' % - (args.column_separator, args.incol_separator)) + logging.info( + 'column_separator = %s in_column_separator = %s' % + (args.column_separator, args.incol_separator) + ) - converter = ModelConfigConverter(args.excel_path, args.output_path, - args.model_type, args.column_separator, - args.incol_separator, args.train_input_path, - args.eval_input_path, args.model_dir) + converter = ModelConfigConverter( + args.excel_path, args.output_path, args.model_type, args.column_separator, + args.incol_separator, args.train_input_path, args.eval_input_path, + args.model_dir + ) converter.convert() logging.info('Conversion done') logging.info('Tips:') diff --git a/easy_rec/python/tools/criteo/convert_data.py b/easy_rec/python/tools/criteo/convert_data.py index 6382865e7..a85c7f582 100644 --- a/easy_rec/python/tools/criteo/convert_data.py +++ b/easy_rec/python/tools/criteo/convert_data.py @@ -4,16 +4,16 @@ import gzip import logging import multiprocessing -import os -import traceback - import numpy as np +import os import pandas as pd import six +import traceback from tensorflow.python.platform import gfile logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) def save_np_bin(labels, dense_arr, cate_arr, prefix): @@ -38,8 +38,10 @@ def save_parquet(labels, dense_arr, cate_arr, prefix): def convert(input_path, prefix, part_record_num, save_format): - logging.info('start to convert %s, part_record_num=%d, save_format=%s' % - (input_path, part_record_num, save_format)) + logging.info( + 'start to convert %s, part_record_num=%d, save_format=%s' % + (input_path, part_record_num, save_format) + ) save_func = save_np_bin if save_format == 'parquet': save_func = save_parquet @@ -75,8 +77,10 @@ def convert(input_path, prefix, part_record_num, save_format): total_line += sid sid = 0 if sid > 0: - save_func(labels[:sid], dense_arr[:sid], cate_arr[:sid], - prefix + '_' + str(part_id)) + save_func( + labels[:sid], dense_arr[:sid], cate_arr[:sid], + prefix + '_' + str(part_id) + ) logging.info('\t%s write final part: %d' % (input_path, part_id)) part_id += 1 total_line += sid @@ -84,8 +88,10 @@ def convert(input_path, prefix, part_record_num, save_format): logging.error('convert %s failed: %s' % (input_path, str(ex))) logging.error(traceback.format_exc()) return - logging.info('done convert %s, total_line=%d, part_num=%d' % - (input_path, total_line, part_id)) + logging.info( + 'done convert %s, total_line=%d, part_num=%d' % + (input_path, total_line, part_id) + ) if __name__ == '__main__': @@ -105,27 +111,32 @@ def convert(input_path, prefix, part_record_num, save_format): parser = argparse.ArgumentParser() parser.add_argument( - '--input_dir', type=str, default=None, help='criteo 1t data dir') + '--input_dir', type=str, default=None, help='criteo 1t data dir' + ) parser.add_argument( - '--save_dir', - type=str, - default=None, - help='criteo binary data output dir ') + '--save_dir', + type=str, + default=None, + help='criteo binary data output dir ' + ) parser.add_argument( - '--save_format', - type=str, - default='npy', - help='save format, choices: npy|parquet') + '--save_format', + type=str, + default='npy', + help='save format, choices: npy|parquet' + ) parser.add_argument( - '--part_record_num', - type=int, - default=1024 * 1024 * 8, - help='the maximal number of samples in each binary file') + '--part_record_num', + type=int, + default=1024 * 1024 * 8, + help='the maximal number of samples in each binary file' + ) parser.add_argument( - '--dt', - nargs='*', - type=int, - help='select days to convert, default to select all: 0-23') + '--dt', + nargs='*', + type=int, + help='select days to convert, default to select all: 0-23' + ) args = parser.parse_args() @@ -148,8 +159,9 @@ def convert(input_path, prefix, part_record_num, save_format): input_path = os.path.join(args.input_dir, 'day_%d.gz' % d) prefix = os.path.join(args.save_dir, str(d)) proc = multiprocessing.Process( - target=convert, - args=(input_path, prefix, args.part_record_num, args.save_format)) + target=convert, + args=(input_path, prefix, args.part_record_num, args.save_format) + ) convert(input_path, prefix, args.part_record_num, args.save_format) proc.start() proc_arr.append(proc) diff --git a/easy_rec/python/tools/edit_lookup_graph.py b/easy_rec/python/tools/edit_lookup_graph.py index 7f1fc1fbf..1cf72144b 100644 --- a/easy_rec/python/tools/edit_lookup_graph.py +++ b/easy_rec/python/tools/edit_lookup_graph.py @@ -4,19 +4,18 @@ import logging import os import sys - import tensorflow as tf from tensorflow.core.protobuf import saved_model_pb2 -from tensorflow.python.lib.io.file_io import file_exists -from tensorflow.python.lib.io.file_io import recursive_create_dir +from tensorflow.python.lib.io.file_io import file_exists, recursive_create_dir from tensorflow.python.platform.gfile import GFile import easy_rec from easy_rec.python.utils.meta_graph_editor import MetaGraphEditor logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) if __name__ == '__main__': """Replace the default embedding_lookup ops with self defined embedding lookup ops. @@ -32,15 +31,21 @@ parser = argparse.ArgumentParser() parser.add_argument( - '--saved_model_dir', type=str, default=None, help='saved model dir') - parser.add_argument('--output_dir', type=str, default=None, help='output dir') + '--saved_model_dir', type=str, default=None, help='saved model dir' + ) + parser.add_argument( + '--output_dir', type=str, default=None, help='output dir' + ) parser.add_argument( - '--redis_url', type=str, default='127.0.0.1:6379', help='redis url') + '--redis_url', type=str, default='127.0.0.1:6379', help='redis url' + ) parser.add_argument( - '--redis_passwd', type=str, default='', help='redis password') + '--redis_passwd', type=str, default='', help='redis password' + ) parser.add_argument('--time_out', type=int, default=1500, help='timeout') parser.add_argument( - '--test_data_path', type=str, default='', help='test data path') + '--test_data_path', type=str, default='', help='test data path' + ) parser.add_argument('--verbose', action='store_true', default=False) args = parser.parse_args() @@ -59,19 +64,20 @@ recursive_create_dir(args.output_dir) meta_graph_editor = MetaGraphEditor( - lookup_lib_path, - args.saved_model_dir, - args.redis_url, - args.redis_passwd, - args.time_out, - meta_graph_def=None, - debug_dir=args.output_dir if args.verbose else '') + lookup_lib_path, + args.saved_model_dir, + args.redis_url, + args.redis_passwd, + args.time_out, + meta_graph_def=None, + debug_dir=args.output_dir if args.verbose else '' + ) meta_graph_editor.edit_graph() meta_graph_version = meta_graph_editor.meta_graph_version if meta_graph_version == '': export_ts = [ - x for x in args.saved_model_dir.split('/') if x != '' and x is not None + x for x in args.saved_model_dir.split('/') if x != '' and x is not None ] meta_graph_version = export_ts[-1] @@ -79,16 +85,21 @@ tf.reset_default_graph() saver = tf.train.import_meta_graph(meta_graph_editor._meta_graph_def) - embed_name_to_id_file = os.path.join(args.output_dir, 'embed_name_to_ids.txt') + embed_name_to_id_file = os.path.join( + args.output_dir, 'embed_name_to_ids.txt' + ) with GFile(embed_name_to_id_file, 'w') as fout: for tmp_norm_name in meta_graph_editor._embed_name_to_ids: fout.write( - '%s\t%s\n' % - (tmp_norm_name, meta_graph_editor._embed_name_to_ids[tmp_norm_name])) + '%s\t%s\n' % + (tmp_norm_name, meta_graph_editor._embed_name_to_ids[tmp_norm_name]) + ) tf.add_to_collection( - tf.GraphKeys.ASSET_FILEPATHS, - tf.constant( - embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt')) + tf.GraphKeys.ASSET_FILEPATHS, + tf.constant( + embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt' + ) + ) graph = tf.get_default_graph() inputs = meta_graph_editor.signature_def.inputs @@ -106,7 +117,8 @@ saver.restore(sess, args.saved_model_dir + '/variables/variables') output_dir = os.path.join(args.output_dir, meta_graph_version) tf.saved_model.simple_save( - sess, output_dir, inputs=inputs_map, outputs=outputs_map) + sess, output_dir, inputs=inputs_map, outputs=outputs_map + ) # the meta_graph_version could not be passed via existing interfaces # so we could only write it by the raw methods saved_model = saved_model_pb2.SavedModel() @@ -114,7 +126,8 @@ saved_model.ParseFromString(fin.read()) saved_model.meta_graphs[ - 0].meta_info_def.meta_graph_version = meta_graph_editor.meta_graph_version + 0 + ].meta_info_def.meta_graph_version = meta_graph_editor.meta_graph_version with GFile(os.path.join(output_dir, 'saved_model.pb'), 'wb') as fout: fout.write(saved_model.SerializeToString()) @@ -130,5 +143,6 @@ if len(feature_vals) >= 32: break out_vals = sess.run( - outputs_map, feed_dict={inputs_map['features']: feature_vals}) + outputs_map, feed_dict={inputs_map['features']: feature_vals} + ) logging.info('test_data probs:' + str(out_vals)) diff --git a/easy_rec/python/tools/faiss_index_pai.py b/easy_rec/python/tools/faiss_index_pai.py index e9ebe3f89..4adb1447b 100644 --- a/easy_rec/python/tools/faiss_index_pai.py +++ b/easy_rec/python/tools/faiss_index_pai.py @@ -2,18 +2,18 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from __future__ import print_function +import faiss import logging +import numpy as np import os import sys - -import faiss -import numpy as np import tensorflow as tf from easy_rec.python.utils import io_util logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) tf.app.flags.DEFINE_string('tables', '', 'tables passed by pai command') tf.app.flags.DEFINE_integer('batch_size', 1024, 'batch size') @@ -30,10 +30,12 @@ def main(argv): reader = tf.python_io.TableReader( - FLAGS.tables, slice_id=0, slice_count=1, capacity=FLAGS.batch_size * 2) + FLAGS.tables, slice_id=0, slice_count=1, capacity=FLAGS.batch_size * 2 + ) i = 0 id_map_f = tf.gfile.GFile( - os.path.join(FLAGS.index_output_dir, 'id_mapping'), 'w') + os.path.join(FLAGS.index_output_dir, 'id_mapping'), 'w' + ) embeddings = [] while True: try: @@ -44,7 +46,8 @@ def main(argv): id_map_f.write('%s\n' % eid) embeddings.extend( - [list(map(float, record[1].split(b','))) for record in records]) + [list(map(float, record[1].split(b','))) for record in records] + ) i += 1 if i % 100 == 0: logging.info('read %d embeddings.' % (i * FLAGS.batch_size)) @@ -56,11 +59,14 @@ def main(argv): logging.info('Building faiss index..') if FLAGS.index_type == 'IVFFlat': quantizer = faiss.IndexFlatIP(FLAGS.embedding_dim) - index = faiss.IndexIVFFlat(quantizer, FLAGS.embedding_dim, FLAGS.ivf_nlist, - faiss.METRIC_INNER_PRODUCT) + index = faiss.IndexIVFFlat( + quantizer, FLAGS.embedding_dim, FLAGS.ivf_nlist, + faiss.METRIC_INNER_PRODUCT + ) elif FLAGS.index_type == 'HNSWFlat': - index = faiss.IndexHNSWFlat(FLAGS.embedding_dim, FLAGS.hnsw_M, - faiss.METRIC_INNER_PRODUCT) + index = faiss.IndexHNSWFlat( + FLAGS.embedding_dim, FLAGS.hnsw_M, faiss.METRIC_INNER_PRODUCT + ) index.hnsw.efConstruction = FLAGS.hnsw_efConstruction else: raise NotImplementedError @@ -75,7 +81,8 @@ def main(argv): faiss.write_index(index, 'faiss_index') with tf.gfile.GFile( - os.path.join(FLAGS.index_output_dir, 'faiss_index'), 'wb') as f_out: + os.path.join(FLAGS.index_output_dir, 'faiss_index'), 'wb' + ) as f_out: with open('faiss_index', 'rb') as f_in: f_out.write(f_in.read()) @@ -83,14 +90,16 @@ def main(argv): # IVFFlat for ivf_nlist in [100, 500, 1000, 2000]: quantizer = faiss.IndexFlatIP(FLAGS.embedding_dim) - index = faiss.IndexIVFFlat(quantizer, FLAGS.embedding_dim, ivf_nlist, - faiss.METRIC_INNER_PRODUCT) + index = faiss.IndexIVFFlat( + quantizer, FLAGS.embedding_dim, ivf_nlist, faiss.METRIC_INNER_PRODUCT + ) index.train(embeddings) index.add(embeddings) index_name = 'faiss_index_ivfflat_nlist%d' % ivf_nlist faiss.write_index(index, index_name) with tf.gfile.GFile( - os.path.join(FLAGS.index_output_dir, index_name), 'wb') as f_out: + os.path.join(FLAGS.index_output_dir, index_name), 'wb' + ) as f_out: with open(index_name, 'rb') as f_in: f_out.write(f_in.read()) @@ -99,14 +108,18 @@ def main(argv): for hnsw_efConstruction in [64, 128, 256, 512, 1024, 2048, 4096, 8196]: if hnsw_efConstruction < hnsw_M * 2: continue - index = faiss.IndexHNSWFlat(FLAGS.embedding_dim, hnsw_M, - faiss.METRIC_INNER_PRODUCT) + index = faiss.IndexHNSWFlat( + FLAGS.embedding_dim, hnsw_M, faiss.METRIC_INNER_PRODUCT + ) index.hnsw.efConstruction = hnsw_efConstruction index.add(embeddings) - index_name = 'faiss_index_hnsw_M%d_ef%d' % (hnsw_M, hnsw_efConstruction) + index_name = 'faiss_index_hnsw_M%d_ef%d' % ( + hnsw_M, hnsw_efConstruction + ) faiss.write_index(index, index_name) with tf.gfile.GFile( - os.path.join(FLAGS.index_output_dir, index_name), 'wb') as f_out: + os.path.join(FLAGS.index_output_dir, index_name), 'wb' + ) as f_out: with open(index_name, 'rb') as f_in: f_out.write(f_in.read()) diff --git a/easy_rec/python/tools/feature_selection.py b/easy_rec/python/tools/feature_selection.py index f50a00fac..a0478cd9d 100644 --- a/easy_rec/python/tools/feature_selection.py +++ b/easy_rec/python/tools/feature_selection.py @@ -1,51 +1,57 @@ -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function import json -import os -import sys -from collections import OrderedDict - import numpy as np +import os import pandas as pd +import sys import tensorflow as tf +from collections import OrderedDict from tensorflow.python.framework.meta_graph import read_meta_graph_file -from easy_rec.python.utils import config_util -from easy_rec.python.utils import io_util +from easy_rec.python.utils import config_util, io_util if tf.__version__ >= '2.0': tf = tf.compat.v1 import matplotlib # NOQA + matplotlib.use('Agg') # NOQA import matplotlib.pyplot as plt # NOQA -tf.app.flags.DEFINE_string('model_type', 'variational_dropout', - 'feature selection model type') -tf.app.flags.DEFINE_string('config_path', '', - 'feature selection model config path') -tf.app.flags.DEFINE_string('checkpoint_path', None, - 'feature selection model checkpoint path') -tf.app.flags.DEFINE_string('output_dir', '', - 'feature selection result directory') +tf.app.flags.DEFINE_string( + 'model_type', 'variational_dropout', 'feature selection model type' +) +tf.app.flags.DEFINE_string( + 'config_path', '', 'feature selection model config path' +) +tf.app.flags.DEFINE_string( + 'checkpoint_path', None, 'feature selection model checkpoint path' +) +tf.app.flags.DEFINE_string( + 'output_dir', '', 'feature selection result directory' +) tf.app.flags.DEFINE_integer( - 'topk', 100, 'select topk importance features for each feature group') + 'topk', 100, 'select topk importance features for each feature group' +) tf.app.flags.DEFINE_string('fg_path', '', 'fg config path') -tf.app.flags.DEFINE_bool('visualize', False, - 'visualization feature selection result or not') +tf.app.flags.DEFINE_bool( + 'visualize', False, 'visualization feature selection result or not' +) FLAGS = tf.app.flags.FLAGS class VariationalDropoutFS: - def __init__(self, - config_path, - output_dir, - topk, - checkpoint_path=None, - fg_path=None, - visualize=False): + def __init__( + self, + config_path, + output_dir, + topk, + checkpoint_path=None, + fg_path=None, + visualize=False + ): self._config_path = config_path self._output_dir = output_dir self._topk = topk @@ -64,7 +70,8 @@ def process(self): for group_name, feature_dim_dropout_p in feature_dim_dropout_p_map.items(): tf.logging.info('Calculating %s feature importance ...' % group_name) feature_importance = self._get_feature_importance( - feature_dim_dropout_p, embedding_wise_variational_dropout) + feature_dim_dropout_p, embedding_wise_variational_dropout + ) feature_importance_map[group_name] = feature_importance tf.logging.info('Dump %s feature importance to csv ...' % group_name) @@ -83,7 +90,8 @@ def _feature_dim_dropout_ratio(self): """Get dropout ratio of embedding-wise or feature-wise.""" config = config_util.get_configs_from_pipeline_file(self._config_path) assert config.model_config.HasField( - 'variational_dropout'), 'variational_dropout must be in model_config' + 'variational_dropout' + ), 'variational_dropout must be in model_config' embedding_wise_variational_dropout = config.model_config.variational_dropout.embedding_wise_variational_dropout @@ -94,8 +102,8 @@ def _feature_dim_dropout_ratio(self): meta_graph_def = read_meta_graph_file(checkpoint_path + '.meta') features_dimension_map = dict() - for col_def in meta_graph_def.collection_def[ - 'variational_dropout'].bytes_list.value: + for col_def in meta_graph_def.collection_def['variational_dropout' + ].bytes_list.value: name, features_dimension = json.loads(col_def) name = 'all' if name == '' else name features_dimension_map[name] = OrderedDict(features_dimension) @@ -120,12 +128,12 @@ def _feature_dim_dropout_ratio(self): feature_dim_dropout_p = {} if embedding_wise_variational_dropout: index_end = 0 - for feature_name, feature_dim in features_dimension_map[ - group_name].items(): + for feature_name, feature_dim in features_dimension_map[group_name + ].items(): index_start = index_end index_end = index_start + feature_dim feature_dim_dropout_p[feature_name] = feature_dims_importance[ - index_start:index_end] + index_start:index_end] else: index = 0 for feature_name in features_dimension_map[group_name].keys(): @@ -135,8 +143,9 @@ def _feature_dim_dropout_ratio(self): feature_dim_dropout_p_map[group_name] = feature_dim_dropout_p return feature_dim_dropout_p_map, embedding_wise_variational_dropout - def _get_feature_importance(self, feature_dim_dropout_p, - embedding_wise_variational_dropout): + def _get_feature_importance( + self, feature_dim_dropout_p, embedding_wise_variational_dropout + ): """Calculate feature importance.""" if embedding_wise_variational_dropout: feature_importance = {} @@ -144,10 +153,12 @@ def _get_feature_importance(self, feature_dim_dropout_p, dropout_rate_mean = np.mean(item[1]) feature_importance[item[0]] = dropout_rate_mean feature_importance = OrderedDict( - sorted(feature_importance.items(), key=lambda e: e[1])) + sorted(feature_importance.items(), key=lambda e: e[1]) + ) else: feature_importance = OrderedDict( - sorted(feature_dim_dropout_p.items(), key=lambda e: e[1])) + sorted(feature_dim_dropout_p.items(), key=lambda e: e[1]) + ) return feature_importance def _process_config(self, feature_importance_map): @@ -199,8 +210,9 @@ def _process_config(self, feature_importance_map): feature_group.ClearField('feature_names') feature_group.feature_names.extend(feature_names) config_util.save_message( - config, - os.path.join(self._output_dir, os.path.basename(self._config_path))) + config, + os.path.join(self._output_dir, os.path.basename(self._config_path)) + ) if self._fg_path is not None and len(self._fg_path) > 0: with tf.gfile.Open(self._fg_path) as f: @@ -214,18 +226,21 @@ def _process_config(self, feature_importance_map): features.append(feature) fg_json['features'] = features with tf.gfile.Open( - os.path.join(self._output_dir, os.path.basename(self._fg_path)), - 'w') as f: + os.path.join(self._output_dir, os.path.basename(self._fg_path)), 'w' + ) as f: json.dump(fg_json, f, indent=4) def _dump_to_csv(self, feature_importance, group_name): """Dump feature importance data to a csv file.""" with tf.gfile.Open( - os.path.join(self._output_dir, - 'feature_dropout_ratio_%s.csv' % group_name), 'w') as f: + os.path.join( + self._output_dir, 'feature_dropout_ratio_%s.csv' % group_name + ), 'w' + ) as f: df = pd.DataFrame( - columns=['feature_name', 'mean_drop_p'], - data=[list(kv) for kv in feature_importance.items()]) + columns=['feature_name', 'mean_drop_p'], + data=[list(kv) for kv in feature_importance.items()] + ) df.to_csv(f, encoding='gbk') def _visualize_embedding_dim_importance(self, feature_dim_dropout_p): @@ -246,20 +261,22 @@ def _visualize_embedding_dim_importance(self, feature_dim_dropout_p): performance_list.append(feature_dropout_p[i]) fig, ax = plt.subplots() b = ax.barh( - y_pos, - performance_list, - align='center', - alpha=0.4, - label='dropout_rate', - lw=1) + y_pos, + performance_list, + align='center', + alpha=0.4, + label='dropout_rate', + lw=1 + ) for rect in b: w = rect.get_width() ax.text( - w, - rect.get_y() + rect.get_height() / 2, - '%.4f' % w, - ha='left', - va='center') + w, + rect.get_y() + rect.get_height() / 2, + '%.4f' % w, + ha='left', + va='center' + ) plt.yticks(y_pos, embedding_dims) plt.xlabel(feature_name) plt.title('Dropout ratio') @@ -270,8 +287,9 @@ def _visualize_embedding_dim_importance(self, feature_dim_dropout_p): def _visualize_feature_importance(self, feature_importance, group_name): """Draw feature importance histogram.""" df = pd.DataFrame( - columns=['feature_name', 'mean_drop_p'], - data=[list(kv) for kv in feature_importance.items()]) + columns=['feature_name', 'mean_drop_p'], + data=[list(kv) for kv in feature_importance.items()] + ) df['color'] = ['red' if x < 0.5 else 'green' for x in df['mean_drop_p']] df.sort_values('mean_drop_p', inplace=True, ascending=False) df.reset_index(inplace=True) @@ -280,23 +298,26 @@ def _visualize_feature_importance(self, feature_importance, group_name): plt.hlines(y=df.index, xmin=0, xmax=df.mean_drop_p) for x, y, tex in zip(df.mean_drop_p, df.index, df.mean_drop_p): plt.text( - x, - y, - round(tex, 2), - horizontalalignment='right' if x < 0 else 'left', - verticalalignment='center', - fontdict={ - 'color': 'red' if x < 0 else 'green', - 'size': 14 - }) + x, + y, + round(tex, 2), + horizontalalignment='right' if x < 0 else 'left', + verticalalignment='center', + fontdict={ + 'color': 'red' if x < 0 else 'green', + 'size': 14 + } + ) # Decorations plt.yticks(df.index, df.feature_name, fontsize=20) plt.title('Dropout Ratio', fontdict={'size': 30}) plt.grid(linestyle='--', alpha=0.5) plt.xlim(0, 1) with tf.gfile.GFile( - os.path.join(self._output_dir, - 'feature_dropout_pic_%s.png' % group_name), 'wb') as f: + os.path.join( + self._output_dir, 'feature_dropout_pic_%s.png' % group_name + ), 'wb' + ) as f: plt.savefig(f, format='png') @@ -304,13 +325,15 @@ def _visualize_feature_importance(self, feature_importance, group_name): sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv) if FLAGS.model_type == 'variational_dropout': fs = VariationalDropoutFS( - FLAGS.config_path, - FLAGS.output_dir, - FLAGS.topk, - checkpoint_path=FLAGS.checkpoint_path, - fg_path=FLAGS.fg_path, - visualize=FLAGS.visualize) + FLAGS.config_path, + FLAGS.output_dir, + FLAGS.topk, + checkpoint_path=FLAGS.checkpoint_path, + fg_path=FLAGS.fg_path, + visualize=FLAGS.visualize + ) fs.process() else: - raise ValueError('Unknown feature selection model type %s' % - FLAGS.model_type) + raise ValueError( + 'Unknown feature selection model type %s' % FLAGS.model_type + ) diff --git a/easy_rec/python/tools/hit_rate_ds.py b/easy_rec/python/tools/hit_rate_ds.py index 5528e0aa2..f8c4b9fa5 100644 --- a/easy_rec/python/tools/hit_rate_ds.py +++ b/easy_rec/python/tools/hit_rate_ds.py @@ -13,25 +13,19 @@ # limitations under the License. # ============================================================================= # """Evaluation of Top k hitrate.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function +import graphlearn as gl import json import logging import os import sys - -import graphlearn as gl import tensorflow as tf from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.utils import config_util -from easy_rec.python.utils import io_util +from easy_rec.python.utils import config_util, io_util from easy_rec.python.utils.config_util import process_multi_file_input_path -from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch -from easy_rec.python.utils.hit_rate_utils import load_graph -from easy_rec.python.utils.hit_rate_utils import reduce_hitrate +from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch, load_graph, reduce_hitrate # NOQA from easy_rec.python.utils.hive_utils import HiveUtils if tf.__version__ >= '2.0': @@ -40,15 +34,18 @@ from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_train_worker_num_on_ds # NOQA logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) tf.app.flags.DEFINE_string('item_emb_table', '', 'item embedding table name') tf.app.flags.DEFINE_string('gt_table', '', 'ground truth table name') -tf.app.flags.DEFINE_string('hitrate_details_result', '', - 'hitrate detail file path') -tf.app.flags.DEFINE_string('total_hitrate_result', '', - 'total hitrate result file path') +tf.app.flags.DEFINE_string( + 'hitrate_details_result', '', 'hitrate detail file path' +) +tf.app.flags.DEFINE_string( + 'total_hitrate_result', '', 'total hitrate result file path' +) tf.app.flags.DEFINE_string('pipeline_config_path', '', 'pipeline config path') tf.app.flags.DEFINE_integer('batch_size', 512, 'batch size') @@ -60,8 +57,9 @@ tf.app.flags.DEFINE_integer('timeout', '60', 'timeout') tf.app.flags.DEFINE_integer('num_interests', 1, 'max number of interests') tf.app.flags.DEFINE_string('gt_table_field_sep', '\t', 'gt_table_field_sep') -tf.app.flags.DEFINE_string('item_emb_table_field_sep', '\t', - 'item_emb_table_field_sep') +tf.app.flags.DEFINE_string( + 'item_emb_table_field_sep', '\t', 'item_emb_table_field_sep' +) tf.app.flags.DEFINE_bool('is_on_ds', False, help='is on ds') FLAGS = tf.app.flags.FLAGS @@ -94,18 +92,21 @@ def compute_hitrate(g, gt_all, hitrate_writer, gt_table=None): hitrates = [str(hitrate) for hitrate in hitrates] topk_recalls = [','.join(str(x) for x in ids) for ids in recall_ids] topk_dists = [ - ','.join('|'.join(str(x) - for x in dist) - for dist in dists) - for dists in recall_distances + ','.join('|'.join(str(x) for x in dist) for dist in dists) + for dists in recall_distances ] bad_cases = [','.join(str(x) for x in bad_case) for bad_case in bad_cases] bad_dists = [','.join(str(x) for x in dist) for dist in bad_dists] - hitrate_writer.write('\n'.join([ - '\t'.join(line) for line in zip(src_ids, topk_recalls, topk_dists, - hitrates, bad_cases, bad_dists) - ])) + hitrate_writer.write( + '\n'.join( + [ + '\t'.join(line) for line in zip( + src_ids, topk_recalls, topk_dists, hitrates, bad_cases, bad_dists + ) + ] + ) + ) print('total_hits: ', total_hits) print('total_gt_count: ', total_gt_count) return total_hits, total_gt_count @@ -148,7 +149,8 @@ def main(): gt_table = FLAGS.gt_table pipeline_config = config_util.get_configs_from_pipeline_file( - FLAGS.pipeline_config_path) + FLAGS.pipeline_config_path + ) logging.info('i_emb_table %s', i_emb_table) input_type = pipeline_config.data_config.input_type @@ -157,27 +159,32 @@ def main(): i_emb_table = process_multi_file_input_path(i_emb_table) else: hive_utils = HiveUtils( - data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input) + data_config=pipeline_config.data_config, + hive_config=pipeline_config.hive_train_input + ) i_emb_table = hive_utils.get_table_location(i_emb_table) - g = load_graph(i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, - FLAGS.knn_strict) + g = load_graph( + i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, + FLAGS.knn_strict + ) gl.set_tracker_mode(0) gl.set_field_delimiter(FLAGS.item_emb_table_field_sep) - cluster = tf.train.ClusterSpec({ + cluster = tf.train.ClusterSpec( + { 'ps': tf_config['cluster']['ps'], 'worker': tf_config['cluster']['worker'] - }) + } + ) server = tf.train.Server(cluster, job_name=job_name, task_index=task_index) if job_name == 'ps': server.join() else: worker_hosts = [ - str(host.split(':')[0]) + ':888' + str(i) - for i, host in enumerate(tf_config['cluster']['worker']) + str(host.split(':')[0]) + ':888' + str(i) + for i, host in enumerate(tf_config['cluster']['worker']) ] worker_hosts = ','.join(worker_hosts) g.init(task_index=task_index, task_count=worker_count, hosts=worker_hosts) @@ -187,23 +194,28 @@ def main(): gt_all = gt_hdfs(gt_table, FLAGS.batch_size, FLAGS.gt_table_field_sep) else: gt_reader = HiveUtils( - data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input, - selected_cols='*') + data_config=pipeline_config.data_config, + hive_config=pipeline_config.hive_train_input, + selected_cols='*' + ) gt_all = gt_reader.hive_read_lines(gt_table, FLAGS.batch_size) if not tf.gfile.IsDirectory(hitrate_details_result): tf.gfile.MakeDirs(hitrate_details_result) - hitrate_details_result = os.path.join(hitrate_details_result, - 'part-%s' % task_index) + hitrate_details_result = os.path.join( + hitrate_details_result, 'part-%s' % task_index + ) details_writer = tf.gfile.GFile(hitrate_details_result, 'w') print('Start compute hitrate...') - total_hits, total_gt_count = compute_hitrate(g, gt_all, details_writer, - gt_table) + total_hits, total_gt_count = compute_hitrate( + g, gt_all, details_writer, gt_table + ) var_total_hitrate, var_worker_count = reduce_hitrate( - cluster, total_hits, total_gt_count, task_index) + cluster, total_hits, total_gt_count, task_index + ) with tf.train.MonitoredTrainingSession( - master=server.target, is_chief=(task_index == 0)) as sess: + master=server.target, is_chief=(task_index == 0) + ) as sess: outs = sess.run([var_total_hitrate, var_worker_count]) # write after all workers have completed the calculation of hitrate. diff --git a/easy_rec/python/tools/hit_rate_pai.py b/easy_rec/python/tools/hit_rate_pai.py index 977df20be..4a5e218a9 100644 --- a/easy_rec/python/tools/hit_rate_pai.py +++ b/easy_rec/python/tools/hit_rate_pai.py @@ -13,18 +13,13 @@ # limitations under the License. # ============================================================================= """Evaluation of Top k hitrate.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import sys - import tensorflow as tf from easy_rec.python.utils import io_util -from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch -from easy_rec.python.utils.hit_rate_utils import load_graph -from easy_rec.python.utils.hit_rate_utils import reduce_hitrate +from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch, load_graph, reduce_hitrate # NOQA flags = tf.app.flags FLAGS = flags.FLAGS @@ -68,16 +63,19 @@ def compute_hitrate(g, gt_reader, hitrate_writer): total_gt_count += gt_count topk_recalls = [','.join(str(x) for x in ids) for ids in recall_ids] topk_dists = [ - ','.join(str(x) for x in dists) for dists in recall_distances + ','.join(str(x) for x in dists) for dists in recall_distances ] bad_cases = [','.join(str(x) for x in case) for case in bad_cases] bad_dists = [','.join(str(x) for x in dist) for dist in bad_dists] hitrate_writer.write( - list( - zip(src_ids, topk_recalls, topk_dists, hitrates, bad_cases, - bad_dists)), - indices=[0, 1, 2, 3, 4, 5]) + list( + zip( + src_ids, topk_recalls, topk_dists, hitrates, bad_cases, bad_dists + ) + ), + indices=[0, 1, 2, 3, 4, 5] + ) except tf.python_io.OutOfRangeException: break return total_hits, total_gt_count @@ -88,38 +86,49 @@ def main(): input_tables = FLAGS.tables.split(',') if FLAGS.recall_type == 'u2i': i_emb_table, gt_table = input_tables - g = load_graph(i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, - FLAGS.knn_strict) + g = load_graph( + i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, + FLAGS.knn_strict + ) else: i_emb_table, gt_table = input_tables[-2], input_tables[-1] - g = load_graph(i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, - FLAGS.knn_strict) + g = load_graph( + i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, + FLAGS.knn_strict + ) hitrate_details_table, total_hitrate_table = FLAGS.outputs.split(',') - cluster = tf.train.ClusterSpec({ + cluster = tf.train.ClusterSpec( + { 'ps': FLAGS.ps_hosts.split(','), 'worker': FLAGS.worker_hosts.split(',') - }) + } + ) server = tf.train.Server( - cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index) + cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index + ) if FLAGS.job_name == 'ps': server.join() else: g.init(task_index=FLAGS.task_index, task_count=worker_count) gt_reader = tf.python_io.TableReader( - gt_table, - slice_id=FLAGS.task_index, - slice_count=worker_count, - capacity=2048) + gt_table, + slice_id=FLAGS.task_index, + slice_count=worker_count, + capacity=2048 + ) details_writer = tf.python_io.TableWriter( - hitrate_details_table, slice_id=FLAGS.task_index) + hitrate_details_table, slice_id=FLAGS.task_index + ) print('Start compute hitrate...') total_hits, total_gt_count = compute_hitrate(g, gt_reader, details_writer) var_total_hitrate, var_worker_count = reduce_hitrate( - cluster, total_hits, total_gt_count, FLAGS.task_index) + cluster, total_hits, total_gt_count, FLAGS.task_index + ) with tf.train.MonitoredTrainingSession( - master=server.target, is_chief=(FLAGS.task_index == 0)) as sess: + master=server.target, is_chief=(FLAGS.task_index == 0) + ) as sess: outs = sess.run([var_total_hitrate, var_worker_count]) # write after all workers have completed the calculation of hitrate. diff --git a/easy_rec/python/tools/pre_check.py b/easy_rec/python/tools/pre_check.py index da7f1923b..b3bb8fbd2 100644 --- a/easy_rec/python/tools/pre_check.py +++ b/easy_rec/python/tools/pre_check.py @@ -4,35 +4,33 @@ import logging import os import sys - import tensorflow as tf from easy_rec.python.input.input import Input -from easy_rec.python.utils import config_util -from easy_rec.python.utils import fg_util -from easy_rec.python.utils import io_util -from easy_rec.python.utils.check_utils import check_env_and_input_path -from easy_rec.python.utils.check_utils import check_sequence +from easy_rec.python.utils import config_util, fg_util, io_util +from easy_rec.python.utils.check_utils import check_env_and_input_path, check_sequence # NOQA if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) -tf.app.flags.DEFINE_string('pipeline_config_path', None, - 'Path to pipeline config ' - 'file.') + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) +tf.app.flags.DEFINE_string( + 'pipeline_config_path', None, 'Path to pipeline config ' + 'file.' +) tf.app.flags.DEFINE_multi_string( - 'data_input_path', None, help='data input path') + 'data_input_path', None, help='data input path' +) FLAGS = tf.app.flags.FLAGS -def _get_input_fn(data_config, - feature_configs, - data_path=None, - export_config=None): +def _get_input_fn( + data_config, feature_configs, data_path=None, export_config=None +): """Build estimator input function. Args: @@ -58,19 +56,21 @@ def _get_input_fn(data_config, task_index = 0 input_obj = input_class( - data_config, - feature_configs, - data_path, - task_index=task_index, - task_num=worker_num, - check_mode=True) + data_config, + feature_configs, + data_path, + task_index=task_index, + task_num=worker_num, + check_mode=True + ) input_fn = input_obj.create_input(export_config) return input_fn def loda_pipeline_config(pipeline_config_path): pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False) + pipeline_config_path, False + ) if pipeline_config.fg_json_path: fg_util.load_fg_json_to_config(pipeline_config) config_util.auto_expand_share_feature_configs(pipeline_config) @@ -81,16 +81,18 @@ def run_check(pipeline_config, input_path): logging.info('data_input_path: %s' % input_path) check_env_and_input_path(pipeline_config, input_path) feature_configs = config_util.get_compatible_feature_configs(pipeline_config) - eval_input_fn = _get_input_fn(pipeline_config.data_config, feature_configs, - input_path) + eval_input_fn = _get_input_fn( + pipeline_config.data_config, feature_configs, input_path + ) eval_spec = tf.estimator.EvalSpec( - name='val', - input_fn=eval_input_fn, - steps=None, - throttle_secs=10, - exporters=[]) - input_iter = eval_spec.input_fn( - mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() + name='val', + input_fn=eval_input_fn, + steps=None, + throttle_secs=10, + exporters=[] + ) + input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL + ).make_one_shot_iterator() with tf.Session() as sess: try: while (True): diff --git a/easy_rec/python/tools/predict_and_chk.py b/easy_rec/python/tools/predict_and_chk.py index bc7353f76..407dff2d0 100644 --- a/easy_rec/python/tools/predict_and_chk.py +++ b/easy_rec/python/tools/predict_and_chk.py @@ -3,11 +3,10 @@ import argparse import json import logging +import numpy as np import os import sys -import numpy as np - import easy_rec from easy_rec.python.inference.predictor import Predictor @@ -18,40 +17,49 @@ logging.warning('exception: %s' % str(ex)) logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--saved_model_dir', type=str, default=None, help='saved model directory') + '--saved_model_dir', type=str, default=None, help='saved model directory' + ) parser.add_argument( - '--input_path', type=str, default=None, help='input feature path') + '--input_path', type=str, default=None, help='input feature path' + ) parser.add_argument('--save_path', type=str, default=None, help='save path') parser.add_argument( - '--cmp_res_path', type=str, default=None, help='compare result path') + '--cmp_res_path', type=str, default=None, help='compare result path' + ) parser.add_argument( - '--cmp_key', type=str, default='probs', help='compare key') + '--cmp_key', type=str, default='probs', help='compare key' + ) parser.add_argument( - '--rtp_fea_id', - type=int, - default=-1, - help='rtp feature column index, default to the last column') + '--rtp_fea_id', + type=int, + default=-1, + help='rtp feature column index, default to the last column' + ) parser.add_argument('--tol', type=float, default=1e-5, help='tolerance') parser.add_argument( - '--label_id', - nargs='*', - type=int, - help='the label column, which is to be excluded') + '--label_id', + nargs='*', + type=int, + help='the label column, which is to be excluded' + ) parser.add_argument( - '--separator', - type=str, - default='', - help='separator between features, default to \\u0002') + '--separator', + type=str, + default='', + help='separator between features, default to \\u0002' + ) parser.add_argument( - '--rtp_separator', - type=str, - default='', - help='separator, default to \\u0001') + '--rtp_separator', + type=str, + default='', + help='separator, default to \\u0001' + ) args = parser.parse_args() if not args.saved_model_dir: @@ -72,7 +80,7 @@ predictor = Predictor(args.saved_model_dir) if len(predictor.input_names) == 1: assert len( - args.label_id + args.label_id ) == 0, 'label_id should not be set if rtp feature format is used.' with open(args.input_path, 'r') as fin: @@ -82,8 +90,8 @@ line_tok = line_str.split(args.rtp_separator) feature = line_tok[args.rtp_fea_id] feature = [ - x for fid, x in enumerate(feature.split(args.separator)) - if fid not in args.label_id + x for fid, x in enumerate(feature.split(args.separator)) + if fid not in args.label_id ] if 'features' in predictor.input_names: feature = args.separator.join(feature) @@ -104,8 +112,8 @@ for line_id, line_str in enumerate(fin): line_str = line_str.strip() line_pred = json.loads(line_str) - assert np.abs( - line_pred[args.cmp_key] - - output[line_id][args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( - line_id, - np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key])) + assert np.abs(line_pred[args.cmp_key] - output[line_id][ + args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( + line_id, + np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]) + ) diff --git a/easy_rec/python/tools/read_kafka.py b/easy_rec/python/tools/read_kafka.py index 57578b863..cedd0ed51 100644 --- a/easy_rec/python/tools/read_kafka.py +++ b/easy_rec/python/tools/read_kafka.py @@ -4,12 +4,12 @@ import logging import os import sys - from kafka import KafkaConsumer from kafka.structs import TopicPartition logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -27,9 +27,10 @@ servers = args.servers.split(',') consumer = KafkaConsumer( - group_id=args.group, - bootstrap_servers=servers, - consumer_timeout_ms=args.timeout * 1000) + group_id=args.group, + bootstrap_servers=servers, + consumer_timeout_ms=args.timeout * 1000 + ) if args.partitions is not None: partitions = [int(x) for x in args.partitions.split(',')] @@ -38,16 +39,18 @@ logging.info('partitions: %s' % partitions) topics = [ - TopicPartition(topic=args.topic, partition=part_id) - for part_id in partitions + TopicPartition(topic=args.topic, partition=part_id) + for part_id in partitions ] consumer.assign(topics) consumer.seek_to_beginning() record_id = 0 for x in consumer: - logging.info('%d: key=%s\toffset=%d\ttimestamp=%d\tlen=%d' % - (record_id, x.key, x.offset, x.timestamp, len(x.value))) + logging.info( + '%d: key=%s\toffset=%d\ttimestamp=%d\tlen=%d' % + (record_id, x.key, x.offset, x.timestamp, len(x.value)) + ) if args.save_dir is not None: save_path = os.path.join(args.save_dir, x.key) with open(save_path, 'wb') as fout: diff --git a/easy_rec/python/tools/split_model_pai.py b/easy_rec/python/tools/split_model_pai.py index d86791708..d5e325292 100644 --- a/easy_rec/python/tools/split_model_pai.py +++ b/easy_rec/python/tools/split_model_pai.py @@ -3,11 +3,9 @@ import logging import os import sys - import tensorflow as tf from tensorflow.core.framework import graph_pb2 -from tensorflow.python.framework import importer -from tensorflow.python.framework import ops +from tensorflow.python.framework import importer, ops from tensorflow.python.framework.dtypes import _TYPE_TO_STRING from tensorflow.python.ops.resource_variable_ops import _from_proto_fn from tensorflow.python.saved_model import signature_constants @@ -30,7 +28,8 @@ tf.app.flags.DEFINE_string('item_fg_json_path', '', '') logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) def search_pb(directory): @@ -131,7 +130,8 @@ def load_meta_graph_def(model_dir): variable_protos = {} meta_graph_def = saved_model_utils.get_meta_graph_def( - model_dir, tf.saved_model.tag_constants.SERVING) + model_dir, tf.saved_model.tag_constants.SERVING + ) signatures = meta_graph_def.signature_def collections = meta_graph_def.collection_def @@ -151,16 +151,19 @@ def load_meta_graph_def(model_dir): # parse signature info for SavedModel for sig_name in signatures: if signatures[ - sig_name].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: + sig_name + ].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: tf.logging.info('[Signature] inputs:') for input_name in signatures[sig_name].inputs: input_tensor_shape = [] input_tensor = signatures[sig_name].inputs[input_name] for dim in input_tensor.tensor_shape.dim: input_tensor_shape.append(int(dim.size)) - tf.logging.info('"%s": %s; %s' % - (input_name, _TYPE_TO_STRING[input_tensor.dtype], - input_tensor_shape)) + tf.logging.info( + '"%s": %s; %s' % ( + input_name, _TYPE_TO_STRING[input_tensor.dtype], input_tensor_shape + ) + ) input_tensor_names[input_name] = input_tensor.name tf.logging.info('[Signature] outputs:') for output_name in signatures[sig_name].outputs: @@ -168,16 +171,21 @@ def load_meta_graph_def(model_dir): output_tensor = signatures[sig_name].outputs[output_name] for dim in output_tensor.tensor_shape.dim: output_tensor_shape.append(int(dim.size)) - tf.logging.info('"%s": %s; %s' % - (output_name, _TYPE_TO_STRING[output_tensor.dtype], - output_tensor_shape)) + tf.logging.info( + '"%s": %s; %s' % ( + output_name, _TYPE_TO_STRING[output_tensor.dtype + ], output_tensor_shape + ) + ) output_tensor_names[output_name] = output_tensor.name return meta_graph_def, variable_protos, input_tensor_names, output_tensor_names -def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, - output_tensor_names, part_name, part_dir): +def export( + model_dir, meta_graph_def, variable_protos, input_tensor_names, + output_tensor_names, part_name, part_dir +): """Export subpart saved model. Args: @@ -190,16 +198,16 @@ def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, part_dir: subpart model export directory. """ output_tensor_names = { - x: output_tensor_names[x] - for x in output_tensor_names.keys() - if part_name in x + x: output_tensor_names[x] + for x in output_tensor_names.keys() if part_name in x } output_node_names = [ - _node_name(output_tensor_names[x]) for x in output_tensor_names.keys() + _node_name(output_tensor_names[x]) for x in output_tensor_names.keys() ] inference_graph, variables_to_keep = extract_sub_graph( - meta_graph_def.graph_def, output_node_names, variable_protos) + meta_graph_def.graph_def, output_node_names, variable_protos + ) tf.reset_default_graph() with tf.Session() as sess: @@ -217,7 +225,8 @@ def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, for input_name in input_tensor_names: try: tensor_info = tf.saved_model.utils.build_tensor_info( - graph.get_tensor_by_name(input_tensor_names[input_name])) + graph.get_tensor_by_name(input_tensor_names[input_name]) + ) signature_inputs[input_name] = tensor_info except Exception: print('ignore input: %s' % input_name) @@ -225,22 +234,25 @@ def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, signature_outputs = {} for output_name in output_tensor_names: tensor_info = tf.saved_model.utils.build_tensor_info( - graph.get_tensor_by_name(output_tensor_names[output_name])) + graph.get_tensor_by_name(output_tensor_names[output_name]) + ) signature_outputs[output_name] = tensor_info prediction_signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=signature_inputs, - outputs=signature_outputs, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME - )) + tf.saved_model.signature_def_utils.build_signature_def( + inputs=signature_inputs, + outputs=signature_outputs, + method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME + ) + ) builder.add_meta_graph_and_variables( - sess, [tf.saved_model.tag_constants.SERVING], - signature_def_map={ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: - prediction_signature, - }) + sess, [tf.saved_model.tag_constants.SERVING], + signature_def_map={ + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + prediction_signature, + } + ) builder.save() config_path = os.path.join(model_dir, 'assets/pipeline.config') assert tf.gfile.Exists(config_path) @@ -260,25 +272,28 @@ def main(argv): model_dir = search_pb(FLAGS.model_dir) tf.logging.info('Loading meta graph...') meta_graph_def, variable_protos, input_tensor_names, output_tensor_names = load_meta_graph_def( - model_dir) + model_dir + ) tf.logging.info('Exporting user part model...') export( - model_dir, - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, - part_name='user', - part_dir=FLAGS.user_model_dir) + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name='user', + part_dir=FLAGS.user_model_dir + ) tf.logging.info('Exporting item part model...') export( - model_dir, - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, - part_name='item', - part_dir=FLAGS.item_model_dir) + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name='item', + part_dir=FLAGS.item_model_dir + ) if __name__ == '__main__': diff --git a/easy_rec/python/tools/split_pdn_model_pai.py b/easy_rec/python/tools/split_pdn_model_pai.py index 78932c297..8d4cb312b 100644 --- a/easy_rec/python/tools/split_pdn_model_pai.py +++ b/easy_rec/python/tools/split_pdn_model_pai.py @@ -3,11 +3,9 @@ import logging import os import sys - import tensorflow as tf from tensorflow.core.framework import graph_pb2 -from tensorflow.python.framework import importer -from tensorflow.python.framework import ops +from tensorflow.python.framework import importer, ops from tensorflow.python.framework.dtypes import _TYPE_TO_STRING from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model.utils_impl import get_variables_path @@ -22,7 +20,8 @@ tf.app.flags.DEFINE_string('sim_model_dir', '', '') logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) def search_pb(directory): @@ -123,7 +122,8 @@ def load_meta_graph_def(model_dir): variable_protos = {} meta_graph_def = saved_model_utils.get_meta_graph_def( - model_dir, tf.saved_model.tag_constants.SERVING) + model_dir, tf.saved_model.tag_constants.SERVING + ) signatures = meta_graph_def.signature_def collections = meta_graph_def.collection_def @@ -143,16 +143,19 @@ def load_meta_graph_def(model_dir): # parse signature info for SavedModel for sig_name in signatures: if signatures[ - sig_name].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: + sig_name + ].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: tf.logging.info('[Signature] inputs:') for input_name in signatures[sig_name].inputs: input_tensor_shape = [] input_tensor = signatures[sig_name].inputs[input_name] for dim in input_tensor.tensor_shape.dim: input_tensor_shape.append(int(dim.size)) - tf.logging.info('"%s": %s; %s' % - (input_name, _TYPE_TO_STRING[input_tensor.dtype], - input_tensor_shape)) + tf.logging.info( + '"%s": %s; %s' % ( + input_name, _TYPE_TO_STRING[input_tensor.dtype], input_tensor_shape + ) + ) input_tensor_names[input_name] = input_tensor.name tf.logging.info('[Signature] outputs:') for output_name in signatures[sig_name].outputs: @@ -160,16 +163,21 @@ def load_meta_graph_def(model_dir): output_tensor = signatures[sig_name].outputs[output_name] for dim in output_tensor.tensor_shape.dim: output_tensor_shape.append(int(dim.size)) - tf.logging.info('"%s": %s; %s' % - (output_name, _TYPE_TO_STRING[output_tensor.dtype], - output_tensor_shape)) + tf.logging.info( + '"%s": %s; %s' % ( + output_name, _TYPE_TO_STRING[output_tensor.dtype + ], output_tensor_shape + ) + ) output_tensor_names[output_name] = output_tensor.name return meta_graph_def, variable_protos, input_tensor_names, output_tensor_names -def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, - output_tensor_names, part_name, part_dir): +def export( + model_dir, meta_graph_def, variable_protos, input_tensor_names, + output_tensor_names, part_name, part_dir +): """Export subpart saved model. Args: @@ -182,16 +190,16 @@ def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, part_dir: subpart model export directory. """ output_tensor_names = { - x: output_tensor_names[x] - for x in output_tensor_names.keys() - if part_name in x + x: output_tensor_names[x] + for x in output_tensor_names.keys() if part_name in x } output_node_names = [ - _node_name(output_tensor_names[x]) for x in output_tensor_names.keys() + _node_name(output_tensor_names[x]) for x in output_tensor_names.keys() ] inference_graph, variables_to_keep = extract_sub_graph( - meta_graph_def.graph_def, output_node_names, variable_protos) + meta_graph_def.graph_def, output_node_names, variable_protos + ) tf.reset_default_graph() with tf.Session() as sess: @@ -209,7 +217,8 @@ def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, for input_name in input_tensor_names: try: tensor_info = tf.saved_model.utils.build_tensor_info( - graph.get_tensor_by_name(input_tensor_names[input_name])) + graph.get_tensor_by_name(input_tensor_names[input_name]) + ) signature_inputs[input_name] = tensor_info except Exception: print('ignore input: %s' % input_name) @@ -217,22 +226,25 @@ def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, signature_outputs = {} for output_name in output_tensor_names: tensor_info = tf.saved_model.utils.build_tensor_info( - graph.get_tensor_by_name(output_tensor_names[output_name])) + graph.get_tensor_by_name(output_tensor_names[output_name]) + ) signature_outputs[output_name] = tensor_info prediction_signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=signature_inputs, - outputs=signature_outputs, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME - )) + tf.saved_model.signature_def_utils.build_signature_def( + inputs=signature_inputs, + outputs=signature_outputs, + method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME + ) + ) builder.add_meta_graph_and_variables( - sess, [tf.saved_model.tag_constants.SERVING], - signature_def_map={ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: - prediction_signature, - }) + sess, [tf.saved_model.tag_constants.SERVING], + signature_def_map={ + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + prediction_signature, + } + ) builder.save() config_path = os.path.join(model_dir, 'assets/pipeline.config') assert tf.gfile.Exists(config_path) @@ -246,25 +258,28 @@ def main(argv): model_dir = search_pb(FLAGS.model_dir) tf.logging.info('Loading meta graph...') meta_graph_def, variable_protos, input_tensor_names, output_tensor_names = load_meta_graph_def( - model_dir) + model_dir + ) tf.logging.info('Exporting trigger part model...') export( - model_dir, - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, - part_name='trigger_out', - part_dir=FLAGS.trigger_model_dir) + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name='trigger_out', + part_dir=FLAGS.trigger_model_dir + ) tf.logging.info('Exporting sim part model...') export( - model_dir, - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, - part_name='sim_out', - part_dir=FLAGS.sim_model_dir) + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name='sim_out', + part_dir=FLAGS.sim_model_dir + ) if __name__ == '__main__': diff --git a/easy_rec/python/tools/test_saved_model.py b/easy_rec/python/tools/test_saved_model.py index 35889a668..bd3560ef7 100644 --- a/easy_rec/python/tools/test_saved_model.py +++ b/easy_rec/python/tools/test_saved_model.py @@ -3,17 +3,17 @@ import argparse import json import logging -import os - import numpy as np +import os import tensorflow as tf import easy_rec from easy_rec.python.inference.predictor import Predictor logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) lookup_op_path = os.path.join(easy_rec.ops_dir, 'libkv_lookup.so') lookup_op = tf.load_op_library(lookup_op_path) @@ -29,20 +29,26 @@ parser = argparse.ArgumentParser() parser.add_argument( - '--saved_model_dir', type=str, default=None, help='saved model dir') - parser.add_argument('--input_path', type=str, default=None, help='output dir') + '--saved_model_dir', type=str, default=None, help='saved model dir' + ) + parser.add_argument( + '--input_path', type=str, default=None, help='output dir' + ) parser.add_argument('--save_path', type=str, default=None, help='save path') parser.add_argument('--separator', type=str, default=',', help='separator') parser.add_argument( - '--cmp_res_path', type=str, default=None, help='compare result path') + '--cmp_res_path', type=str, default=None, help='compare result path' + ) parser.add_argument( - '--cmp_key', type=str, default='probs', help='compare key') + '--cmp_key', type=str, default='probs', help='compare key' + ) parser.add_argument('--tol', type=float, default=1e-5, help='tolerance') parser.add_argument( - '--with_lbl', - action='store_true', - default=False, - help='whether the test data has label field') + '--with_lbl', + action='store_true', + default=False, + help='whether the test data has label field' + ) args = parser.parse_args() logging.info('saved_model_dir: %s' % args.saved_model_dir) @@ -73,8 +79,8 @@ for line_id, line_str in enumerate(fin): line_str = line_str.strip() line_pred = json.loads(line_str) - assert np.abs( - line_pred[args.cmp_key] - - output[line_id][args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( - line_id, - np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key])) + assert np.abs(line_pred[args.cmp_key] - output[line_id][ + args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( + line_id, + np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]) + ) diff --git a/easy_rec/python/tools/view_saved_model.py b/easy_rec/python/tools/view_saved_model.py index 022bcf1aa..43cea4c6b 100644 --- a/easy_rec/python/tools/view_saved_model.py +++ b/easy_rec/python/tools/view_saved_model.py @@ -2,21 +2,23 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import argparse import logging - from google.protobuf import text_format from tensorflow.core.protobuf import saved_model_pb2 from tensorflow.python.platform.gfile import GFile logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--input', type=str, default=None, help='saved model path') + '--input', type=str, default=None, help='saved model path' + ) parser.add_argument( - '--output', type=str, default=None, help='saved model save path') + '--output', type=str, default=None, help='saved model save path' + ) args = parser.parse_args() assert args.input is not None and args.output is not None diff --git a/easy_rec/python/tools/write_kafka.py b/easy_rec/python/tools/write_kafka.py index 5dfa7dfd2..1c74d3b36 100644 --- a/easy_rec/python/tools/write_kafka.py +++ b/easy_rec/python/tools/write_kafka.py @@ -3,16 +3,15 @@ import argparse import logging import sys - # from kafka import KafkaConsumer -from kafka import KafkaAdminClient -from kafka import KafkaProducer +from kafka import KafkaAdminClient, KafkaProducer from kafka.admin import NewTopic # from kafka.structs import TopicPartition logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -38,21 +37,24 @@ admin_clt = KafkaAdminClient(bootstrap_servers=servers) if args.topic not in admin_clt.list_topics(): admin_clt.create_topics( - new_topics=[ - NewTopic( - name=args.topic, - num_partitions=1, - replication_factor=1, - topic_configs={'max.message.bytes': 1024 * 1024 * 1024}) - ], - validate_only=False) + new_topics=[ + NewTopic( + name=args.topic, + num_partitions=1, + replication_factor=1, + topic_configs={'max.message.bytes': 1024 * 1024 * 1024} + ) + ], + validate_only=False + ) logging.info('create increment save topic: %s' % args.topic) admin_clt.close() producer = KafkaProducer( - bootstrap_servers=servers, - request_timeout_ms=args.timeout * 1000, - api_version=(0, 10, 1)) + bootstrap_servers=servers, + request_timeout_ms=args.timeout * 1000, + api_version=(0, 10, 1) + ) i = 1 with open(args.input_path, 'r') as fin: diff --git a/easy_rec/python/train_eval.py b/easy_rec/python/train_eval.py index bafdf0c1a..fa24dce17 100644 --- a/easy_rec/python/train_eval.py +++ b/easy_rec/python/train_eval.py @@ -4,19 +4,12 @@ import json import logging import os - import tensorflow as tf from easy_rec.python.main import _train_and_evaluate_impl from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.utils import config_util -from easy_rec.python.utils import ds_util -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import fg_util -from easy_rec.python.utils import hpo_util -from easy_rec.python.utils.config_util import process_neg_sampler_data_path -from easy_rec.python.utils.config_util import set_eval_input_path -from easy_rec.python.utils.config_util import set_train_input_path +from easy_rec.python.utils import config_util, ds_util, estimator_utils, fg_util, hpo_util # NOQA +from easy_rec.python.utils.config_util import process_neg_sampler_data_path, set_eval_input_path, set_train_input_path # NOQA if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile @@ -29,87 +22,103 @@ tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO +) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--pipeline_config_path', - type=str, - default=None, - help='Path to pipeline config file.') - parser.add_argument( - '--continue_train', - action='store_true', - default=False, - help='continue train using existing model_dir') - parser.add_argument( - '--hpo_param_path', - type=str, - default=None, - help='hyperparam tuning param path') - parser.add_argument( - '--hpo_metric_save_path', - type=str, - default=None, - help='hyperparameter save metric path') - parser.add_argument( - '--model_dir', - type=str, - default=None, - help='will update the model_dir in pipeline_config') - parser.add_argument( - '--train_input_path', - type=str, - nargs='*', - default=None, - help='train data input path') - parser.add_argument( - '--eval_input_path', - type=str, - nargs='*', - default=None, - help='eval data input path') - parser.add_argument( - '--fit_on_eval', - action='store_true', - default=False, - help='Fit evaluation data after fitting and evaluating train data') - parser.add_argument( - '--fit_on_eval_steps', - type=int, - default=None, - help='Fit evaluation data steps') - parser.add_argument( - '--fine_tune_checkpoint', - type=str, - default=None, - help='will update the train_config.fine_tune_checkpoint in pipeline_config' - ) - parser.add_argument( - '--edit_config_json', - type=str, - default=None, - help='edit pipeline config str, example: {"model_dir":"experiments/",' - '"feature_config.feature[0].boundaries":[4,5,6,7]}') - parser.add_argument( - '--ignore_finetune_ckpt_error', - action='store_true', - default=False, - help='During incremental training, ignore the problem of missing fine_tune_checkpoint files' - ) - parser.add_argument( - '--odps_config', type=str, default=None, help='odps config path') - parser.add_argument( - '--is_on_ds', action='store_true', default=False, help='is on ds') - parser.add_argument( - '--check_mode', - action='store_true', - default=False, - help='is use check mode') - parser.add_argument( - '--selected_cols', type=str, default=None, help='select input columns') + '--pipeline_config_path', + type=str, + default=None, + help='Path to pipeline config file.' + ) + parser.add_argument( + '--continue_train', + action='store_true', + default=False, + help='continue train using existing model_dir' + ) + parser.add_argument( + '--hpo_param_path', + type=str, + default=None, + help='hyperparam tuning param path' + ) + parser.add_argument( + '--hpo_metric_save_path', + type=str, + default=None, + help='hyperparameter save metric path' + ) + parser.add_argument( + '--model_dir', + type=str, + default=None, + help='will update the model_dir in pipeline_config' + ) + parser.add_argument( + '--train_input_path', + type=str, + nargs='*', + default=None, + help='train data input path' + ) + parser.add_argument( + '--eval_input_path', + type=str, + nargs='*', + default=None, + help='eval data input path' + ) + parser.add_argument( + '--fit_on_eval', + action='store_true', + default=False, + help='Fit evaluation data after fitting and evaluating train data' + ) + parser.add_argument( + '--fit_on_eval_steps', + type=int, + default=None, + help='Fit evaluation data steps' + ) + parser.add_argument( + '--fine_tune_checkpoint', + type=str, + default=None, + help='will update the train_config.fine_tune_checkpoint in pipeline_config' + ) + parser.add_argument( + '--edit_config_json', + type=str, + default=None, + help='edit pipeline config str, example: {"model_dir":"experiments/",' + '"feature_config.feature[0].boundaries":[4,5,6,7]}' + ) + parser.add_argument( + '--ignore_finetune_ckpt_error', + action='store_true', + default=False, + help= + 'During incremental training, ignore the problem of missing fine_tune_checkpoint files' + ) + parser.add_argument( + '--odps_config', type=str, default=None, help='odps config path' + ) + parser.add_argument( + '--is_on_ds', action='store_true', default=False, help='is on ds' + ) + parser.add_argument( + '--check_mode', + action='store_true', + default=False, + help='is use check mode' + ) + parser.add_argument( + '--selected_cols', type=str, default=None, help='select input columns' + ) parser.add_argument('--gpu', type=str, default=None, help='gpu id') args, extra_args = parser.parse_known_args() @@ -125,7 +134,8 @@ if args.pipeline_config_path is not None: pipeline_config = config_util.get_configs_from_pipeline_file( - args.pipeline_config_path, False) + args.pipeline_config_path, False + ) if args.selected_cols: pipeline_config.data_config.selected_cols = args.selected_cols if args.model_dir: @@ -138,7 +148,8 @@ if args.fine_tune_checkpoint: ckpt_path = estimator_utils.get_latest_checkpoint_from_checkpoint_path( - args.fine_tune_checkpoint, args.ignore_finetune_ckpt_error) + args.fine_tune_checkpoint, args.ignore_finetune_ckpt_error + ) if ckpt_path: pipeline_config.train_config.fine_tune_checkpoint = ckpt_path @@ -151,10 +162,12 @@ if len(edit_config_json) > 0: fine_tune_checkpoint = edit_config_json.get('train_config', {}).get( - 'fine_tune_checkpoint', None) + 'fine_tune_checkpoint', None + ) if fine_tune_checkpoint: ckpt_path = estimator_utils.get_latest_checkpoint_from_checkpoint_path( - args.fine_tune_checkpoint, args.ignore_finetune_ckpt_error) + args.fine_tune_checkpoint, args.ignore_finetune_ckpt_error + ) edit_config_json['train_config']['fine_tune_checkpoint'] = ckpt_path config_util.edit_config(pipeline_config, edit_config_json) @@ -167,12 +180,12 @@ ds_util.cache_ckpt(pipeline_config) if pipeline_config.train_config.train_distribute in [ - DistributionStrategy.HorovodStrategy, + DistributionStrategy.HorovodStrategy, ]: estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ - DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy ]: estimator_utils.init_hvd() estimator_utils.init_sok() @@ -183,19 +196,22 @@ hpo_params = hpo_config['param'] config_util.edit_config(pipeline_config, hpo_params) config_util.auto_expand_share_feature_configs(pipeline_config) - _train_and_evaluate_impl(pipeline_config, args.continue_train, - args.check_mode) + _train_and_evaluate_impl( + pipeline_config, args.continue_train, args.check_mode + ) hpo_util.save_eval_metrics( - pipeline_config.model_dir, - metric_save_path=args.hpo_metric_save_path, - has_evaluator=False) + pipeline_config.model_dir, + metric_save_path=args.hpo_metric_save_path, + has_evaluator=False + ) else: config_util.auto_expand_share_feature_configs(pipeline_config) _train_and_evaluate_impl( - pipeline_config, - args.continue_train, - args.check_mode, - fit_on_eval=args.fit_on_eval, - fit_on_eval_steps=args.fit_on_eval_steps) + pipeline_config, + args.continue_train, + args.check_mode, + fit_on_eval=args.fit_on_eval, + fit_on_eval_steps=args.fit_on_eval_steps + ) else: raise ValueError('pipeline_config_path should not be empty when training!') diff --git a/easy_rec/python/utils/activation.py b/easy_rec/python/utils/activation.py index 89044f7a3..7f4c296db 100644 --- a/easy_rec/python/utils/activation.py +++ b/easy_rec/python/utils/activation.py @@ -27,17 +27,19 @@ def dice(_x, axis=-1, epsilon=1e-9, name='dice', training=True): ACM, 2018: 1059-1068.] (https://arxiv.org/pdf/1706.06978.pdf) """ alphas = tf.get_variable( - 'alpha_' + name, - _x.get_shape()[-1], - initializer=tf.constant_initializer(0.0), - dtype=tf.float32) + 'alpha_' + name, + _x.get_shape()[-1], + initializer=tf.constant_initializer(0.0), + dtype=tf.float32 + ) inputs_normed = tf.layers.batch_normalization( - inputs=_x, - axis=axis, - epsilon=epsilon, - center=False, - scale=False, - training=training) + inputs=_x, + axis=axis, + epsilon=epsilon, + center=False, + scale=False, + training=training + ) x_p = tf.sigmoid(inputs_normed) return alphas * (1.0 - x_p) * _x + x_p * _x @@ -56,8 +58,9 @@ def gelu(x, name='gelu'): `x` with the GELU activation applied. """ with tf.name_scope(name): - cdf = 0.5 * (1.0 + tf.tanh( - (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + cdf = 0.5 * ( + 1.0 + tf.tanh((np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3)))) + ) return x * cdf diff --git a/easy_rec/python/utils/check_utils.py b/easy_rec/python/utils/check_utils.py index 5a7551745..2c6282643 100644 --- a/easy_rec/python/utils/check_utils.py +++ b/easy_rec/python/utils/check_utils.py @@ -10,8 +10,9 @@ def check_split(line, sep, requried_field_num, field_name=''): - assert sep, 'must have separator.' + (' field: %s.' % - field_name) if field_name else '' + assert sep, 'must have separator.' + ( + ' field: %s.' % field_name + ) if field_name else '' for one_line in line: field_num = len(one_line.split(sep)) @@ -62,12 +63,12 @@ def check_env_and_input_path(pipeline_config, input_path): input_type = pipeline_config.data_config.input_type input_type_name = DatasetConfig.InputType.Name(input_type) ignore_input_list = [ - DatasetConfig.InputType.TFRecordInput, - DatasetConfig.InputType.BatchTFRecordInput, - DatasetConfig.InputType.KafkaInput, - DatasetConfig.InputType.DataHubInput, - DatasetConfig.InputType.HiveInput, - DatasetConfig.InputType.DummyInput, + DatasetConfig.InputType.TFRecordInput, + DatasetConfig.InputType.BatchTFRecordInput, + DatasetConfig.InputType.KafkaInput, + DatasetConfig.InputType.DataHubInput, + DatasetConfig.InputType.HiveInput, + DatasetConfig.InputType.DummyInput, ] if input_type in ignore_input_list: return True diff --git a/easy_rec/python/utils/config_util.py b/easy_rec/python/utils/config_util.py index 3c6f385e7..53171dbae 100644 --- a/easy_rec/python/utils/config_util.py +++ b/easy_rec/python/utils/config_util.py @@ -8,15 +8,13 @@ import datetime import json import logging +import numpy as np import os import re -import sys - -import numpy as np import six +import sys import tensorflow as tf -from google.protobuf import json_format -from google.protobuf import text_format +from google.protobuf import json_format, text_format from tensorflow.python.lib.io import file_io from easy_rec.python.protos import pipeline_pb2 @@ -59,7 +57,7 @@ def get_configs_from_pipeline_file(pipeline_config_path, auto_expand=True): return pipeline_config_path assert tf.gfile.Exists( - pipeline_config_path + pipeline_config_path ), 'pipeline_config_path [%s] not exists' % pipeline_config_path pipeline_config = pipeline_pb2.EasyRecConfig() @@ -158,9 +156,9 @@ def create_pipeline_proto_from_configs(configs): return pipeline_config -def save_pipeline_config(pipeline_config, - directory, - filename='pipeline.config'): +def save_pipeline_config( + pipeline_config, directory, filename='pipeline.config' +): """Saves a pipeline config text file to disk. Args: @@ -178,10 +176,10 @@ def save_pipeline_config(pipeline_config, def _get_basic_types(): dtypes = [ - bool, int, str, float, - type(u''), np.float16, np.float32, np.float64, np.char, np.byte, np.uint8, - np.int8, np.int16, np.uint16, np.uint32, np.int32, np.uint64, np.int64, - bool, str + bool, int, str, float, + type(u''), np.float16, np.float32, np.float64, np.char, np.byte, np.uint8, + np.int8, np.int16, np.uint16, np.uint32, np.int32, np.uint64, np.int64, + bool, str ] if six.PY2: dtypes.append(long) # noqa: F821 @@ -265,11 +263,11 @@ def _get_attr(obj, attr, only_last=False): # for complex conditions a[optimizer.lr=20] op_func_map = { - '>=': lambda x, y: x >= y, - '<=': lambda x, y: x <= y, - '<': lambda x, y: x < y, - '>': lambda x, y: x > y, - '=': lambda x, y: x == y + '>=': lambda x, y: x >= y, + '<=': lambda x, y: x <= y, + '<': lambda x, y: x < y, + '>': lambda x, y: x > y, + '=': lambda x, y: x == y } cond_key = None cond_val = None @@ -287,7 +285,8 @@ def _get_attr(obj, attr, only_last=False): for tid, update_obj in enumerate(update_objs): tmp, tmp_parent, _, _ = _get_attr( - update_obj, cond_key, only_last=True) + update_obj, cond_key, only_last=True + ) cond_val = _type_convert(tmp, cond_val, tmp_parent) @@ -381,8 +380,10 @@ def add_boundaries_to_config(pipeline_config, tables): if feature_name in feature_boundaries_info: if feature_config.feature_type != feature_config.SequenceFeature: logging.info( - 'feature = {0}, type = {1}, will turn to RawFeature.'.format( - feature_name, feature_config.feature_type)) + 'feature = {0}, type = {1}, will turn to RawFeature.'.format( + feature_name, feature_config.feature_type + ) + ) feature_config.feature_type = feature_config.RawFeature feature_config.hash_bucket_size = 0 feature_config.ClearField('boundaries') @@ -409,8 +410,9 @@ def parse_time(time_data): if isinstance(time_data, str) or isinstance(time_data, type(u'')): if len(time_data) == 17: return int( - datetime.datetime.strptime(time_data, - '%Y%m%d %H:%M:%S').strftime('%s')) + datetime.datetime.strptime(time_data, + '%Y%m%d %H:%M:%S').strftime('%s') + ) elif len(time_data) == 10: return int(time_data) else: @@ -473,16 +475,18 @@ def set_train_input_path(pipeline_config, train_input_path): if pipeline_config.WhichOneof('train_path') == 'hive_train_input': if isinstance(train_input_path, list): assert len( - train_input_path + train_input_path ) <= 1, 'only support one hive_train_input.table_name when hive input' pipeline_config.hive_train_input.table_name = train_input_path[0] else: assert len( - train_input_path.split(',') + train_input_path.split(',') ) <= 1, 'only support one hive_train_input.table_name when hive input' pipeline_config.hive_train_input.table_name = train_input_path - logging.info('update hive_train_input.table_name to %s' % - pipeline_config.hive_train_input.table_name) + logging.info( + 'update hive_train_input.table_name to %s' % + pipeline_config.hive_train_input.table_name + ) elif pipeline_config.WhichOneof('train_path') == 'kafka_train_input': if isinstance(train_input_path, list): @@ -499,8 +503,9 @@ def set_train_input_path(pipeline_config, train_input_path): pipeline_config.train_input_path = ','.join(train_input_path) else: pipeline_config.train_input_path = train_input_path - logging.info('update train_input_path to %s' % - pipeline_config.train_input_path) + logging.info( + 'update train_input_path to %s' % pipeline_config.train_input_path + ) return pipeline_config @@ -508,16 +513,18 @@ def set_eval_input_path(pipeline_config, eval_input_path): if pipeline_config.WhichOneof('eval_path') == 'hive_eval_input': if isinstance(eval_input_path, list): assert len( - eval_input_path + eval_input_path ) <= 1, 'only support one hive_eval_input.table_name when hive input' pipeline_config.hive_eval_input.table_name = eval_input_path[0] else: assert len( - eval_input_path.split(',') + eval_input_path.split(',') ) <= 1, 'only support one hive_eval_input.table_name when hive input' pipeline_config.hive_eval_input.table_name = eval_input_path - logging.info('update hive_eval_input.table_name to %s' % - pipeline_config.hive_eval_input.table_name) + logging.info( + 'update hive_eval_input.table_name to %s' % + pipeline_config.hive_eval_input.table_name + ) elif pipeline_config.WhichOneof('eval_path') == 'parquet_eval_input': if isinstance(eval_input_path, list): pipeline_config.parquet_eval_input = ','.join(eval_input_path) @@ -533,8 +540,9 @@ def set_eval_input_path(pipeline_config, eval_input_path): pipeline_config.eval_input_path = ','.join(eval_input_path) else: pipeline_config.eval_input_path = eval_input_path - logging.info('update eval_input_path to %s' % - pipeline_config.eval_input_path) + logging.info( + 'update eval_input_path to %s' % pipeline_config.eval_input_path + ) return pipeline_config @@ -559,25 +567,31 @@ def process_neg_sampler_data_path(pipeline_config): if pipeline_config.WhichOneof('train_path') != 'hive_train_input': return hive_util = HiveUtils( - data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input) + data_config=pipeline_config.data_config, + hive_config=pipeline_config.hive_train_input + ) sampler_type = pipeline_config.data_config.WhichOneof('sampler') sampler_config = getattr(pipeline_config.data_config, sampler_type) if hasattr(sampler_config, 'input_path'): - sampler_config.input_path = process_data_path(sampler_config.input_path, - hive_util) + sampler_config.input_path = process_data_path( + sampler_config.input_path, hive_util + ) if hasattr(sampler_config, 'user_input_path'): sampler_config.user_input_path = process_data_path( - sampler_config.user_input_path, hive_util) + sampler_config.user_input_path, hive_util + ) if hasattr(sampler_config, 'item_input_path'): sampler_config.item_input_path = process_data_path( - sampler_config.item_input_path, hive_util) + sampler_config.item_input_path, hive_util + ) if hasattr(sampler_config, 'pos_edge_input_path'): sampler_config.pos_edge_input_path = process_data_path( - sampler_config.pos_edge_input_path, hive_util) + sampler_config.pos_edge_input_path, hive_util + ) if hasattr(sampler_config, 'hard_neg_edge_input_path'): sampler_config.hard_neg_edge_input_path = process_data_path( - sampler_config.hard_neg_edge_input_path, hive_util) + sampler_config.hard_neg_edge_input_path, hive_util + ) def parse_extra_config_param(extra_args, edit_config_json): @@ -613,8 +627,9 @@ def process_multi_file_input_path(sampler_config_input_path): if '*' in sampler_config_input_path: input_path = ','.join( - file_path - for file_path in tf.gfile.Glob(sampler_config_input_path.split(','))) + file_path + for file_path in tf.gfile.Glob(sampler_config_input_path.split(',')) + ) else: input_path = sampler_config_input_path diff --git a/easy_rec/python/utils/constant.py b/easy_rec/python/utils/constant.py index 84366fbf5..7b886d3d3 100644 --- a/easy_rec/python/utils/constant.py +++ b/easy_rec/python/utils/constant.py @@ -35,8 +35,8 @@ def enable_avx_str_split(): def has_avx_str_split(): - return ENABLE_AVX_STR_SPLIT in os.environ and os.environ[ - ENABLE_AVX_STR_SPLIT] == '1' + return ENABLE_AVX_STR_SPLIT in os.environ and os.environ[ENABLE_AVX_STR_SPLIT + ] == '1' def disable_avx_str_split(): diff --git a/easy_rec/python/utils/convert_rtp_fg.py b/easy_rec/python/utils/convert_rtp_fg.py index d665fcd74..cd96298d2 100644 --- a/easy_rec/python/utils/convert_rtp_fg.py +++ b/easy_rec/python/utils/convert_rtp_fg.py @@ -3,15 +3,12 @@ import json import logging import sys -import traceback - import tensorflow as tf +import traceback from google.protobuf import text_format from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.protos.feature_config_pb2 import FeatureConfig -from easy_rec.python.protos.feature_config_pb2 import FeatureGroupConfig -from easy_rec.python.protos.feature_config_pb2 import WideOrDeep +from easy_rec.python.protos.feature_config_pb2 import FeatureConfig, FeatureGroupConfig, WideOrDeep # NOQA from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig from easy_rec.python.utils import config_util @@ -21,8 +18,9 @@ MAX_HASH_BUCKET_SIZE = 9223372036854775807 -def _gen_raw_config(feature, input_field, feature_config, is_multi, - curr_embed_dim): +def _gen_raw_config( + feature, input_field, feature_config, is_multi, curr_embed_dim +): if 'bucketize_boundaries' in feature: if is_multi: input_field.input_type = DatasetConfig.STRING @@ -31,7 +29,8 @@ def _gen_raw_config(feature, input_field, feature_config, is_multi, input_field.input_type = DatasetConfig.INT32 feature_config.feature_type = feature_config.IdFeature feature_config.num_buckets = len( - feature['bucketize_boundaries'].split(',')) + 1 + feature['bucketize_boundaries'].split(',') + ) + 1 feature_config.embedding_dim = curr_embed_dim else: feature_config.feature_type = feature_config.RawFeature @@ -57,8 +56,9 @@ def _set_hash_bucket(feature, feature_config, input_field): if feature_config.hash_bucket_size > 10000000: if 'max_partitions' not in feature: logging.error( - 'it is suggested to set max_partitions > 1 for large hash buckets[%s]' - % feature['feature_name']) + 'it is suggested to set max_partitions > 1 for large hash buckets[%s]' + % feature['feature_name'] + ) sys.exit(1) if feature.get('filter_freq', -1) >= 0: feature_config.ev_params.filter_freq = feature['filter_freq'] @@ -77,20 +77,23 @@ def _set_hash_bucket(feature, feature_config, input_field): assert False, 'one of hash_bucket_size,vocab_file,vocab_list,num_buckets must be set' -def process_features(feature_type, - feature_name, - feature, - pipeline_config, - embedding_dim, - incol_separator, - is_sequence=False): +def process_features( + feature_type, + feature_name, + feature, + pipeline_config, + embedding_dim, + incol_separator, + is_sequence=False +): feature_config = FeatureConfig() feature_config.input_names.append(feature_name) feature_config.separator = incol_separator input_field = DatasetConfig.Field() input_field.input_name = feature_name - curr_embed_dim = feature.get('embedding_dimension', - feature.get('embedding_dim', embedding_dim)) + curr_embed_dim = feature.get( + 'embedding_dimension', feature.get('embedding_dim', embedding_dim) + ) curr_combiner = feature.get('combiner', 'sum') if feature.get('is_cache', False): logging.info('will cache %s' % feature_name) @@ -128,8 +131,9 @@ def process_features(feature_type, elif feature_type == 'lookup_feature': need_discrete = feature.get('needDiscrete', True) if not need_discrete: - _gen_raw_config(feature, input_field, feature_config, is_multi, - curr_embed_dim) + _gen_raw_config( + feature, input_field, feature_config, is_multi, curr_embed_dim + ) else: feature_config.feature_type = feature_config.TagFeature if feature.get('needWeighting', False): @@ -138,8 +142,9 @@ def process_features(feature_type, _set_hash_bucket(feature, feature_config, input_field) feature_config.combiner = curr_combiner elif feature_type == 'raw_feature': - _gen_raw_config(feature, input_field, feature_config, is_multi, - curr_embed_dim) + _gen_raw_config( + feature, input_field, feature_config, is_multi, curr_embed_dim + ) elif feature_type == 'match_feature': need_discrete = feature.get('needDiscrete', True) if feature.get('matchType', '') == 'multihit': @@ -153,8 +158,9 @@ def process_features(feature_type, feature_config.combiner = curr_combiner else: assert 'bucketize_boundaries' not in feature - _gen_raw_config(feature, input_field, feature_config, is_multi, - curr_embed_dim) + _gen_raw_config( + feature, input_field, feature_config, is_multi, curr_embed_dim + ) elif feature_type == 'combo_feature': feature_config.feature_type = feature_config.TagFeature _set_hash_bucket(feature, feature_config, input_field) @@ -183,7 +189,7 @@ def process_features(feature_type, extra_combo_info = feature['extra_combo_info'] feature_names = extra_combo_info.get('feature_names', []) assert len( - feature_names + feature_names ) >= 1, 'The feature number for ComboFeature must be greater than 2.' combo_feature_config = FeatureConfig() combo_feature_config.input_names.append(feature_name) @@ -192,13 +198,15 @@ def process_features(feature_type, combo_feature_config.input_names.append(fea_name) final_feature_name = 'combo__' + '_'.join(combo_feature_config.input_names) - final_feature_name = extra_combo_info.get('final_feature_name', - final_feature_name) + final_feature_name = extra_combo_info.get( + 'final_feature_name', final_feature_name + ) combo_feature_config.feature_name = final_feature_name combo_feature_config.feature_type = combo_feature_config.ComboFeature curr_embed_dim = extra_combo_info.get( - 'embedding_dimension', - extra_combo_info.get('embedding_dim', embedding_dim)) + 'embedding_dimension', + extra_combo_info.get('embedding_dim', embedding_dim) + ) curr_combiner = extra_combo_info.get('combiner', 'mean') combo_feature_config.embedding_dim = curr_embed_dim combo_feature_config.combiner = curr_combiner @@ -212,10 +220,9 @@ def process_features(feature_type, return pipeline_config -def load_input_field_and_feature_config(rtp_fg, - label_fields, - embedding_dim=16, - incol_separator='\003'): +def load_input_field_and_feature_config( + rtp_fg, label_fields, embedding_dim=16, incol_separator='\003' +): embedding_dim = rtp_fg.get('embedding_dim', embedding_dim) logging.info('embedding_dim = %s' % embedding_dim) logging.info('label_fields = %s' % ','.join(label_fields)) @@ -237,9 +244,10 @@ def load_input_field_and_feature_config(rtp_fg, if 'feature_name' in feature: feature_type = feature['feature_type'] feature_name = feature['feature_name'] - pipeline_config = process_features(feature_type, feature_name, feature, - pipeline_config, embedding_dim, - incol_separator) + pipeline_config = process_features( + feature_type, feature_name, feature, pipeline_config, embedding_dim, + incol_separator + ) elif 'sequence_name' in feature: logging.info('Set sequence_features group later.') sequence_name = feature['sequence_name'] @@ -248,33 +256,38 @@ def load_input_field_and_feature_config(rtp_fg, sub_feature_name = sub_feature['feature_name'] all_sub_feature_name = sequence_name + '_' + sub_feature_name pipeline_config = process_features( - sub_feature_type, - all_sub_feature_name, - sub_feature, - pipeline_config, - embedding_dim, - incol_separator, - is_sequence=True) + sub_feature_type, + all_sub_feature_name, + sub_feature, + pipeline_config, + embedding_dim, + incol_separator, + is_sequence=True + ) except Exception: - logging.info('convert feature[%s] exception[%s]' % - (str(feature), traceback.format_exc())) + logging.info( + 'convert feature[%s] exception[%s]' % + (str(feature), traceback.format_exc()) + ) sys.exit(1) return pipeline_config -def convert_rtp_fg(rtp_fg, - embedding_dim=16, - batch_size=1024, - label_fields=[], - num_steps=10, - model_type='', - separator='\002', - incol_separator='\003', - train_input_path=None, - eval_input_path=None, - selected_cols='', - input_type='OdpsRTPInput', - is_async=False): +def convert_rtp_fg( + rtp_fg, + embedding_dim=16, + batch_size=1024, + label_fields=[], + num_steps=10, + model_type='', + separator='\002', + incol_separator='\003', + train_input_path=None, + eval_input_path=None, + selected_cols='', + input_type='OdpsRTPInput', + is_async=False +): with tf.gfile.GFile(rtp_fg, 'r') as fin: rtp_fg = json.load(fin) @@ -292,9 +305,9 @@ def convert_rtp_fg(rtp_fg, logging.info('model_path = %s' % model_path) logging.info('edit_config_json = %s' % edit_config_json) - pipeline_config = load_input_field_and_feature_config(rtp_fg, label_fields, - embedding_dim, - incol_separator) + pipeline_config = load_input_field_and_feature_config( + rtp_fg, label_fields, embedding_dim, incol_separator + ) pipeline_config.model_dir = model_dir pipeline_config.data_config.separator = separator if selected_cols: @@ -336,8 +349,10 @@ def convert_rtp_fg(rtp_fg, sync_replicas: %s } - """ % ('adam_optimizer' if not is_async else 'adam_async_optimizer', - 'true' if not is_async else 'false') + """ % ( + 'adam_optimizer' if not is_async else 'adam_async_optimizer', + 'true' if not is_async else 'false' + ) text_format.Merge(train_config_str, pipeline_config) pipeline_config.train_config.num_steps = num_steps @@ -410,11 +425,11 @@ def convert_rtp_fg(rtp_fg, feature_groups = {} group_map = { - 'u': 'user', - 'i': 'item', - 'ctx': 'combo', - 'q': 'combo', - 'comb': 'combo' + 'u': 'user', + 'i': 'item', + 'ctx': 'combo', + 'q': 'combo', + 'comb': 'combo' } for feature in rtp_features: feature_name = feature['feature_name'].strip() @@ -432,8 +447,8 @@ def convert_rtp_fg(rtp_fg, feature_groups[group_name] = [feature_name] logging.info( - 'if group is specified, group will be used as feature group name; ' - 'otherwise, the prefix of feature_name in fg.json is used as feature group name' + 'if group is specified, group will be used as feature group name; ' + 'otherwise, the prefix of feature_name in fg.json is used as feature group name' ) logging.info('prefix map: %s' % str(group_map)) for group_name in feature_groups: @@ -596,17 +611,20 @@ def convert_rtp_fg(rtp_fg, pipeline_config.model_config.embedding_regularization = 5e-6 if model_type in ['wide_and_deep', 'deepfm', 'multi_tower']: - text_format.Merge(""" + text_format.Merge( + """ metrics_set { auc {} } - """, pipeline_config.eval_config) + """, pipeline_config.eval_config + ) text_format.Merge( - """ export_config { + """ export_config { multi_placeholder: false } - """, pipeline_config) + """, pipeline_config + ) if edit_config_json: for edit_obj in edit_config_json: diff --git a/easy_rec/python/utils/dag.py b/easy_rec/python/utils/dag.py index 8d0d4b094..f16191997 100644 --- a/easy_rec/python/utils/dag.py +++ b/easy_rec/python/utils/dag.py @@ -1,8 +1,6 @@ import logging -from collections import OrderedDict -from collections import defaultdict -from copy import copy -from copy import deepcopy +from collections import OrderedDict, defaultdict +from copy import copy, deepcopy class DAG(object): @@ -113,8 +111,10 @@ def all_downstreams(self, node, graph=None): nodes.append(downstream_node) i += 1 return list( - filter(lambda node: node in nodes_seen, - self.topological_sort(graph=graph))) + filter( + lambda node: node in nodes_seen, self.topological_sort(graph=graph) + ) + ) def all_leaves(self, graph=None): """Return a list of all leaves (nodes with no downstreams).""" @@ -146,7 +146,8 @@ def independent_nodes(self, graph=None): graph = self.graph dependent_nodes = set( - node for dependents in graph.values() for node in dependents) + node for dependents in graph.values() for node in dependents + ) return [node for node in graph.keys() if node not in dependent_nodes] def validate(self, graph=None): diff --git a/easy_rec/python/utils/distribution_utils.py b/easy_rec/python/utils/distribution_utils.py index 22ae9ecf8..8a8d35dac 100644 --- a/easy_rec/python/utils/distribution_utils.py +++ b/easy_rec/python/utils/distribution_utils.py @@ -5,29 +5,28 @@ import json import logging import os - import tensorflow as tf from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils.estimator_utils import chief_to_master -from easy_rec.python.utils.estimator_utils import master_to_chief +from easy_rec.python.utils.estimator_utils import chief_to_master, master_to_chief # NOQA DistributionStrategyMap = { - '': DistributionStrategy.NoStrategy, - 'ps': DistributionStrategy.PSStrategy, - 'ess': DistributionStrategy.ExascaleStrategy, - 'mirrored': DistributionStrategy.MirroredStrategy, - 'collective': DistributionStrategy.CollectiveAllReduceStrategy + '': DistributionStrategy.NoStrategy, + 'ps': DistributionStrategy.PSStrategy, + 'ess': DistributionStrategy.ExascaleStrategy, + 'mirrored': DistributionStrategy.MirroredStrategy, + 'collective': DistributionStrategy.CollectiveAllReduceStrategy } -def set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, - distribute_strategy): +def set_distribution_config( + pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy +): if distribute_strategy in [ - DistributionStrategy.PSStrategy, DistributionStrategy.MirroredStrategy, - DistributionStrategy.CollectiveAllReduceStrategy, - DistributionStrategy.ExascaleStrategy + DistributionStrategy.PSStrategy, DistributionStrategy.MirroredStrategy, + DistributionStrategy.CollectiveAllReduceStrategy, + DistributionStrategy.ExascaleStrategy ]: pipeline_config.train_config.sync_replicas = False pipeline_config.train_config.train_distribute = distribute_strategy @@ -37,15 +36,17 @@ def set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, def set_tf_config_and_get_train_worker_num( - ps_hosts, - worker_hosts, - task_index, - job_name, - distribute_strategy=DistributionStrategy.NoStrategy, - eval_method='none'): + ps_hosts, + worker_hosts, + task_index, + job_name, + distribute_strategy=DistributionStrategy.NoStrategy, + eval_method='none' +): logging.info( - 'set_tf_config_and_get_train_worker_num: distribute_strategy = %d' % - distribute_strategy) + 'set_tf_config_and_get_train_worker_num: distribute_strategy = %d' % + distribute_strategy + ) worker_hosts = worker_hosts.split(',') ps_hosts = ps_hosts.split(',') if ps_hosts else [] @@ -53,16 +54,18 @@ def set_tf_config_and_get_train_worker_num( train_worker_num = total_worker_num print('Original TF_CONFIG=%s' % os.environ.get('TF_CONFIG', '')) - print('worker_hosts=%s ps_hosts=%s task_index=%d job_name=%s' % - (','.join(worker_hosts), ','.join(ps_hosts), task_index, job_name)) + print( + 'worker_hosts=%s ps_hosts=%s task_index=%d job_name=%s' % + (','.join(worker_hosts), ','.join(ps_hosts), task_index, job_name) + ) print('eval_method=%s' % eval_method) if distribute_strategy == DistributionStrategy.MirroredStrategy: assert total_worker_num == 1, 'mirrored distribute strategy only need 1 worker' elif distribute_strategy in [ - DistributionStrategy.NoStrategy, DistributionStrategy.PSStrategy, - DistributionStrategy.CollectiveAllReduceStrategy, - DistributionStrategy.ExascaleStrategy + DistributionStrategy.NoStrategy, DistributionStrategy.PSStrategy, + DistributionStrategy.CollectiveAllReduceStrategy, + DistributionStrategy.ExascaleStrategy ]: cluster, task_type, task_index_ = estimator_utils.parse_tf_config() train_worker_num = 0 @@ -79,11 +82,11 @@ def set_tf_config_and_get_train_worker_num( if distribute_strategy == DistributionStrategy.NoStrategy: del cluster['evaluator'] tf_config = { - 'cluster': cluster, - 'task': { - 'type': task_type, - 'index': task_index_ - } + 'cluster': cluster, + 'task': { + 'type': task_type, + 'index': task_index_ + } } os.environ['TF_CONFIG'] = json.dumps(tf_config) else: @@ -99,38 +102,46 @@ def set_tf_config_and_get_train_worker_num( if len(ps_hosts) > 0: cluster['ps'] = ps_hosts if job_name == 'ps': - os.environ['TF_CONFIG'] = json.dumps({ + os.environ['TF_CONFIG'] = json.dumps( + { 'cluster': cluster, 'task': { - 'type': job_name, - 'index': task_index + 'type': job_name, + 'index': task_index } - }) + } + ) elif job_name == 'worker': if task_index == 0: - os.environ['TF_CONFIG'] = json.dumps({ + os.environ['TF_CONFIG'] = json.dumps( + { 'cluster': cluster, 'task': { - 'type': 'chief', - 'index': 0 + 'type': 'chief', + 'index': 0 } - }) + } + ) elif task_index == 1: - os.environ['TF_CONFIG'] = json.dumps({ + os.environ['TF_CONFIG'] = json.dumps( + { 'cluster': cluster, 'task': { - 'type': 'evaluator', - 'index': 0 + 'type': 'evaluator', + 'index': 0 } - }) + } + ) else: - os.environ['TF_CONFIG'] = json.dumps({ + os.environ['TF_CONFIG'] = json.dumps( + { 'cluster': cluster, 'task': { - 'type': job_name, - 'index': task_index - 2 + 'type': job_name, + 'index': task_index - 2 } - }) + } + ) else: if 'evaluator' in cluster: evaluator = cluster['evaluator'] @@ -148,19 +159,19 @@ def set_tf_config_and_get_train_worker_num( cluster['worker'] = [evaluator[0]] if task_type == 'evaluator': tf_config = { - 'cluster': cluster, - 'task': { - 'type': 'worker', - 'index': train_worker_num - 2 - } + 'cluster': cluster, + 'task': { + 'type': 'worker', + 'index': train_worker_num - 2 + } } else: tf_config = { - 'cluster': cluster, - 'task': { - 'type': task_type, - 'index': task_index_ - } + 'cluster': cluster, + 'task': { + 'type': task_type, + 'index': task_index_ + } } os.environ['TF_CONFIG'] = json.dumps(tf_config) else: @@ -169,30 +180,36 @@ def set_tf_config_and_get_train_worker_num( if len(ps_hosts) > 0: cluster['ps'] = ps_hosts if job_name == 'ps': - os.environ['TF_CONFIG'] = json.dumps({ + os.environ['TF_CONFIG'] = json.dumps( + { 'cluster': cluster, 'task': { - 'type': job_name, - 'index': task_index + 'type': job_name, + 'index': task_index } - }) + } + ) else: if task_index == 0: - os.environ['TF_CONFIG'] = json.dumps({ + os.environ['TF_CONFIG'] = json.dumps( + { 'cluster': cluster, 'task': { - 'type': 'chief', - 'index': 0 + 'type': 'chief', + 'index': 0 } - }) + } + ) else: - os.environ['TF_CONFIG'] = json.dumps({ + os.environ['TF_CONFIG'] = json.dumps( + { 'cluster': cluster, 'task': { - 'type': 'worker', - 'index': task_index - 1 + 'type': 'worker', + 'index': task_index - 1 } - }) + } + ) if eval_method == 'none': # change master to chief, will not evaluate master_to_chief() @@ -205,8 +222,10 @@ def set_tf_config_and_get_train_worker_num( cluster, task_type, task_index = estimator_utils.parse_tf_config() print('Final TF_CONFIG = %s' % os.environ.get('TF_CONFIG', '')) tf.logging.info('TF_CONFIG %s' % os.environ.get('TF_CONFIG', '')) - tf.logging.info('distribute_stategy %s, train_worker_num: %d' % - (distribute_strategy, train_worker_num)) + tf.logging.info( + 'distribute_stategy %s, train_worker_num: %d' % + (distribute_strategy, train_worker_num) + ) # remove pai chief-worker waiting strategy # which is conflicted with worker waiting strategy in easyrec @@ -220,7 +239,8 @@ def set_tf_config_and_get_train_worker_num_on_ds(): return tf_config = json.loads(os.environ['TF_CONFIG']) if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and ( - 'evaluator' not in tf_config['cluster']): + 'evaluator' not in tf_config['cluster'] + ): easyrec_tf_config = dict() easyrec_tf_config['cluster'] = {} easyrec_tf_config['task'] = {} @@ -228,11 +248,12 @@ def set_tf_config_and_get_train_worker_num_on_ds(): easyrec_tf_config['cluster']['chief'] = [tf_config['cluster']['worker'][0]] easyrec_tf_config['cluster']['worker'] = tf_config['cluster']['worker'][2:] - if tf_config['task']['type'] == 'worker' and tf_config['task']['index'] == 0: + if tf_config['task']['type'] == 'worker' and tf_config['task']['index' + ] == 0: easyrec_tf_config['task']['type'] = 'chief' easyrec_tf_config['task']['index'] = 0 - elif tf_config['task']['type'] == 'worker' and tf_config['task'][ - 'index'] == 1: + elif tf_config['task']['type'] == 'worker' and tf_config['task']['index' + ] == 1: easyrec_tf_config['task']['type'] = 'evaluator' easyrec_tf_config['task']['index'] = 0 elif tf_config['task']['type'] == 'worker': @@ -248,7 +269,8 @@ def set_tf_config_and_get_distribute_eval_worker_num_on_ds(): assert 'TF_CONFIG' in os.environ, "'TF_CONFIG' must in os.environ" tf_config = json.loads(os.environ['TF_CONFIG']) if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and ( - 'evaluator' not in tf_config['cluster']): + 'evaluator' not in tf_config['cluster'] + ): easyrec_tf_config = dict() easyrec_tf_config['cluster'] = {} easyrec_tf_config['task'] = {} @@ -256,7 +278,8 @@ def set_tf_config_and_get_distribute_eval_worker_num_on_ds(): easyrec_tf_config['cluster']['chief'] = [tf_config['cluster']['worker'][0]] easyrec_tf_config['cluster']['worker'] = tf_config['cluster']['worker'][1:] - if tf_config['task']['type'] == 'worker' and tf_config['task']['index'] == 0: + if tf_config['task']['type'] == 'worker' and tf_config['task']['index' + ] == 0: easyrec_tf_config['task']['type'] = 'chief' easyrec_tf_config['task']['index'] = 0 elif tf_config['task']['type'] == 'worker': diff --git a/easy_rec/python/utils/ds_util.py b/easy_rec/python/utils/ds_util.py index 883e7bcee..b1b74f9c4 100644 --- a/easy_rec/python/utils/ds_util.py +++ b/easy_rec/python/utils/ds_util.py @@ -4,7 +4,6 @@ import os import subprocess import traceback - from tensorflow.python.platform import gfile from easy_rec.python.utils import estimator_utils @@ -52,7 +51,8 @@ def cache_ckpt(pipeline_config): logging.info('will copy %s to local path %s' % (src_path, dst_path)) try: output = subprocess.check_output( - 'hadoop fs -get %s %s' % (src_path, dst_path), shell=True) + 'hadoop fs -get %s %s' % (src_path, dst_path), shell=True + ) logging.info('copy succeed: %s' % output) except Exception: logging.warning('exception: %s' % traceback.format_exc()) diff --git a/easy_rec/python/utils/embedding_utils.py b/easy_rec/python/utils/embedding_utils.py index 960513801..268737971 100644 --- a/easy_rec/python/utils/embedding_utils.py +++ b/easy_rec/python/utils/embedding_utils.py @@ -1,12 +1,10 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import os - import tensorflow as tf from tensorflow.python.framework import ops -from easy_rec.python.utils import constant -from easy_rec.python.utils import proto_util +from easy_rec.python.utils import constant, proto_util if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/utils/estimator_utils.py b/easy_rec/python/utils/estimator_utils.py index ea15063d1..c894cb006 100644 --- a/easy_rec/python/utils/estimator_utils.py +++ b/easy_rec/python/utils/estimator_utils.py @@ -1,37 +1,27 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import json import logging +import numpy as np import os import re -import sys -import time - -import numpy as np import six +import sys import tensorflow as tf +import time from tensorflow.core.framework.summary_pb2 import Summary from tensorflow.python.client import device_lib -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import meta_graph -from tensorflow.python.framework import ops +from tensorflow.python.framework import errors_impl, meta_graph, ops from tensorflow.python.ops import array_ops from tensorflow.python.platform import gfile -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import session_run_hook +from tensorflow.python.training import basic_session_run_hooks, session_run_hook # NOQA +from tensorflow.python.training.basic_session_run_hooks import SecondOrStepTimer # NOQA from tensorflow.python.training.summary_io import SummaryWriterCache -from easy_rec.python.ops.incr_record import get_sparse_indices -from easy_rec.python.ops.incr_record import kv_resource_incr_gather -from easy_rec.python.utils import constant -from easy_rec.python.utils import embedding_utils -from easy_rec.python.utils import shape_utils - -from tensorflow.python.training.basic_session_run_hooks import SecondOrStepTimer # NOQA +from easy_rec.python.ops.incr_record import get_sparse_indices, kv_resource_incr_gather # NOQA +from easy_rec.python.utils import constant, embedding_utils, shape_utils try: import horovod.tensorflow as hvd @@ -44,7 +34,7 @@ sok = None try: - from kafka import KafkaProducer, KafkaAdminClient + from kafka import KafkaAdminClient, KafkaProducer from kafka.admin import NewTopic except ImportError as ex: logging.warning('kafka-python is not installed: %s' % str(ex)) @@ -92,25 +82,29 @@ def begin(self): """Count the number of workers and masters, and setup barrier queue.""" tf.logging.info('number workers(including master) = %d' % self._num_worker) with tf.device( - tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0)): + tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0) + ): self._queue = tf.FIFOQueue( - capacity=self._num_worker, - dtypes=[tf.float32], - shapes=[()], - name='exit_counter', - shared_name='exit_counter') + capacity=self._num_worker, + dtypes=[tf.float32], + shapes=[()], + name='exit_counter', + shared_name='exit_counter' + ) self._signal_que = tf.FIFOQueue( - capacity=self._num_worker, - dtypes=[tf.string], - shapes=[()], - name='exit_counter_signal', - shared_name='exit_counter_signal') + capacity=self._num_worker, + dtypes=[tf.string], + shapes=[()], + name='exit_counter_signal', + shared_name='exit_counter_signal' + ) self._enque = self._queue.enqueue(1.0) self._que_size = self._queue.size() self._deque = self._queue.dequeue() if self._is_chief: - self._flag_file = os.path.join(self._model_dir, - 'atexit_sync_' + str(int(time.time()))) + self._flag_file = os.path.join( + self._model_dir, 'atexit_sync_' + str(int(time.time())) + ) self._send = self._signal_que.enqueue([self._flag_file]) else: self._recv = self._signal_que.dequeue() @@ -137,8 +131,9 @@ def end(self, session): que_size = session.run(self._que_size) time.sleep(5) tf.logging.info( - 'waiting for other worker to exit, finished %d, total %d' % - (que_size, self._num_worker)) + 'waiting for other worker to exit, finished %d, total %d' % + (que_size, self._num_worker) + ) # prepare on_exit synchronize base on self._flag_file if self._is_chief: for i in range(self._num_worker - 1): @@ -147,8 +142,9 @@ def end(self, session): self._flag_file = session.run(self._recv) def _check_flag_file(is_chief, flag_file): - logging.info('_check_flag_file: is_chief = %d flag_file=%s' % - (is_chief, flag_file)) + logging.info( + '_check_flag_file: is_chief = %d flag_file=%s' % (is_chief, flag_file) + ) if is_chief: with gfile.GFile(flag_file, 'w') as fout: fout.write('atexit time: %d' % int(time.time())) @@ -158,7 +154,8 @@ def _check_flag_file(is_chief, flag_file): from atexit import register register( - _check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file) + _check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file + ) logging.info('ExitBarrier passed') @@ -188,25 +185,29 @@ def begin(self): """Count the number of workers and masters, and setup barrier queue.""" tf.logging.info('number workers(including master) = %d' % self._num_worker) with tf.device( - tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0)): + tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0) + ): self._queue = tf.FIFOQueue( - capacity=self._num_worker, - dtypes=[tf.float32], - shapes=[()], - name='exit_counter', - shared_name='exit_counter') + capacity=self._num_worker, + dtypes=[tf.float32], + shapes=[()], + name='exit_counter', + shared_name='exit_counter' + ) self._signal_que = tf.FIFOQueue( - capacity=self._num_worker, - dtypes=[tf.string], - shapes=[()], - name='exit_counter_signal', - shared_name='exit_counter_signal') + capacity=self._num_worker, + dtypes=[tf.string], + shapes=[()], + name='exit_counter_signal', + shared_name='exit_counter_signal' + ) self._enque = self._queue.enqueue(1.0) self._que_size = self._queue.size() self._deque = self._queue.dequeue() if self._is_chief: - self._flag_file = os.path.join(self._model_dir, - 'atexit_sync_' + str(int(time.time()))) + self._flag_file = os.path.join( + self._model_dir, 'atexit_sync_' + str(int(time.time())) + ) self._send = self._signal_que.enqueue([self._flag_file]) else: self._recv = self._signal_que.dequeue() @@ -233,8 +234,9 @@ def end(self, session): que_size = session.run(self._que_size) time.sleep(5) tf.logging.info( - 'waiting for other worker to exit, finished %d, total %d' % - (que_size, self._num_worker)) + 'waiting for other worker to exit, finished %d, total %d' % + (que_size, self._num_worker) + ) # prepare on_exit synchronize base on self._flag_file if self._is_chief: self.eval_result = session.run(self.metric_ops) @@ -244,8 +246,9 @@ def end(self, session): self._flag_file = session.run(self._recv) def _check_flag_file(is_chief, flag_file): - logging.info('_check_flag_file: is_chief = %d flag_file=%s' % - (is_chief, flag_file)) + logging.info( + '_check_flag_file: is_chief = %d flag_file=%s' % (is_chief, flag_file) + ) if is_chief: with gfile.GFile(flag_file, 'w') as fout: fout.write('atexit time: %d' % int(time.time())) @@ -255,7 +258,8 @@ def _check_flag_file(is_chief, flag_file): from atexit import register register( - _check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file) + _check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file + ) session.run(self.metric_ops) logging.info('ExitBarrier passed') @@ -284,9 +288,10 @@ def before_run(self, run_context): return tf.train.SessionRunArgs([tf.train.get_global_step()]) def after_run( - self, - run_context, # pylint: disable=unused-argument - run_values): + self, + run_context, # pylint: disable=unused-argument + run_values + ): if self._is_chief: global_step = run_values.results[0] curr_progress = global_step / self._num_steps @@ -307,17 +312,19 @@ def end(self, session): class CheckpointSaverHook(CheckpointSaverHook): """Saves checkpoints every N steps or seconds.""" - def __init__(self, - checkpoint_dir, - save_secs=None, - save_steps=None, - saver=None, - checkpoint_basename='model.ckpt', - scaffold=None, - listeners=None, - write_graph=True, - data_offset_var=None, - increment_save_config=None): + def __init__( + self, + checkpoint_dir, + save_secs=None, + save_steps=None, + saver=None, + checkpoint_basename='model.ckpt', + scaffold=None, + listeners=None, + write_graph=True, + data_offset_var=None, + increment_save_config=None + ): """Initializes a `CheckpointSaverHook`. Args: @@ -339,13 +346,14 @@ def __init__(self, ValueError: At most one of saver or scaffold should be set. """ super(CheckpointSaverHook, self).__init__( - checkpoint_dir, - save_secs=save_secs, - save_steps=save_steps, - saver=saver, - checkpoint_basename=checkpoint_basename, - scaffold=scaffold, - listeners=listeners) + checkpoint_dir, + save_secs=save_secs, + save_steps=save_steps, + saver=saver, + checkpoint_basename=checkpoint_basename, + scaffold=scaffold, + listeners=listeners + ) self._cuda_profile_start = 0 self._cuda_profile_stop = 0 self._steps_per_run = 1 @@ -357,31 +365,35 @@ def __init__(self, if increment_save_config is not None: self._kafka_timeout_ms = os.environ.get('KAFKA_TIMEOUT', 600) * 1000 logging.info('KAFKA_TIMEOUT: %dms' % self._kafka_timeout_ms) - self._kafka_max_req_size = os.environ.get('KAFKA_MAX_REQ_SIZE', - 1024 * 1024 * 64) + self._kafka_max_req_size = os.environ.get( + 'KAFKA_MAX_REQ_SIZE', 1024 * 1024 * 64 + ) logging.info('KAFKA_MAX_REQ_SIZE: %d' % self._kafka_max_req_size) - self._kafka_max_msg_size = os.environ.get('KAFKA_MAX_MSG_SIZE', - 1024 * 1024 * 1024) + self._kafka_max_msg_size = os.environ.get( + 'KAFKA_MAX_MSG_SIZE', 1024 * 1024 * 1024 + ) logging.info('KAFKA_MAX_MSG_SIZE: %d' % self._kafka_max_msg_size) self._dense_name_to_ids = embedding_utils.get_dense_name_to_ids() self._sparse_name_to_ids = embedding_utils.get_sparse_name_to_ids() with gfile.GFile( - os.path.join(checkpoint_dir, constant.DENSE_UPDATE_VARIABLES), - 'w') as fout: + os.path.join(checkpoint_dir, constant.DENSE_UPDATE_VARIABLES), 'w' + ) as fout: json.dump(self._dense_name_to_ids, fout, indent=2) save_secs = increment_save_config.dense_save_secs save_steps = increment_save_config.dense_save_steps self._dense_timer = SecondOrStepTimer( - every_secs=save_secs if save_secs > 0 else None, - every_steps=save_steps if save_steps > 0 else None) + every_secs=save_secs if save_secs > 0 else None, + every_steps=save_steps if save_steps > 0 else None + ) save_secs = increment_save_config.sparse_save_secs save_steps = increment_save_config.sparse_save_steps self._sparse_timer = SecondOrStepTimer( - every_secs=save_secs if save_secs > 0 else None, - every_steps=save_steps if save_steps > 0 else None) + every_secs=save_secs if save_secs > 0 else None, + every_steps=save_steps if save_steps > 0 else None + ) self._dense_timer.update_last_triggered_step(0) self._sparse_timer.update_last_triggered_step(0) @@ -393,18 +405,22 @@ def __init__(self, with ops.control_dependencies([tf.train.get_global_step()]): with ops.colocate_with(sparse_var): sparse_indice = get_sparse_indices( - var_name=sparse_var.op.name, ktype=indice_dtype) + var_name=sparse_var.op.name, ktype=indice_dtype + ) # sparse_indice = sparse_indice.global_indices self._sparse_indices.append(sparse_indice) if 'EmbeddingVariable' in str(type(sparse_var)): self._sparse_values.append( - kv_resource_incr_gather( - sparse_var._handle, sparse_indice, - np.zeros(sparse_var.shape.as_list(), dtype=np.float32))) + kv_resource_incr_gather( + sparse_var._handle, sparse_indice, + np.zeros(sparse_var.shape.as_list(), dtype=np.float32) + ) + ) # sparse_var.sparse_read(sparse_indice)) else: self._sparse_values.append( - array_ops.gather(sparse_var, sparse_indice)) + array_ops.gather(sparse_var, sparse_indice) + ) self._kafka_producer = None self._incr_save_dir = None @@ -413,30 +429,32 @@ def __init__(self, logging.info('increment save topic: %s' % self._topic) admin_clt = KafkaAdminClient( - bootstrap_servers=increment_save_config.kafka.server, - request_timeout_ms=self._kafka_timeout_ms, - api_version_auto_timeout_ms=self._kafka_timeout_ms) + bootstrap_servers=increment_save_config.kafka.server, + request_timeout_ms=self._kafka_timeout_ms, + api_version_auto_timeout_ms=self._kafka_timeout_ms + ) if self._topic not in admin_clt.list_topics(): admin_clt.create_topics( - new_topics=[ - NewTopic( - name=self._topic, - num_partitions=1, - replication_factor=1, - topic_configs={ - 'max.message.bytes': self._kafka_max_msg_size - }) - ], - validate_only=False) + new_topics=[ + NewTopic( + name=self._topic, + num_partitions=1, + replication_factor=1, + topic_configs={'max.message.bytes': self._kafka_max_msg_size} + ) + ], + validate_only=False + ) logging.info('create increment save topic: %s' % self._topic) admin_clt.close() servers = increment_save_config.kafka.server.split(',') self._kafka_producer = KafkaProducer( - bootstrap_servers=servers, - max_request_size=self._kafka_max_req_size, - api_version_auto_timeout_ms=self._kafka_timeout_ms, - request_timeout_ms=self._kafka_timeout_ms) + bootstrap_servers=servers, + max_request_size=self._kafka_max_req_size, + api_version_auto_timeout_ms=self._kafka_timeout_ms, + request_timeout_ms=self._kafka_timeout_ms + ) elif increment_save_config.HasField('fs'): fs = increment_save_config.fs if fs.relative: @@ -448,10 +466,13 @@ def __init__(self, if not gfile.IsDirectory(self._incr_save_dir): gfile.MakeDirs(self._incr_save_dir) elif increment_save_config.HasField('datahub'): - raise NotImplementedError('datahub increment saving is in development.') + raise NotImplementedError( + 'datahub increment saving is in development.' + ) else: raise ValueError( - 'incr_update not specified correctly, must be oneof: kafka,fs') + 'incr_update not specified correctly, must be oneof: kafka,fs' + ) self._debug_save_update = increment_save_config.debug_save_update else: @@ -464,12 +485,15 @@ def after_create_session(self, session, coord): # We do write graph and saver_def at the first call of before_run. # We cannot do this in begin, since we let other hooks to change graph and # add variables at begin. Graph is finalized after all begin calls. - tf.train.write_graph(tf.get_default_graph().as_graph_def(add_shapes=True), - self._checkpoint_dir, 'graph.pbtxt') + tf.train.write_graph( + tf.get_default_graph().as_graph_def(add_shapes=True), + self._checkpoint_dir, 'graph.pbtxt' + ) saver_def = self._get_saver().saver_def if self._get_saver() else None graph = tf.get_default_graph() meta_graph_def = meta_graph.create_meta_graph_def( - graph_def=graph.as_graph_def(add_shapes=True), saver_def=saver_def) + graph_def=graph.as_graph_def(add_shapes=True), saver_def=saver_def + ) self._summary_writer.add_graph(graph) self._summary_writer.add_meta_graph(meta_graph_def) @@ -484,7 +508,9 @@ def before_run(self, run_context): # pylint: disable=unused-argument def _send_dense(self, global_step, session): dense_train_vars = ops.get_collection(constant.DENSE_UPDATE_VARIABLES) dense_train_vals = session.run(dense_train_vars) - logging.info('global_step=%d, increment save dense variables' % global_step) + logging.info( + 'global_step=%d, increment save dense variables' % global_step + ) # build msg header msg_num = len(dense_train_vals) @@ -503,13 +529,17 @@ def _send_dense(self, global_step, session): if self._kafka_producer is not None: msg_key = 'dense_update_%d' % global_step send_res = self._kafka_producer.send( - self._topic, bytes_buf, key=msg_key.encode('utf-8')) - logging.info('kafka send dense: %d exception: %s' % - (global_step, send_res.exception)) + self._topic, bytes_buf, key=msg_key.encode('utf-8') + ) + logging.info( + 'kafka send dense: %d exception: %s' % + (global_step, send_res.exception) + ) if self._incr_save_dir is not None: - save_path = os.path.join(self._incr_save_dir, - 'dense_update_%d' % global_step) + save_path = os.path.join( + self._incr_save_dir, 'dense_update_%d' % global_step + ) with gfile.GFile(save_path, 'wb') as fout: fout.write(bytes_buf) save_flag = save_path + '.done' @@ -526,8 +556,9 @@ def _send_dense(self, global_step, session): fout.write(bytes_buf) logging.info( - 'global_step=%d, increment update dense variables, msg_num=%d' % - (global_step, msg_num)) + 'global_step=%d, increment update dense variables, msg_num=%d' % + (global_step, msg_num) + ) def _send_sparse(self, global_step, session): sparse_train_vars = ops.get_collection(constant.SPARSE_UPDATE_VARIABLES) @@ -540,14 +571,15 @@ def _send_sparse(self, global_step, session): sparse_train_vars = [sparse_train_vars[i][0] for i in sel_ids] sel_embed_ids = [ - self._sparse_name_to_ids[x.name] for x in sparse_train_vars + self._sparse_name_to_ids[x.name] for x in sparse_train_vars ] msg_num = len(sel_ids) if msg_num == 0: - logging.warning('there are no sparse updates, will skip this send: %d' % - global_step) + logging.warning( + 'there are no sparse updates, will skip this send: %d' % global_step + ) return # build msg header @@ -559,9 +591,9 @@ def _send_sparse(self, global_step, session): bytes_buf = np.array(msg_header, dtype=np.int32).tobytes() # build msg body - for tmp_id, tmp_key, tmp_val, tmp_var in zip(sel_embed_ids, sparse_key_res, - sparse_val_res, - sparse_train_vars): + for tmp_id, tmp_key, tmp_val, tmp_var in zip( + sel_embed_ids, sparse_key_res, sparse_val_res, sparse_train_vars + ): # for non kv embedding variables, add partition offset to tmp_key if 'EmbeddingVariable' not in str(type(tmp_var)): if tmp_var._save_slice_info is not None: @@ -571,13 +603,16 @@ def _send_sparse(self, global_step, session): if self._kafka_producer is not None: msg_key = 'sparse_update_%d' % global_step send_res = self._kafka_producer.send( - self._topic, bytes_buf, key=msg_key.encode('utf-8')) - logging.info('kafka send sparse: %d %s' % - (global_step, send_res.exception)) + self._topic, bytes_buf, key=msg_key.encode('utf-8') + ) + logging.info( + 'kafka send sparse: %d %s' % (global_step, send_res.exception) + ) if self._incr_save_dir is not None: - save_path = os.path.join(self._incr_save_dir, - 'sparse_update_%d' % global_step) + save_path = os.path.join( + self._incr_save_dir, 'sparse_update_%d' % global_step + ) with gfile.GFile(save_path, 'wb') as fout: fout.write(bytes_buf) save_flag = save_path + '.done' @@ -594,21 +629,24 @@ def _send_sparse(self, global_step, session): fout.write(bytes_buf) logging.info( - 'global_step=%d, increment update sparse variables, msg_num=%d, msg_size=%d' - % (global_step, msg_num, len(bytes_buf))) + 'global_step=%d, increment update sparse variables, msg_num=%d, msg_size=%d' + % (global_step, msg_num, len(bytes_buf)) + ) def after_run(self, run_context, run_values): super(CheckpointSaverHook, self).after_run(run_context, run_values) stale_global_step = run_values.results global_step = -1 if self._dense_timer is not None and self._dense_timer.should_trigger_for_step( - stale_global_step + self._steps_per_run): + stale_global_step + self._steps_per_run + ): global_step = run_context.session.run(self._global_step_tensor) self._dense_timer.update_last_triggered_step(global_step) self._send_dense(global_step, run_context.session) if self._sparse_timer is not None and self._sparse_timer.should_trigger_for_step( - stale_global_step + self._steps_per_run): + stale_global_step + self._steps_per_run + ): if global_step < 0: global_step = run_context.session.run(self._global_step_tensor) @@ -628,28 +666,32 @@ def _save(self, session, step): for x in save_data_offset: if x: data_offset_json.update(json.loads(x)) - save_offset_path = os.path.join(self._checkpoint_dir, - 'model.ckpt-%d.offset' % step) + save_offset_path = os.path.join( + self._checkpoint_dir, 'model.ckpt-%d.offset' % step + ) with gfile.GFile(save_offset_path, 'w') as fout: json.dump(data_offset_json, fout) self._get_saver().save( - session, - self._save_path, - global_step=step, - write_meta_graph=self._write_graph) + session, + self._save_path, + global_step=step, + write_meta_graph=self._write_graph + ) self._summary_writer.add_session_log( - tf.SessionLog( - status=tf.SessionLog.CHECKPOINT, checkpoint_path=self._save_path), - step) + tf.SessionLog( + status=tf.SessionLog.CHECKPOINT, checkpoint_path=self._save_path + ), step + ) should_stop = False for l in self._listeners: # noqa: E741 if l.after_save(session, step): logging.info( - 'A CheckpointSaverListener requested that training be stopped. ' - 'listener: {}'.format(l)) + 'A CheckpointSaverListener requested that training be stopped. ' + 'listener: {}'.format(l) + ) should_stop = True return should_stop @@ -695,8 +737,9 @@ def begin(self): assign_ops.append(var.assign(var_data)) else: logging.error( - 'variable [%s] shape not match %r vs %r' % - (var.name.split(':')[0], var_shape, list(var_data.shape))) + 'variable [%s] shape not match %r vs %r' % + (var.name.split(':')[0], var_shape, list(var_data.shape)) + ) has_shape_unmatch = True elif 'Momentum' not in var_name and 'global_step' not in var_name: logging.error('variable [%s] not found in ckpt' % var_name) @@ -734,12 +777,15 @@ def begin(self): assign_ops = [] for var, var_tmp in six.iteritems(self._incompatible_shape_var_map): assign_ops.append( - var.assign( - shape_utils.pad_or_clip_nd(var_tmp, - var.get_shape().as_list()))) + var.assign( + shape_utils.pad_or_clip_nd(var_tmp, + var.get_shape().as_list()) + ) + ) logging.info( - 'Assign variable[%s] from shape%s to shape%s' % - (var.name, var_tmp.get_shape().as_list(), var.get_shape().as_list())) + 'Assign variable[%s] from shape%s to shape%s' % + (var.name, var_tmp.get_shape().as_list(), var.get_shape().as_list()) + ) self._restore_op = tf.group(assign_ops) def after_create_session(self, session, coord): @@ -777,8 +823,9 @@ def begin(self): if var_name in ckpt_var2shape_map: if restore_status[var_name]: logging.warning( - 'variable %s find in more than one checkpoint, skipped %s' % - (var_name, ckpt_path)) + 'variable %s find in more than one checkpoint, skipped %s' % + (var_name, ckpt_path) + ) continue name2var[var_name] = var restore_status[var_name] = True @@ -821,8 +868,9 @@ def end(self, session): self._summary_writer.add_summary(summary, global_step=global_step) self._summary_writer.flush() - eval_result_file = os.path.join(self._output_dir, - 'online_eval_result.txt-%s' % global_step) + eval_result_file = os.path.join( + self._output_dir, 'online_eval_result.txt-%s' % global_step + ) logging.info('Saving online eval result to file %s' % eval_result_file) with gfile.GFile(eval_result_file, 'w') as ofile: result_to_write = {} @@ -881,8 +929,9 @@ def get_ckpt_version(ckpt_path): return int(toks[-1]) -def get_latest_checkpoint_from_checkpoint_path(checkpoint_path, - ignore_ckpt_error): +def get_latest_checkpoint_from_checkpoint_path( + checkpoint_path, ignore_ckpt_error +): ckpt_path = None if checkpoint_path.endswith('/') or gfile.IsDirectory(checkpoint_path + '/'): checkpoint_dir = checkpoint_path @@ -892,8 +941,9 @@ def get_latest_checkpoint_from_checkpoint_path(checkpoint_path, ckpt_path = latest_checkpoint(checkpoint_dir) if ckpt_path: logging.info( - 'fine_tune_checkpoint is directory, will use the latest checkpoint: %s' - % ckpt_path) + 'fine_tune_checkpoint is directory, will use the latest checkpoint: %s' + % ckpt_path + ) else: assert ignore_ckpt_error, 'fine_tune_checkpoint(%s) is not exists.' % checkpoint_path else: @@ -1013,7 +1063,7 @@ def has_sok(): def init_hvd(): if hvd is None: logging.error( - 'horovod is not installed: HOROVOD_WITH_TENSORFLOW=1 pip install horovod' + 'horovod is not installed: HOROVOD_WITH_TENSORFLOW=1 pip install horovod' ) sys.exit(1) diff --git a/easy_rec/python/utils/export_big_model.py b/easy_rec/python/utils/export_big_model.py index 243847a5f..fc4bc5a29 100644 --- a/easy_rec/python/utils/export_big_model.py +++ b/easy_rec/python/utils/export_big_model.py @@ -3,32 +3,23 @@ import json import logging -import os -import time - import numpy as np +import os import tensorflow as tf +import time from google.protobuf import json_format from tensorflow.core.protobuf import config_pb2 from tensorflow.python.framework import ops from tensorflow.python.ops.variables import global_variables -from tensorflow.python.platform.gfile import DeleteRecursively -from tensorflow.python.platform.gfile import Exists -from tensorflow.python.platform.gfile import GFile -from tensorflow.python.platform.gfile import Remove +from tensorflow.python.platform.gfile import DeleteRecursively, Exists, GFile, Remove # NOQA from tensorflow.python.saved_model import signature_constants from tensorflow.python.training.device_setter import replica_device_setter -from tensorflow.python.training.monitored_session import ChiefSessionCreator -from tensorflow.python.training.monitored_session import Scaffold +from tensorflow.python.training.monitored_session import ChiefSessionCreator, Scaffold # NOQA from tensorflow.python.training.saver import export_meta_graph import easy_rec -from easy_rec.python.utils import constant -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import io_util -from easy_rec.python.utils import proto_util -from easy_rec.python.utils.meta_graph_editor import EMBEDDING_INITIALIZERS -from easy_rec.python.utils.meta_graph_editor import MetaGraphEditor +from easy_rec.python.utils import constant, estimator_utils, io_util, proto_util # NOQA +from easy_rec.python.utils.meta_graph_editor import EMBEDDING_INITIALIZERS, MetaGraphEditor # NOQA if tf.__version__ >= '2.0': from tensorflow.python.framework.ops import disable_eager_execution @@ -40,8 +31,10 @@ INCR_UPDATE_SIGNATURE_KEY = 'incr_update_sig' -def export_big_model(export_dir, pipeline_config, redis_params, - serving_input_fn, estimator, checkpoint_path, verbose): +def export_big_model( + export_dir, pipeline_config, redis_params, serving_input_fn, estimator, + checkpoint_path, verbose +): for key in redis_params: logging.info('%s: %s' % (key, redis_params[key])) @@ -58,14 +51,17 @@ def export_big_model(export_dir, pipeline_config, redis_params, kv_module = tf.load_op_library(write_kv_lib_path) try: - sparse_kv_lib_path = os.path.join(easy_rec.ops_dir, 'libwrite_sparse_kv.so') + sparse_kv_lib_path = os.path.join( + easy_rec.ops_dir, 'libwrite_sparse_kv.so' + ) sparse_kv_module = tf.load_op_library(sparse_kv_lib_path) except Exception as ex: logging.warning('load libwrite_sparse_kv.so failed: %s' % str(ex)) sparse_kv_module = None if not checkpoint_path: checkpoint_path = estimator_utils.latest_checkpoint( - pipeline_config.model_dir) + pipeline_config.model_dir + ) logging.info('checkpoint_path = %s' % checkpoint_path) server = None @@ -76,7 +72,8 @@ def export_big_model(export_dir, pipeline_config, redis_params, if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) server = tf.train.Server( - cluster, job_name='ps', task_index=tf_config['task']['index']) + cluster, job_name='ps', task_index=tf_config['task']['index'] + ) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: @@ -92,10 +89,11 @@ def export_big_model(export_dir, pipeline_config, redis_params, if cluster: logging.info('cluster = ' + str(cluster)) with tf.device( - replica_device_setter( - worker_device='/job:master/task:0', cluster=cluster)): - outputs = estimator._export_model_fn(features, None, None, - estimator.params).predictions + replica_device_setter(worker_device='/job:master/task:0', cluster=cluster) + ): + outputs = estimator._export_model_fn( + features, None, None, estimator.params + ).predictions meta_graph_def = export_meta_graph() redis_embedding_version = redis_params.get('redis_embedding_version', '') @@ -105,8 +103,9 @@ def export_big_model(export_dir, pipeline_config, redis_params, else: meta_graph_def.meta_info_def.meta_graph_version = redis_embedding_version - logging.info('meta_graph_version = %s' % - meta_graph_def.meta_info_def.meta_graph_version) + logging.info( + 'meta_graph_version = %s' % meta_graph_def.meta_info_def.meta_graph_version + ) embed_var_parts = {} embed_norm_name = {} @@ -120,11 +119,13 @@ def export_big_model(export_dir, pipeline_config, redis_params, norm_name_to_ids[norm_name] = 1 tmp_export = x.export() if x.device not in embedding_vars: - embedding_vars[x.device] = [(norm_name, tmp_export.keys, - tmp_export.values)] + embedding_vars[x.device] = [ + (norm_name, tmp_export.keys, tmp_export.values) + ] else: embedding_vars[x.device].append( - (norm_name, tmp_export.keys, tmp_export.values)) + (norm_name, tmp_export.keys, tmp_export.values) + ) elif '/embedding_weights:' in x.name or '/embedding_weights/part_' in x.name: norm_name, part_id = proto_util.get_norm_embed_name(x.name) norm_name_to_ids[norm_name] = 1 @@ -176,17 +177,18 @@ def export_big_model(export_dir, pipeline_config, redis_params, tmp_names = [embed_norm_name[v] for v in tmp_vars] tmp_spos = [np.array(embed_spos[v], dtype=np.int64) for v in tmp_vars] write_kv_res = kv_module.write_kv( - tmp_names, - tmp_vars, - tmp_spos, - url=redis_url, - password=redis_passwd, - timeout=redis_params.get('redis_timeout', 1500), - version=meta_graph_def.meta_info_def.meta_graph_version, - threads=redis_params.get('redis_threads', 5), - batch_size=redis_params.get('redis_batch_size', 32), - expire=redis_params.get('redis_expire', 24), - verbose=verbose) + tmp_names, + tmp_vars, + tmp_spos, + url=redis_url, + password=redis_passwd, + timeout=redis_params.get('redis_timeout', 1500), + version=meta_graph_def.meta_info_def.meta_graph_version, + threads=redis_params.get('redis_threads', 5), + batch_size=redis_params.get('redis_batch_size', 32), + expire=redis_params.get('redis_expire', 24), + verbose=verbose + ) all_write_res.append(write_kv_res) for tmp_dev in embedding_vars: @@ -196,55 +198,62 @@ def export_big_model(export_dir, pipeline_config, redis_params, tmp_sparse_keys = [x[1] for x in tmp_vs] tmp_sparse_vals = [x[2] for x in tmp_vs] write_sparse_kv_res = sparse_kv_module.write_sparse_kv( - tmp_sparse_names, - tmp_sparse_vals, - tmp_sparse_keys, - url=redis_url, - password=redis_passwd, - timeout=redis_params.get('redis_timeout', 1500), - version=meta_graph_def.meta_info_def.meta_graph_version, - threads=redis_params.get('redis_threads', 5), - batch_size=redis_params.get('redis_batch_size', 32), - expire=redis_params.get('redis_expire', 24), - verbose=verbose) + tmp_sparse_names, + tmp_sparse_vals, + tmp_sparse_keys, + url=redis_url, + password=redis_passwd, + timeout=redis_params.get('redis_timeout', 1500), + version=meta_graph_def.meta_info_def.meta_graph_version, + threads=redis_params.get('redis_threads', 5), + batch_size=redis_params.get('redis_batch_size', 32), + expire=redis_params.get('redis_expire', 24), + verbose=verbose + ) all_write_res.append(write_sparse_kv_res) session_config = ConfigProto( - allow_soft_placement=True, log_device_placement=False) + allow_soft_placement=True, log_device_placement=False + ) chief_sess_creator = ChiefSessionCreator( - master=server.target if server else '', - checkpoint_filename_with_path=checkpoint_path, - config=session_config) + master=server.target if server else '', + checkpoint_filename_with_path=checkpoint_path, + config=session_config + ) with tf.train.MonitoredSession( - session_creator=chief_sess_creator, - hooks=None, - stop_grace_period_secs=120) as sess: + session_creator=chief_sess_creator, + hooks=None, + stop_grace_period_secs=120 + ) as sess: dump_flags = sess.run(all_write_res) logging.info('write embedding to redis succeed: %s' % str(dump_flags)) else: - logging.info('will skip write embedding to redis because ' - 'redis_write_kv is set to 0.') + logging.info( + 'will skip write embedding to redis because ' + 'redis_write_kv is set to 0.' + ) # delete embedding_weights collections so that it could be re imported tmp_drop = [] for k in meta_graph_def.collection_def: v = meta_graph_def.collection_def[k] - if len( - v.node_list.value) > 0 and 'embedding_weights' in v.node_list.value[0]: + if len(v.node_list.value + ) > 0 and 'embedding_weights' in v.node_list.value[0]: tmp_drop.append(k) for k in tmp_drop: meta_graph_def.collection_def.pop(k) meta_graph_editor = MetaGraphEditor( - os.path.join(easy_rec.ops_dir, 'libembed_op.so'), - None, - redis_url, - redis_passwd, - redis_timeout=redis_params.get('redis_timeout', 600), - redis_cache_names=redis_cache_names, - meta_graph_def=meta_graph_def, - norm_name_to_ids=norm_name_to_ids, - debug_dir=export_dir if verbose else '') + os.path.join(easy_rec.ops_dir, 'libembed_op.so'), + None, + redis_url, + redis_passwd, + redis_timeout=redis_params.get('redis_timeout', 600), + redis_cache_names=redis_cache_names, + meta_graph_def=meta_graph_def, + norm_name_to_ids=norm_name_to_ids, + debug_dir=export_dir if verbose else '' + ) meta_graph_editor.edit_graph() tf.reset_default_graph() @@ -256,12 +265,15 @@ def export_big_model(export_dir, pipeline_config, redis_params, for tmp_norm_name in norm_name_to_ids: fout.write('%s\t%s\n' % (tmp_norm_name, norm_name_to_ids[tmp_norm_name])) ops.add_to_collection( - tf.GraphKeys.ASSET_FILEPATHS, - tf.constant( - embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt')) - - export_dir = os.path.join(export_dir, - meta_graph_def.meta_info_def.meta_graph_version) + tf.GraphKeys.ASSET_FILEPATHS, + tf.constant( + embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt' + ) + ) + + export_dir = os.path.join( + export_dir, meta_graph_def.meta_info_def.meta_graph_version + ) export_dir = io_util.fix_oss_dir(export_dir) logging.info('export_dir=%s' % export_dir) if Exists(export_dir): @@ -280,27 +292,31 @@ def export_big_model(export_dir, pipeline_config, redis_params, tensor_info_outputs[tmp_key] = \ tf.saved_model.utils.build_tensor_info(tmp) signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=tensor_info_inputs, - outputs=tensor_info_outputs, - method_name=signature_constants.PREDICT_METHOD_NAME)) + tf.saved_model.signature_def_utils.build_signature_def( + inputs=tensor_info_inputs, + outputs=tensor_info_outputs, + method_name=signature_constants.PREDICT_METHOD_NAME + ) + ) session_config = ConfigProto( - allow_soft_placement=True, log_device_placement=True) + allow_soft_placement=True, log_device_placement=True + ) saver = tf.train.Saver() with tf.Session(target=server.target if server else '') as sess: saver.restore(sess, checkpoint_path) builder.add_meta_graph_and_variables( - sess, [tf.saved_model.tag_constants.SERVING], - signature_def_map={ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature, - }, - assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), - saver=saver, - strip_default_attrs=True, - clear_devices=True) + sess, [tf.saved_model.tag_constants.SERVING], + signature_def_map={ + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature, + }, + assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), + saver=saver, + strip_default_attrs=True, + clear_devices=True + ) builder.save() # remove temporary files @@ -308,9 +324,10 @@ def export_big_model(export_dir, pipeline_config, redis_params, return export_dir -def export_big_model_to_oss(export_dir, pipeline_config, oss_params, - serving_input_fn, estimator, checkpoint_path, - verbose): +def export_big_model_to_oss( + export_dir, pipeline_config, oss_params, serving_input_fn, estimator, + checkpoint_path, verbose +): for key in oss_params: logging.info('%s: %s' % (key, oss_params[key])) @@ -319,7 +336,8 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, if not checkpoint_path: checkpoint_path = estimator_utils.latest_checkpoint( - pipeline_config.model_dir) + pipeline_config.model_dir + ) logging.info('checkpoint_path = %s' % checkpoint_path) server = None @@ -330,7 +348,8 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) server = tf.train.Server( - cluster, job_name='ps', task_index=tf_config['task']['index']) + cluster, job_name='ps', task_index=tf_config['task']['index'] + ) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: @@ -346,10 +365,11 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, if cluster: logging.info('cluster = ' + str(cluster)) with tf.device( - replica_device_setter( - worker_device='/job:master/task:0', cluster=cluster)): - outputs = estimator._export_model_fn(features, None, None, - estimator.params).predictions + replica_device_setter(worker_device='/job:master/task:0', cluster=cluster) + ): + outputs = estimator._export_model_fn( + features, None, None, estimator.params + ).predictions meta_graph_def = export_meta_graph() meta_graph_def.meta_info_def.meta_graph_version = str(int(time.time())) @@ -360,8 +380,9 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, else: meta_graph_def.meta_info_def.meta_graph_version = oss_embedding_version - logging.info('meta_graph_version = %s' % - meta_graph_def.meta_info_def.meta_graph_version) + logging.info( + 'meta_graph_version = %s' % meta_graph_def.meta_info_def.meta_graph_version + ) embed_var_parts = {} embed_norm_name = {} @@ -376,11 +397,13 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, norm_name_to_ids[norm_name] = 1 tmp_export = x.export() if x.device not in embedding_vars: - embedding_vars[x.device] = [(norm_name, tmp_export.keys, - tmp_export.values, part_id)] + embedding_vars[x.device] = [ + (norm_name, tmp_export.keys, tmp_export.values, part_id) + ] else: embedding_vars[x.device].append( - (norm_name, tmp_export.keys, tmp_export.values, part_id)) + (norm_name, tmp_export.keys, tmp_export.values, part_id) + ) elif '/embedding_weights:' in x.name or '/embedding_weights/part_' in x.name: norm_name, part_id = proto_util.get_norm_embed_name(x.name) norm_name_to_ids[norm_name] = 1 @@ -411,8 +434,9 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, oss_endpoint = oss_params.get('oss_endpoint', '') oss_ak = oss_params.get('oss_ak', '') oss_sk = oss_params.get('oss_sk', '') - logging.info('will export to oss: %s %s %s %s', oss_path, oss_endpoint, - oss_ak, oss_sk) + logging.info( + 'will export to oss: %s %s %s %s', oss_path, oss_endpoint, oss_ak, oss_sk + ) if oss_params.get('oss_write_kv', ''): # group embed by devices @@ -430,17 +454,18 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, tmp_names = [embed_norm_name[v] for v in tmp_vars] tmp_spos = [np.array(embed_spos[v], dtype=np.int64) for v in tmp_vars] write_kv_res = kv_module.oss_write_kv( - tmp_names, - tmp_vars, - tmp_spos, - osspath=oss_path, - endpoint=oss_endpoint, - ak=oss_ak, - sk=oss_sk, - threads=oss_params.get('oss_threads', 5), - timeout=5, - expire=5, - verbose=verbose) + tmp_names, + tmp_vars, + tmp_spos, + osspath=oss_path, + endpoint=oss_endpoint, + ak=oss_ak, + sk=oss_sk, + threads=oss_params.get('oss_threads', 5), + timeout=5, + expire=5, + verbose=verbose + ) all_write_res.append(write_kv_res) for tmp_dev in embedding_vars: @@ -451,57 +476,64 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, tmp_sparse_vals = [x[2] for x in tmp_vs] tmp_part_ids = [x[3] for x in tmp_vs] write_sparse_kv_res = kv_module.oss_write_sparse_kv( - tmp_sparse_names, - tmp_sparse_vals, - tmp_sparse_keys, - tmp_part_ids, - osspath=oss_path, - endpoint=oss_endpoint, - ak=oss_ak, - sk=oss_sk, - version=meta_graph_def.meta_info_def.meta_graph_version, - threads=oss_params.get('oss_threads', 5), - verbose=verbose) + tmp_sparse_names, + tmp_sparse_vals, + tmp_sparse_keys, + tmp_part_ids, + osspath=oss_path, + endpoint=oss_endpoint, + ak=oss_ak, + sk=oss_sk, + version=meta_graph_def.meta_info_def.meta_graph_version, + threads=oss_params.get('oss_threads', 5), + verbose=verbose + ) all_write_res.append(write_sparse_kv_res) session_config = ConfigProto( - allow_soft_placement=True, log_device_placement=False) + allow_soft_placement=True, log_device_placement=False + ) chief_sess_creator = ChiefSessionCreator( - master=server.target if server else '', - checkpoint_filename_with_path=checkpoint_path, - config=session_config) + master=server.target if server else '', + checkpoint_filename_with_path=checkpoint_path, + config=session_config + ) with tf.train.MonitoredSession( - session_creator=chief_sess_creator, - hooks=None, - stop_grace_period_secs=120) as sess: + session_creator=chief_sess_creator, + hooks=None, + stop_grace_period_secs=120 + ) as sess: dump_flags = sess.run(all_write_res) logging.info('write embedding to oss succeed: %s' % str(dump_flags)) else: - logging.info('will skip write embedding to oss because ' - 'oss_write_kv is set to 0.') + logging.info( + 'will skip write embedding to oss because ' + 'oss_write_kv is set to 0.' + ) # delete embedding_weights collections so that it could be re imported tmp_drop = [] for k in meta_graph_def.collection_def: v = meta_graph_def.collection_def[k] - if len( - v.node_list.value) > 0 and 'embedding_weights' in v.node_list.value[0]: + if len(v.node_list.value + ) > 0 and 'embedding_weights' in v.node_list.value[0]: tmp_drop.append(k) for k in tmp_drop: meta_graph_def.collection_def.pop(k) meta_graph_editor = MetaGraphEditor( - os.path.join(easy_rec.ops_dir, 'libembed_op.so'), - None, - oss_path=oss_path, - oss_endpoint=oss_endpoint, - oss_ak=oss_ak, - oss_sk=oss_sk, - oss_timeout=oss_params.get('oss_timeout', 1500), - meta_graph_def=meta_graph_def, - norm_name_to_ids=norm_name_to_ids, - incr_update_params=oss_params.get('incr_update', None), - debug_dir=export_dir if verbose else '') + os.path.join(easy_rec.ops_dir, 'libembed_op.so'), + None, + oss_path=oss_path, + oss_endpoint=oss_endpoint, + oss_ak=oss_ak, + oss_sk=oss_sk, + oss_timeout=oss_params.get('oss_timeout', 1500), + meta_graph_def=meta_graph_def, + norm_name_to_ids=norm_name_to_ids, + incr_update_params=oss_params.get('incr_update', None), + debug_dir=export_dir if verbose else '' + ) meta_graph_editor.edit_graph_for_oss() tf.reset_default_graph() @@ -513,19 +545,24 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, for tmp_norm_name in norm_name_to_ids: fout.write('%s\t%s\n' % (tmp_norm_name, norm_name_to_ids[tmp_norm_name])) ops.add_to_collection( - ops.GraphKeys.ASSET_FILEPATHS, - tf.constant( - embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt')) + ops.GraphKeys.ASSET_FILEPATHS, + tf.constant( + embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt' + ) + ) if 'incr_update' in oss_params: dense_train_vars_path = os.path.join( - os.path.dirname(checkpoint_path), constant.DENSE_UPDATE_VARIABLES) + os.path.dirname(checkpoint_path), constant.DENSE_UPDATE_VARIABLES + ) ops.add_to_collection( - ops.GraphKeys.ASSET_FILEPATHS, - tf.constant( - dense_train_vars_path, - dtype=tf.string, - name=constant.DENSE_UPDATE_VARIABLES)) + ops.GraphKeys.ASSET_FILEPATHS, + tf.constant( + dense_train_vars_path, + dtype=tf.string, + name=constant.DENSE_UPDATE_VARIABLES + ) + ) asset_file = 'incr_update.txt' asset_file_path = os.path.join(export_dir, asset_file) @@ -535,24 +572,32 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, if 'kafka' in incr_update: incr_update_json['storage'] = 'kafka' incr_update_json['kafka'] = json.loads( - json_format.MessageToJson( - incr_update['kafka'], preserving_proto_field_name=True)) + json_format.MessageToJson( + incr_update['kafka'], preserving_proto_field_name=True + ) + ) elif 'datahub' in incr_update: incr_update_json['storage'] = 'datahub' incr_update_json['datahub'] = json.loads( - json_format.MessageToJson( - incr_update['datahub'], preserving_proto_field_name=True)) + json_format.MessageToJson( + incr_update['datahub'], preserving_proto_field_name=True + ) + ) elif 'fs' in incr_update: incr_update_json['storage'] = 'fs' - incr_update_json['fs'] = {'incr_save_dir': incr_update['fs'].mount_path} + incr_update_json['fs'] = { + 'incr_save_dir': incr_update['fs'].mount_path + } json.dump(incr_update_json, fout, indent=2) ops.add_to_collection( - ops.GraphKeys.ASSET_FILEPATHS, - tf.constant(asset_file_path, dtype=tf.string, name=asset_file)) + ops.GraphKeys.ASSET_FILEPATHS, + tf.constant(asset_file_path, dtype=tf.string, name=asset_file) + ) - export_dir = os.path.join(export_dir, - meta_graph_def.meta_info_def.meta_graph_version) + export_dir = os.path.join( + export_dir, meta_graph_def.meta_info_def.meta_graph_version + ) export_dir = io_util.fix_oss_dir(export_dir) logging.info('export_dir=%s' % export_dir) if Exists(export_dir): @@ -572,10 +617,12 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, tensor_info_outputs[tmp_key] = \ tf.saved_model.utils.build_tensor_info(tmp) signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=tensor_info_inputs, - outputs=tensor_info_outputs, - method_name=signature_constants.PREDICT_METHOD_NAME)) + tf.saved_model.signature_def_utils.build_signature_def( + inputs=tensor_info_inputs, + outputs=tensor_info_outputs, + method_name=signature_constants.PREDICT_METHOD_NAME + ) + ) if 'incr_update' in oss_params: incr_update_inputs = meta_graph_editor.sparse_update_inputs @@ -593,36 +640,42 @@ def export_big_model_to_oss(export_dir, pipeline_config, oss_params, tensor_info_incr_update_outputs[tmp_key] = \ tf.saved_model.utils.build_tensor_info(tmp) incr_update_signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=tensor_info_incr_update_inputs, - outputs=tensor_info_incr_update_outputs, - method_name=signature_constants.PREDICT_METHOD_NAME)) + tf.saved_model.signature_def_utils.build_signature_def( + inputs=tensor_info_incr_update_inputs, + outputs=tensor_info_incr_update_outputs, + method_name=signature_constants.PREDICT_METHOD_NAME + ) + ) else: incr_update_signature = None session_config = ConfigProto( - allow_soft_placement=True, log_device_placement=True) + allow_soft_placement=True, log_device_placement=True + ) saver = tf.train.Saver() with tf.Session(target=server.target if server else '') as sess: saver.restore(sess, checkpoint_path) - main_op = tf.group([ + main_op = tf.group( + [ Scaffold.default_local_init_op(), ops.get_collection(EMBEDDING_INITIALIZERS) - ]) + ] + ) incr_update_sig_map = { - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature } if incr_update_signature is not None: incr_update_sig_map[INCR_UPDATE_SIGNATURE_KEY] = incr_update_signature builder.add_meta_graph_and_variables( - sess, [tf.saved_model.tag_constants.SERVING], - signature_def_map=incr_update_sig_map, - assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), - saver=saver, - main_op=main_op, - strip_default_attrs=True, - clear_devices=True) + sess, [tf.saved_model.tag_constants.SERVING], + signature_def_map=incr_update_sig_map, + assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), + saver=saver, + main_op=main_op, + strip_default_attrs=True, + clear_devices=True + ) builder.save() # remove temporary files diff --git a/easy_rec/python/utils/expr_util.py b/easy_rec/python/utils/expr_util.py index a236bdfd9..b8fe5277e 100644 --- a/easy_rec/python/utils/expr_util.py +++ b/easy_rec/python/utils/expr_util.py @@ -26,7 +26,7 @@ def _process_enum(enum, input_names, prefix=''): def _get_expression_list(expression, input_names, prefix=''): ops = [ - '+', '-', '*', '/', '(', ')', '>', '>=', '<', '<=', '==', '=', '&', '|' + '+', '-', '*', '/', '(', ')', '>', '>=', '<', '<=', '==', '=', '&', '|' ] expression_list = [] eunm = '' @@ -113,6 +113,8 @@ def _expression_eval(expr_list): def get_expression(expression, input_names, prefix=''): - expression_list = _get_expression_list(expression, input_names, prefix=prefix) + expression_list = _get_expression_list( + expression, input_names, prefix=prefix + ) expression = _expression_eval(expression_list) return expression diff --git a/easy_rec/python/utils/fg_util.py b/easy_rec/python/utils/fg_util.py index c394444bf..fc5596ec5 100644 --- a/easy_rec/python/utils/fg_util.py +++ b/easy_rec/python/utils/fg_util.py @@ -1,12 +1,10 @@ import json import logging - import tensorflow as tf from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.protos.feature_config_pb2 import FeatureConfig from easy_rec.python.utils.config_util import get_compatible_feature_configs - from easy_rec.python.utils.convert_rtp_fg import load_input_field_and_feature_config # NOQA if tf.__version__ >= '2.0': @@ -27,7 +25,8 @@ def load_fg_json_to_config(pipeline_config): rtp_fg = json.load(fin) fg_config = load_input_field_and_feature_config( - rtp_fg, label_fields=label_fields) + rtp_fg, label_fields=label_fields + ) pipeline_config.data_config.ClearField('input_fields') pipeline_config.ClearField('feature_configs') diff --git a/easy_rec/python/utils/hit_rate_utils.py b/easy_rec/python/utils/hit_rate_utils.py index 7ae313548..d70fb8fd5 100644 --- a/easy_rec/python/utils/hit_rate_utils.py +++ b/easy_rec/python/utils/hit_rate_utils.py @@ -1,6 +1,5 @@ -import logging - import graphlearn as gl +import logging import numpy as np import tensorflow as tf @@ -27,19 +26,17 @@ def load_graph(i_emb_table, emb_dim, knn_metric, timeout, knn_strict): option.nlist = 5 option.nprobe = 2 g = gl.Graph().node( - i_emb_table, - node_type='i', - decoder=gl.Decoder(attr_types=['float'] * emb_dim, attr_delimiter=','), - option=option) + i_emb_table, + node_type='i', + decoder=gl.Decoder(attr_types=['float'] * emb_dim, attr_delimiter=','), + option=option + ) return g -def batch_hitrate(src_ids, - recall_ids, - recall_distances, - gt_items, - num_interests, - mask=None): +def batch_hitrate( + src_ids, recall_ids, recall_distances, gt_items, num_interests, mask=None +): """Compute hitrate of a batch of src ids. Args: @@ -109,37 +106,42 @@ def reduce_hitrate(cluster, hits, count, task_index): have completed the calculation of hitrate. """ with tf.device( - tf.train.replica_device_setter( - worker_device='/job:worker/task:%d' % task_index, cluster=cluster)): + tf.train.replica_device_setter( + worker_device='/job:worker/task:%d' % task_index, cluster=cluster + ) + ): with tf.variable_scope('hitrate_var', reuse=tf.AUTO_REUSE): var_worker_count = tf.get_variable( - 'worker_count', - shape=(), - dtype=tf.int32, - initializer=tf.zeros_initializer()) + 'worker_count', + shape=(), + dtype=tf.int32, + initializer=tf.zeros_initializer() + ) var_hits = tf.get_variable( - 'hits', - shape=(), - dtype=tf.float32, - initializer=tf.zeros_initializer()) + 'hits', shape=(), dtype=tf.float32, initializer=tf.zeros_initializer() + ) var_gt_count = tf.get_variable( - 'gt_count', - shape=(), - dtype=tf.float32, - initializer=tf.zeros_initializer()) + 'gt_count', + shape=(), + dtype=tf.float32, + initializer=tf.zeros_initializer() + ) var_total_hitrate = tf.get_variable( - 'total_hitate', - shape=(), - dtype=tf.float32, - initializer=tf.zeros_initializer()) + 'total_hitate', + shape=(), + dtype=tf.float32, + initializer=tf.zeros_initializer() + ) var_hits = tf.assign_add(var_hits, hits, use_locking=True) var_gt_count = tf.assign_add(var_gt_count, count, use_locking=True) var_gt_count = tf.Print( - var_gt_count, [var_gt_count, var_hits], - message='var_gt_count/var_hits') + var_gt_count, [var_gt_count, var_hits], + message='var_gt_count/var_hits' + ) var_total_hitrate = tf.assign( - var_total_hitrate, var_hits / var_gt_count, use_locking=True) + var_total_hitrate, var_hits / var_gt_count, use_locking=True + ) with tf.control_dependencies([var_total_hitrate]): var_worker_count = tf.assign_add(var_worker_count, 1, use_locking=True) return var_total_hitrate, var_worker_count @@ -171,7 +173,8 @@ def _to_float_attrs(x): if x == '': return np.zeros([emb_dim], dtype=np.float32) embed = np.array(x.split(','), dtype=np.float32) - assert len(embed) == emb_dim, 'invalid embed len=%d, x=%s' % (len(embed), x) + assert len(embed + ) == emb_dim, 'invalid embed len=%d, x=%s' % (len(embed), x) return embed def _to_multi_float_attrs(x, userid): @@ -180,29 +183,35 @@ def _to_multi_float_attrs(x, userid): else: arr = [_to_float_attrs(sub_x) for sub_x in x.split('|')] assert len(arr) == num_interests, 'invalid arr len=%d, x=%s, userid=%s' % ( - len(arr), x, userid) + len(arr), x, userid + ) return arr src_ids = np.array([src_items[0] for src_items in gt_record]) - user_embedding = np.array([ + user_embedding = np.array( + [ _to_multi_float_attrs(src_items[2], src_items[0]) for src_items in gt_record - ]) + ] + ) user_emb_num = [src_items[3] for src_items in gt_record] - print('max(user_emb_num) = %d len(src_ids) = %d' % - (np.max(user_emb_num), len(src_ids))) + print( + 'max(user_emb_num) = %d len(src_ids) = %d' % + (np.max(user_emb_num), len(src_ids)) + ) # a list of list. gt_items = [ - list(map(int, src_items[1].split(','))) for src_items in gt_record + list(map(int, src_items[1].split(','))) for src_items in gt_record ] logging.info('src_nodes.float_attrs.shape=%s' % str(user_embedding.shape)) user_embedding = user_embedding.reshape([-1, user_embedding.shape[-1]]) # numpy array - recall_ids, recall_distances = g.search('i', user_embedding, - gl.KnnOption(k=top_k)) + recall_ids, recall_distances = g.search( + 'i', user_embedding, gl.KnnOption(k=top_k) + ) logging.info('recall_ids.shape=%s' % str(recall_ids.shape)) def _make_mask(lens): @@ -214,7 +223,9 @@ def _make_mask(lens): mask = _make_mask(user_emb_num) recall_ids = recall_ids.reshape([-1, num_interests, recall_ids.shape[-1]]) recall_distances = recall_distances.reshape( - [-1, num_interests, recall_distances.shape[-1]]) + [-1, num_interests, recall_distances.shape[-1]] + ) hitrates, bad_cases, bad_dists, hits, gt_count = batch_hitrate( - src_ids, recall_ids, recall_distances, gt_items, num_interests, mask) + src_ids, recall_ids, recall_distances, gt_items, num_interests, mask + ) return hits, gt_count, src_ids, recall_ids, recall_distances, hitrates, bad_cases, bad_dists diff --git a/easy_rec/python/utils/hive_utils.py b/easy_rec/python/utils/hive_utils.py index 250344f13..2cf3a5b8a 100644 --- a/easy_rec/python/utils/hive_utils.py +++ b/easy_rec/python/utils/hive_utils.py @@ -38,13 +38,15 @@ def gen_sql(self): class HiveUtils(object): """Common IO based interface, could run at local or on data science.""" - def __init__(self, - data_config, - hive_config, - selected_cols='', - record_defaults=[], - task_index=0, - task_num=1): + def __init__( + self, + data_config, + hive_config, + selected_cols='', + record_defaults=[], + task_index=0, + task_num=1 + ): self._data_config = data_config self._hive_config = hive_config @@ -65,16 +67,18 @@ def _construct_table_info(self, table_name, limit_num): else: partition_kv = None - table_info = TableInfo(table_name, self._selected_cols, partition_kv, - limit_num) + table_info = TableInfo( + table_name, self._selected_cols, partition_kv, limit_num + ) return table_info def _construct_hive_connect(self): conn = hive.Connection( - host=self._hive_config.host, - port=self._hive_config.port, - username=self._hive_config.username, - database=self._hive_config.database) + host=self._hive_config.host, + port=self._hive_config.port, + username=self._hive_config.username, + database=self._hive_config.database + ) return conn def hive_read_line(self, input_path, limit_num=None): @@ -119,13 +123,13 @@ def run_sql(self, sql): data = [] return data - def is_table_or_partition_exist(self, - table_name, - partition_name=None, - partition_val=None): + def is_table_or_partition_exist( + self, table_name, partition_name=None, partition_val=None + ): if partition_name and partition_val: - sql = 'show partitions %s partition(%s=%s)' % (table_name, partition_name, - partition_val) + sql = 'show partitions %s partition(%s=%s)' % ( + table_name, partition_name, partition_val + ) try: res = self.run_sql(sql) if not res: @@ -174,8 +178,8 @@ def get_all_cols(self, input_path): for col in data: col_name = col[0].strip() - if col_name and (not col_name.startswith('#')) and (col_name - not in col_names): + if col_name and (not col_name.startswith('#') + ) and (col_name not in col_names): if col_name != pt_name: col_names.append(col_name) cols_types.append(col[1].strip()) diff --git a/easy_rec/python/utils/hpo_util.py b/easy_rec/python/utils/hpo_util.py index bebf08476..76f10e63c 100644 --- a/easy_rec/python/utils/hpo_util.py +++ b/easy_rec/python/utils/hpo_util.py @@ -3,7 +3,6 @@ import json import logging import os - import psutil import tensorflow as tf from tensorflow.python.summary import summary_iterator @@ -123,8 +122,9 @@ def kill_old_proc(tmp_dir, platform='pai'): # clear easy_rec_hpo yarn jobs yarn_job_file = os.path.join(tmp_dir, 'yarn_job.txt') os.system( - "yarn application -list | awk '{ if ($2 == \"easy_rec_hpo\") print $1 }' > %s" - % yarn_job_file) + "yarn application -list | awk '{ if ($2 == \"easy_rec_hpo\") print $1 }' > %s" + % yarn_job_file + ) yarn_job_arr = [] with open(yarn_job_file, 'r') as fin: for line_str in fin: @@ -132,6 +132,7 @@ def kill_old_proc(tmp_dir, platform='pai'): yarn_job_arr.append(line_str) yarn_job_arr = list(set(yarn_job_arr)) if len(yarn_job_arr) > 0: - logging.info('will kill the easy_rec_hpo yarn jobs: %s' % - ','.join(yarn_job_arr)) + logging.info( + 'will kill the easy_rec_hpo yarn jobs: %s' % ','.join(yarn_job_arr) + ) os.system('yarn application -kill %s' % ' '.join(yarn_job_arr)) diff --git a/easy_rec/python/utils/hvd_utils.py b/easy_rec/python/utils/hvd_utils.py index 223283486..89f828bea 100644 --- a/easy_rec/python/utils/hvd_utils.py +++ b/easy_rec/python/utils/hvd_utils.py @@ -1,6 +1,5 @@ # -*- encoding: utf-8 -*- import logging - import tensorflow as tf from tensorflow.python.framework import ops from tensorflow.python.training import session_run_hook @@ -46,8 +45,10 @@ def begin(self): # if '/embedding' not in x.name and 'DynamicVariable' not in str(type(x)): if x.name not in embed_para_vars: bcast_vars.append(x) - logging.info('will broadcast variable: name=%s shape=%s' % - (x.name, x.get_shape())) + logging.info( + 'will broadcast variable: name=%s shape=%s' % + (x.name, x.get_shape()) + ) if not self.bcast_op or self.bcast_op.graph != tf.get_default_graph(): with tf.device(self.device): self.bcast_op = broadcast_variables(bcast_vars, self.root_rank) diff --git a/easy_rec/python/utils/input_utils.py b/easy_rec/python/utils/input_utils.py index cd8a5b975..03f41c1ed 100644 --- a/easy_rec/python/utils/input_utils.py +++ b/easy_rec/python/utils/input_utils.py @@ -12,12 +12,12 @@ def get_type_defaults(field_type, default_val=''): type_defaults = { - DatasetConfig.INT32: 0, - DatasetConfig.INT64: 0, - DatasetConfig.STRING: '', - DatasetConfig.BOOL: False, - DatasetConfig.FLOAT: 0.0, - DatasetConfig.DOUBLE: 0.0 + DatasetConfig.INT32: 0, + DatasetConfig.INT64: 0, + DatasetConfig.STRING: '', + DatasetConfig.BOOL: False, + DatasetConfig.FLOAT: 0.0, + DatasetConfig.DOUBLE: 0.0 } assert field_type in type_defaults, 'invalid type: %s' % field_type if default_val == '': @@ -49,23 +49,28 @@ def string_to_number(field, ftype, default_value, name=''): Returns: A name for the operation (optional). """ default_vals = tf.tile(tf.constant([str(default_value)]), tf.shape(field)) - field = tf.where(tf.greater(tf.strings.length(field), 0), field, default_vals) + field = tf.where( + tf.greater(tf.strings.length(field), 0), field, default_vals + ) if ftype in [DatasetConfig.INT32, DatasetConfig.INT64]: # Int type is not supported in fg. # If you specify INT32, INT64 in DatasetConfig, you need to perform a cast at here. tmp_field = tf.string_to_number( - field, tf.double, name='field_as_flt_%s' % name) + field, tf.double, name='field_as_flt_%s' % name + ) if ftype in [DatasetConfig.INT64]: tmp_field = tf.cast(tmp_field, tf.int64) else: tmp_field = tf.cast(tmp_field, tf.int32) elif ftype in [DatasetConfig.FLOAT]: tmp_field = tf.string_to_number( - field, tf.float32, name='field_as_flt_%s' % name) + field, tf.float32, name='field_as_flt_%s' % name + ) elif ftype in [DatasetConfig.DOUBLE]: tmp_field = tf.string_to_number( - field, tf.float64, name='field_as_flt_%s' % name) + field, tf.float64, name='field_as_flt_%s' % name + ) elif ftype in [DatasetConfig.BOOL]: tmp_field = tf.logical_or(tf.equal(field, 'True'), tf.equal(field, 'true')) elif ftype in [DatasetConfig.STRING]: @@ -77,14 +82,14 @@ def string_to_number(field, ftype, default_value, name=''): def np_to_tf_type(np_type): _types_map = { - int: tf.int32, - np.int32: tf.int32, - np.int64: tf.int64, - str: tf.string, - np.float: tf.float32, - np.float32: tf.float32, - float: tf.float32, - np.double: tf.float64 + int: tf.int32, + np.int32: tf.int32, + np.int64: tf.int64, + str: tf.string, + np.float: tf.float32, + np.float32: tf.float32, + float: tf.float32, + np.double: tf.float64 } if np_type in _types_map: return _types_map[np_type] diff --git a/easy_rec/python/utils/io_util.py b/easy_rec/python/utils/io_util.py index 92c9c8a1f..687282997 100644 --- a/easy_rec/python/utils/io_util.py +++ b/easy_rec/python/utils/io_util.py @@ -6,6 +6,7 @@ """ import logging from future import standard_library + standard_library.install_aliases() import os @@ -78,11 +79,14 @@ def download(oss_or_url, dst_dir=''): response = urllib.request.urlopen(oss_or_url, timeout=HTTP_MAX_TIMEOUT) file_content = response.read() except Exception as e: - raise RuntimeError('Download %s failed: %s\n %s' % - (oss_or_url, str(e), traceback.format_exc())) + raise RuntimeError( + 'Download %s failed: %s\n %s' % + (oss_or_url, str(e), traceback.format_exc()) + ) else: - tf.logging.warning('skip downloading %s, seems to be a local file' % - oss_or_url) + tf.logging.warning( + 'skip downloading %s, seems to be a local file' % oss_or_url + ) return oss_or_url if dst_dir != '' and not os.path.exists(dst_dir): @@ -130,20 +134,24 @@ def download_and_uncompress_resource(resource_path, dst_dir=EASY_REC_RES_DIR): _, basename = os.path.split(resource_path) if not basename.endswith('.tar.gz') and not basename.endswith('.zip') and \ not basename.endswith('.py'): - raise ValueError('resource %s should be tar.gz or zip or py' % - resource_path) + raise ValueError( + 'resource %s should be tar.gz or zip or py' % resource_path + ) download(resource_path, dst_dir) stat = 0 if basename.endswith('tar.gz'): - stat, output = getstatusoutput('cd %s && tar -zxf %s' % (dst_dir, basename)) + stat, output = getstatusoutput( + 'cd %s && tar -zxf %s' % (dst_dir, basename) + ) elif basename.endswith('zip'): stat, output = getstatusoutput('cd %s && unzip %s' % (dst_dir, basename)) if stat != 0: - raise ValueError('uncompress resoruce %s failed: %s' % resource_path, - output) + raise ValueError( + 'uncompress resoruce %s failed: %s' % resource_path, output + ) return dst_dir @@ -209,8 +217,8 @@ def convert_tf_flags_to_argparse(flags): flag_type = type(default) help_str = flag.help or '' args[flag_name] = [ - False, flag_type, default, help_str, - flag.choices if hasattr(flag, 'choices') else None + False, flag_type, default, help_str, + flag.choices if hasattr(flag, 'choices') else None ] def str2bool(v): @@ -223,45 +231,53 @@ def str2bool(v): else: raise argparse.ArgumentTypeError('Boolean value expected.') - for flag_name, (multi, flag_type, default, help_str, choices) in args.items(): + for flag_name, (multi, flag_type, default, help_str, + choices) in args.items(): if flag_type == bool: parser.add_argument( - '--' + flag_name, - type=str2bool, - nargs='?', - const=True, - default=False, - help=help_str) + '--' + flag_name, + type=str2bool, + nargs='?', + const=True, + default=False, + help=help_str + ) elif flag_type == str: if choices: parser.add_argument( - '--' + flag_name, - type=str, - choices=choices, - default=default, - help=help_str) + '--' + flag_name, + type=str, + choices=choices, + default=default, + help=help_str + ) elif multi: parser.add_argument( - '--' + flag_name, - type=str, - action='append', - default=default, - help=help_str) + '--' + flag_name, + type=str, + action='append', + default=default, + help=help_str + ) else: parser.add_argument( - '--' + flag_name, type=str, default=default, help=help_str) + '--' + flag_name, type=str, default=default, help=help_str + ) elif flag_type in (list, dict): parser.add_argument( - '--' + flag_name, - type=lambda s: ast.literal_eval(s), - default=default, - help=help_str) + '--' + flag_name, + type=lambda s: ast.literal_eval(s), + default=default, + help=help_str + ) elif flag_type in (int, float): parser.add_argument( - '--' + flag_name, type=flag_type, default=default, help=help_str) + '--' + flag_name, type=flag_type, default=default, help=help_str + ) else: parser.add_argument( - '--' + flag_name, type=str, default=default, help=help_str) + '--' + flag_name, type=str, default=default, help=help_str + ) return parser diff --git a/easy_rec/python/utils/load_class.py b/easy_rec/python/utils/load_class.py index 9ac749c76..133bbfd52 100644 --- a/easy_rec/python/utils/load_class.py +++ b/easy_rec/python/utils/load_class.py @@ -7,11 +7,10 @@ import os import pkgutil import pydoc -import traceback -from abc import ABCMeta - import six import tensorflow as tf +import traceback +from abc import ABCMeta import easy_rec from easy_rec.python.utils import compat @@ -73,8 +72,9 @@ def _get_method_declare(aMethod): return sig_str else: spec = inspect.getargspec(aMethod) - args = inspect.formatargspec(spec.args, spec.varargs, spec.keywords, - spec.defaults) + args = inspect.formatargspec( + spec.args, spec.varargs, spec.keywords, spec.defaults + ) return '%s%s' % (name, args) except TypeError: return '%s(cls, ...)' % name @@ -108,8 +108,10 @@ def check_class(cls, impl_cls, function_names=None): missing[name + '()'] = 'method signature differs' if len(missing) > 0: - raise Exception('incompatible Implementation-implementation %s: %s' % - (impl_cls.__class__.__name__, missing)) + raise Exception( + 'incompatible Implementation-implementation %s: %s' % + (impl_cls.__class__.__name__, missing) + ) def import_pkg(pkg_info, prefix_to_remove=None): @@ -158,8 +160,8 @@ def auto_import(user_path=None): """ # True False indicates import recursively or not pre_defined_dirs = [ - ('easy_rec/python/model', False), - ('easy_rec/python/input', False), + ('easy_rec/python/model', False), + ('easy_rec/python/input', False), ] parent_dir = easy_rec.parent_dir @@ -168,9 +170,10 @@ def auto_import(user_path=None): # to make class name starts with easy_rec if parent_dir != '': for idx in range(len(pre_defined_dirs)): - pre_defined_dirs[idx] = (os.path.join(parent_dir, - pre_defined_dirs[idx][0]), - pre_defined_dirs[idx][1]) + pre_defined_dirs[idx] = ( + os.path.join(parent_dir, + pre_defined_dirs[idx][0]), pre_defined_dirs[idx][1] + ) prefix_to_remove = parent_dir + '/' if user_path is not None: @@ -213,8 +216,10 @@ def create_class(cls, name): if name in class_map: return class_map[name] else: - raise Exception('Class %s is not registered. Available ones are %s' % - (name, list(class_map.keys()))) + raise Exception( + 'Class %s is not registered. Available ones are %s' % + (name, list(class_map.keys())) + ) setattr(newclass, 'create_class', create_class) return newclass @@ -244,6 +249,7 @@ def load_keras_layer(name): return pydoc.locate(path), False except pydoc.ErrorDuringImport: print('load keras layer %s failed' % name) - logging.error('load keras layer %s failed: %s' % - (name, traceback.format_exc())) + logging.error( + 'load keras layer %s failed: %s' % (name, traceback.format_exc()) + ) return None, False diff --git a/easy_rec/python/utils/meta_graph_editor.py b/easy_rec/python/utils/meta_graph_editor.py index 9fc75f1fe..0921d8a1e 100644 --- a/easy_rec/python/utils/meta_graph_editor.py +++ b/easy_rec/python/utils/meta_graph_editor.py @@ -1,8 +1,7 @@ # -*- encoding:utf-8 -*- import logging -import os - import numpy as np +import os import tensorflow as tf from google.protobuf import text_format from tensorflow.python.framework import ops @@ -11,32 +10,31 @@ from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model.loader_impl import SavedModelLoader -from easy_rec.python.utils import conditional -from easy_rec.python.utils import constant -from easy_rec.python.utils import embedding_utils -from easy_rec.python.utils import proto_util +from easy_rec.python.utils import conditional, constant, embedding_utils, proto_util # NOQA EMBEDDING_INITIALIZERS = 'embedding_initializers' class MetaGraphEditor: - def __init__(self, - lookup_lib_path, - saved_model_dir, - redis_url=None, - redis_passwd=None, - redis_timeout=0, - redis_cache_names=[], - oss_path=None, - oss_endpoint=None, - oss_ak=None, - oss_sk=None, - oss_timeout=0, - meta_graph_def=None, - norm_name_to_ids=None, - incr_update_params=None, - debug_dir=''): + def __init__( + self, + lookup_lib_path, + saved_model_dir, + redis_url=None, + redis_passwd=None, + redis_timeout=0, + redis_cache_names=[], + oss_path=None, + oss_endpoint=None, + oss_ak=None, + oss_sk=None, + oss_timeout=0, + meta_graph_def=None, + norm_name_to_ids=None, + incr_update_params=None, + debug_dir='' + ): self._lookup_op = tf.load_op_library(lookup_lib_path) self._debug_dir = debug_dir self._verbose = debug_dir != '' @@ -50,11 +48,12 @@ def __init__(self, tf.reset_default_graph() from tensorflow.python.framework import meta_graph meta_graph.import_scoped_meta_graph_with_return_elements( - meta_graph_def, clear_devices=True) + meta_graph_def, clear_devices=True + ) # tf.train.import_meta_graph(meta_graph_def) self._meta_graph_version = meta_graph_def.meta_info_def.meta_graph_version self._signature_def = meta_graph_def.signature_def[ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] if self._verbose: debug_out_path = os.path.join(self._debug_dir, 'meta_graph_raw.txt') @@ -163,10 +162,12 @@ def _get_share_embed_name(self, x, embed_names): embed_name_sub = '/'.join(tmp_toks) if tmp_name == embed_name_sub: assert not sel_embed_name, 'confusions encountered: %s %s' % ( - x, ','.join(embed_names)) + x, ','.join(embed_names) + ) sel_embed_name = embed_name assert sel_embed_name, '%s not find in shared_embeddings: %s' % ( - tmp_name, ','.join(embed_names)) + tmp_name, ','.join(embed_names) + ) return sel_embed_name def _find_embed_combiners(self, norm_embed_names): @@ -180,30 +181,30 @@ def _find_embed_combiners(self, norm_embed_names): embed_combiners = {} embed_combine_node_cts = {} combiner_map = { - 'SparseSegmentSum': 'sum', - 'SparseSegmentMean': 'mean', - 'SparseSegmentSqrtN': 'sqrtn' + 'SparseSegmentSum': 'sum', + 'SparseSegmentMean': 'mean', + 'SparseSegmentSqrtN': 'sqrtn' } for node in self._meta_graph_def.graph_def.node: if node.op in combiner_map: norm_name, _ = proto_util.get_norm_embed_name(node.name) embed_combiners[norm_name] = combiner_map[node.op] - embed_combine_node_cts[norm_name] = embed_combine_node_cts.get( - norm_name, 0) + 1 + embed_combine_node_cts[norm_name + ] = embed_combine_node_cts.get(norm_name, 0) + 1 elif node.op == 'RealDiv' and len(node.input) == 2: # for tag feature with weights, and combiner == mean if 'SegmentSum' in node.input[0] and 'SegmentSum' in node.input[1]: norm_name, _ = proto_util.get_norm_embed_name(node.name) embed_combiners[norm_name] = 'mean' - embed_combine_node_cts[norm_name] = embed_combine_node_cts.get( - norm_name, 0) + 1 + embed_combine_node_cts[ + norm_name] = embed_combine_node_cts.get(norm_name, 0) + 1 elif node.op == 'SegmentSum': norm_name, _ = proto_util.get_norm_embed_name(node.name) # avoid overwrite RealDiv results if norm_name not in embed_combiners: embed_combiners[norm_name] = 'sum' - embed_combine_node_cts[norm_name] = embed_combine_node_cts.get( - norm_name, 0) + 1 + embed_combine_node_cts[norm_name + ] = embed_combine_node_cts.get(norm_name, 0) + 1 return [embed_combiners[x] for x in norm_embed_names] def _find_lookup_indices_values_shapes(self): @@ -229,10 +230,13 @@ def _get_output_shape(graph_def, input_name): if '_embedding_weights/SparseReshape' in node.name: if node.op == 'SparseReshape': # embed_name, _ = proto_util.get_norm_embed_name(node.name, self._verbose) - fea_name, _ = proto_util.get_norm_embed_name(node.name, self._verbose) + fea_name, _ = proto_util.get_norm_embed_name( + node.name, self._verbose + ) for tmp_input in node.input: - tmp_shape = _get_output_shape(self._meta_graph_def.graph_def, - tmp_input) + tmp_shape = _get_output_shape( + self._meta_graph_def.graph_def, tmp_input + ) if '_embedding_weights/Cast' in tmp_input: continue elif len(tmp_shape.dim) == 2: @@ -240,7 +244,9 @@ def _get_output_shape(graph_def, input_name): elif len(tmp_shape.dim) == 1: shapes[fea_name] = tmp_input elif node.op == 'Identity': - fea_name, _ = proto_util.get_norm_embed_name(node.name, self._verbose) + fea_name, _ = proto_util.get_norm_embed_name( + node.name, self._verbose + ) values[fea_name] = node.input[0] return indices, values, shapes @@ -275,13 +281,15 @@ def _find_embed_names_and_dims(self, norm_embed_names): embed_is_kv = {} for node in self._meta_graph_def.graph_def.node: if 'embedding_weights' in node.name and node.op in [ - 'VariableV2', 'KvVarHandleOp' + 'VariableV2', 'KvVarHandleOp' ]: tmp = node.attr['shape'].shape.dim[-1].size tmp2 = 1 for x in node.attr['shape'].shape.dim[:-1]: tmp2 = tmp2 * x.size - embed_name, _ = proto_util.get_norm_embed_name(node.name, self._verbose) + embed_name, _ = proto_util.get_norm_embed_name( + node.name, self._verbose + ) assert embed_name is not None,\ 'fail to get_norm_embed_name(%s)' % node.name embed_dims[embed_name] = tmp @@ -316,8 +324,10 @@ def find_lookup_inputs(self): weights = self._find_lookup_weights() for fea in shapes.keys(): - logging.info('Lookup Input[%s]: indices=%s values=%s shapes=%s' % - (fea, indices[fea], values[fea], shapes[fea])) + logging.info( + 'Lookup Input[%s]: indices=%s values=%s shapes=%s' % + (fea, indices[fea], values[fea], shapes[fea]) + ) graph = tf.get_default_graph() @@ -351,15 +361,16 @@ def _get_tensor_by_name(tensor_name): if not self._embed_name_to_ids: embed_name_uniq = list(set(self._embed_names)) self._embed_name_to_ids = { - t: tid for tid, t in enumerate(embed_name_uniq) + t: tid + for tid, t in enumerate(embed_name_uniq) } self._embed_ids = [ - int(self._embed_name_to_ids[x]) for x in self._embed_names + int(self._embed_name_to_ids[x]) for x in self._embed_names ] self._is_cache_from_redis = [ - proto_util.is_cache_from_redis(x, self._redis_cache_names) - for x in self._embed_names + proto_util.is_cache_from_redis(x, self._redis_cache_names) + for x in self._embed_names ] # normalized feature names @@ -368,8 +379,10 @@ def _get_tensor_by_name(tensor_name): return lookup_input_indices, lookup_input_values, lookup_input_shapes,\ lookup_input_weights - def add_lookup_op(self, lookup_input_indices, lookup_input_values, - lookup_input_shapes, lookup_input_weights): + def add_lookup_op( + self, lookup_input_indices, lookup_input_values, lookup_input_shapes, + lookup_input_weights + ): logging.info('add custom lookup operation to lookup embeddings from redis') self._lookup_outs = [None for i in range(len(lookup_input_values))] for i in range(len(lookup_input_values)): @@ -378,18 +391,19 @@ def add_lookup_op(self, lookup_input_indices, lookup_input_values, for i in range(len(self._lookup_outs)): i_1 = i + 1 self._lookup_outs[i] = self._lookup_op.kv_lookup( - lookup_input_indices[i:i_1], - lookup_input_values[i:i_1], - lookup_input_shapes[i:i_1], - lookup_input_weights[i:i_1], - url=self._redis_url, - password=self._redis_passwd, - timeout=self._redis_timeout, - combiners=self._embed_combiners[i:i_1], - embedding_dims=self._embed_dims[i:i_1], - embedding_names=self._embed_ids[i:i_1], - cache=self._is_cache_from_redis, - version=self._meta_graph_version)[0] + lookup_input_indices[i:i_1], + lookup_input_values[i:i_1], + lookup_input_shapes[i:i_1], + lookup_input_weights[i:i_1], + url=self._redis_url, + password=self._redis_passwd, + timeout=self._redis_timeout, + combiners=self._embed_combiners[i:i_1], + embedding_dims=self._embed_dims[i:i_1], + embedding_names=self._embed_ids[i:i_1], + cache=self._is_cache_from_redis, + version=self._meta_graph_version + )[0] meta_graph_def = tf.train.export_meta_graph() @@ -397,12 +411,16 @@ def add_lookup_op(self, lookup_input_indices, lookup_input_values, debug_path = os.path.join(self._debug_dir, 'graph_raw.txt') with GFile(debug_path, 'w') as fout: fout.write( - text_format.MessageToString( - self._meta_graph_def.graph_def, as_utf8=True)) + text_format.MessageToString( + self._meta_graph_def.graph_def, as_utf8=True + ) + ) return meta_graph_def - def add_oss_lookup_op(self, lookup_input_indices, lookup_input_values, - lookup_input_shapes, lookup_input_weights): + def add_oss_lookup_op( + self, lookup_input_indices, lookup_input_values, lookup_input_shapes, + lookup_input_weights + ): logging.info('add custom lookup operation to lookup embeddings from oss') place_on_cpu = os.getenv('place_embedding_on_cpu') place_on_cpu = eval(place_on_cpu) if place_on_cpu else False @@ -431,21 +449,22 @@ def add_oss_lookup_op(self, lookup_input_indices, lookup_input_values, # shared_name='embedding_lookup_res', # name='embedding_lookup_fused/lookup')[0] self._lookup_outs = self._lookup_op.oss_read_kv( - lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights, - osspath=self._oss_path, - endpoint=self._oss_endpoint, - ak=self._oss_ak, - sk=self._oss_sk, - timeout=self._oss_timeout, - combiners=self._embed_combiners, - embedding_dims=self._embed_dims, - embedding_ids=self._embed_ids, - embedding_is_kv=self._embed_is_kv, - shared_name='embedding_lookup_res', - name='embedding_lookup_fused/lookup') + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + osspath=self._oss_path, + endpoint=self._oss_endpoint, + ak=self._oss_ak, + sk=self._oss_sk, + timeout=self._oss_timeout, + combiners=self._embed_combiners, + embedding_dims=self._embed_dims, + embedding_ids=self._embed_ids, + embedding_is_kv=self._embed_is_kv, + shared_name='embedding_lookup_res', + name='embedding_lookup_fused/lookup' + ) N = np.max([int(x) for x in self._embed_ids]) + 1 uniq_embed_ids = [x for x in range(N)] @@ -453,24 +472,26 @@ def add_oss_lookup_op(self, lookup_input_indices, lookup_input_values, uniq_embed_combiners = ['mean' for x in range(N)] uniq_embed_is_kvs = [0 for x in range(N)] for embed_id, embed_combiner, embed_is_kv, embed_dim in zip( - self._embed_ids, self._embed_combiners, self._embed_is_kv, - self._embed_dims): + self._embed_ids, self._embed_combiners, self._embed_is_kv, + self._embed_dims + ): uniq_embed_combiners[embed_id] = embed_combiner uniq_embed_is_kvs[embed_id] = embed_is_kv uniq_embed_dims[embed_id] = embed_dim lookup_init_op = self._lookup_op.oss_init( - osspath=self._oss_path, - endpoint=self._oss_endpoint, - ak=self._oss_ak, - sk=self._oss_sk, - combiners=uniq_embed_combiners, - embedding_dims=uniq_embed_dims, - embedding_ids=uniq_embed_ids, - embedding_is_kv=uniq_embed_is_kvs, - N=N, - shared_name='embedding_lookup_res', - name='embedding_lookup_fused/init') + osspath=self._oss_path, + endpoint=self._oss_endpoint, + ak=self._oss_ak, + sk=self._oss_sk, + combiners=uniq_embed_combiners, + embedding_dims=uniq_embed_dims, + embedding_ids=uniq_embed_ids, + embedding_is_kv=uniq_embed_is_kvs, + N=N, + shared_name='embedding_lookup_res', + name='embedding_lookup_fused/init' + ) ops.add_to_collection(EMBEDDING_INITIALIZERS, lookup_init_op) @@ -478,12 +499,13 @@ def add_oss_lookup_op(self, lookup_input_indices, lookup_input_values, # all sparse variables are updated by a single custom operation message_ph = tf.placeholder(tf.int8, [None], name='incr_update/message') embedding_update = self._lookup_op.embedding_update( - message=message_ph, - shared_name='embedding_lookup_res', - name='embedding_lookup_fused/embedding_update') + message=message_ph, + shared_name='embedding_lookup_res', + name='embedding_lookup_fused/embedding_update' + ) self._embedding_update_inputs['incr_update/sparse/message'] = message_ph - self._embedding_update_outputs[ - 'incr_update/sparse/embedding_update'] = embedding_update + self._embedding_update_outputs['incr_update/sparse/embedding_update' + ] = embedding_update # dense variables are updated one by one dense_name_to_ids = embedding_utils.get_dense_name_to_ids() @@ -492,7 +514,8 @@ def add_oss_lookup_op(self, lookup_input_indices, lookup_input_values, dense_input_name = 'incr_update/dense/%d/input' % dense_var_id dense_output_name = 'incr_update/dense/%d/output' % dense_var_id dense_update_input = tf.placeholder( - tf.float32, x.get_shape(), name=dense_input_name) + tf.float32, x.get_shape(), name=dense_input_name + ) self._dense_update_inputs[dense_input_name] = dense_update_input dense_assign_op = tf.assign(x, dense_update_input) self._dense_update_outputs[dense_output_name] = dense_assign_op @@ -503,8 +526,10 @@ def add_oss_lookup_op(self, lookup_input_indices, lookup_input_values, debug_path = os.path.join(self._debug_dir, 'graph_raw.txt') with GFile(debug_path, 'w') as fout: fout.write( - text_format.MessageToString( - self._meta_graph_def.graph_def, as_utf8=True)) + text_format.MessageToString( + self._meta_graph_def.graph_def, as_utf8=True + ) + ) return meta_graph_def def bytes2str(self, x): @@ -522,14 +547,16 @@ def clear_meta_graph_embeding(self, meta_graph_def): def _clear_embedding_in_meta_collect(meta_graph_def, collect_name): tmp_vals = [ - x - for x in meta_graph_def.collection_def[collect_name].bytes_list.value - if 'embedding_weights' not in self.bytes2str(x) + x for x in meta_graph_def.collection_def[collect_name].bytes_list.value + if 'embedding_weights' not in self.bytes2str(x) ] - meta_graph_def.collection_def[collect_name].bytes_list.ClearField('value') + meta_graph_def.collection_def[collect_name].bytes_list.ClearField( + 'value' + ) for tmp_v in tmp_vals: meta_graph_def.collection_def[collect_name].bytes_list.value.append( - tmp_v) + tmp_v + ) _clear_embedding_in_meta_collect(meta_graph_def, 'model_variables') _clear_embedding_in_meta_collect(meta_graph_def, 'trainable_variables') @@ -537,11 +564,11 @@ def _clear_embedding_in_meta_collect(meta_graph_def, collect_name): # clear Kv(pai embedding variable) ops in meta_info_def.stripped_op_list.op kept_ops = [ - x for x in meta_graph_def.meta_info_def.stripped_op_list.op - if x.name not in [ - 'InitializeKvVariableOp', 'KvResourceGather', 'KvResourceImportV2', - 'KvVarHandleOp', 'KvVarIsInitializedOp', 'ReadKvVariableOp' - ] + x for x in meta_graph_def.meta_info_def.stripped_op_list.op + if x.name not in [ + 'InitializeKvVariableOp', 'KvResourceGather', 'KvResourceImportV2', + 'KvVarHandleOp', 'KvVarIsInitializedOp', 'ReadKvVariableOp' + ] ] meta_graph_def.meta_info_def.stripped_op_list.ClearField('op') meta_graph_def.meta_info_def.stripped_op_list.op.extend(kept_ops) @@ -558,7 +585,7 @@ def clear_meta_collect(self, meta_graph_def): val = meta_graph_def.collection_def[key] if val.HasField('node_list'): if 'embedding_weights' in val.node_list.value[ - 0] and 'easy_rec' not in val.node_list.value[0]: + 0] and 'easy_rec' not in val.node_list.value[0]: drop_meta_collects.append(key) elif key == 'saved_model_assets': drop_meta_collects.append(key) @@ -575,7 +602,7 @@ def _should_drop(name): logging.info('remove embedding_weights node in graph_def.node') logging.info( - 'and replace the old embedding_lookup outputs with new lookup_op outputs' + 'and replace the old embedding_lookup outputs with new lookup_op outputs' ) for tid, node in enumerate(self._all_graph_nodes): @@ -586,10 +613,11 @@ def _should_drop(name): for i in range(len(node.input)): if _should_drop(node.input[i]): input_name, _ = proto_util.get_norm_embed_name( - node.input[i], self._verbose) + node.input[i], self._verbose + ) print('REPLACE:' + node.input[i] + '=>' + input_name) - input_name = self._lookup_outs[self._feature_names.index( - input_name)].name + input_name = self._lookup_outs[ + self._feature_names.index(input_name)].name if input_name.endswith(':0'): input_name = input_name.replace(':0', '') node.input[i] = input_name @@ -597,7 +625,7 @@ def _should_drop(name): # drop by ids def _drop_by_ids(self, tmp_obj, key, drop_ids): keep_vals = [ - x for i, x in enumerate(getattr(tmp_obj, key)) if i not in drop_ids + x for i, x in enumerate(getattr(tmp_obj, key)) if i not in drop_ids ] tmp_obj.ClearField(key) getattr(tmp_obj, key).extend(keep_vals) @@ -622,24 +650,28 @@ def clear_save_restore(self): if self._restore_tensor_node: drop_ids = [] for tmp_id, tmp_name in enumerate( - self._restore_tensor_node.attr['value'].tensor.string_val): + self._restore_tensor_node.attr['value'].tensor.string_val + ): if 'embedding_weights' in self.bytes2str(tmp_name): drop_ids.append(tmp_id) - self._drop_by_ids(self._restore_tensor_node.attr['value'].tensor, - 'string_val', drop_ids) + self._drop_by_ids( + self._restore_tensor_node.attr['value'].tensor, 'string_val', drop_ids + ) keep_node_num = len( - self._restore_tensor_node.attr['value'].tensor.string_val) + self._restore_tensor_node.attr['value'].tensor.string_val + ) logging.info( - 'update self._restore_tensor_node: string_val keep_num = %d drop_num = %d' - % (keep_node_num, len(drop_ids))) + 'update self._restore_tensor_node: string_val keep_num = %d drop_num = %d' + % (keep_node_num, len(drop_ids)) + ) self._restore_tensor_node.attr['value'].tensor.tensor_shape.dim[ - 0].size = keep_node_num + 0].size = keep_node_num self._restore_tensor_node.attr['_output_shapes'].list.shape[0].dim[ - 0].size = keep_node_num + 0].size = keep_node_num logging.info( - 'update save/RestoreV2, drop tensor_shapes, _output_shapes, related to embedding_weights' + 'update save/RestoreV2, drop tensor_shapes, _output_shapes, related to embedding_weights' ) self._restore_shard_node = None for node_id, node in enumerate(self._all_graph_nodes): @@ -659,7 +691,7 @@ def clear_save_restore(self): def clear_save_assign(self): logging.info( - 'update save/Assign, drop tensor_shapes, _output_shapes, related to embedding_weights' + 'update save/Assign, drop tensor_shapes, _output_shapes, related to embedding_weights' ) # edit save/Assign drop_save_assigns = [] @@ -695,8 +727,8 @@ def clear_save_assign(self): # node.input[0] in save/RestoreV2/tensor_names # the outputs of save/RestoreV2 is connected to save/Assign tmp_id = [ - self.bytes2str(x) - for x in self._restore_tensor_node.attr['value'].tensor.string_val + self.bytes2str(x) + for x in self._restore_tensor_node.attr['value'].tensor.string_val ].index(node.input[0]) if tmp_id != 0: tmp_input2 = 'save/RestoreV2:%d' % tmp_id @@ -704,8 +736,10 @@ def clear_save_assign(self): tmp_input2 = 'save/RestoreV2' if tmp_input2 != node.input[1]: if self._verbose: - logging.info("update save/Assign[%s]'s input from %s to %s" % - (node.name, node.input[1], tmp_input2)) + logging.info( + "update save/Assign[%s]'s input from %s to %s" % + (node.name, node.input[1], tmp_input2) + ) node.input[1] = tmp_input2 # save/restore_all need save/restore_shard as input @@ -735,7 +769,9 @@ def clear_save_v2(self): save/SaveV2 input: [ save/SaveV2/tensor_names, save/SaveV2/shape_and_slices ] edit save/SaveV2 save/SaveV2/shape_and_slices save/SaveV2/tensor_names. """ - logging.info('update save/SaveV2 input shape, _output_shapes, tensor_shape') + logging.info( + 'update save/SaveV2 input shape, _output_shapes, tensor_shape' + ) save_drop_ids = [] for tid, node in enumerate(self._all_graph_nodes): if not self._all_graph_node_flags[tid]: @@ -754,23 +790,30 @@ def clear_save_v2(self): if node.name == 'save/SaveV2/shape_and_slices' and node.op == 'Const': # _output_shapes # size # string_val node.attr['_output_shapes'].list.shape[0].dim[0].size -= len( - save_drop_ids) - node.attr['value'].tensor.tensor_shape.dim[0].size -= len(save_drop_ids) - self._drop_by_ids(node.attr['value'].tensor, 'string_val', - save_drop_ids) + save_drop_ids + ) + node.attr['value'].tensor.tensor_shape.dim[0].size -= len( + save_drop_ids + ) + self._drop_by_ids( + node.attr['value'].tensor, 'string_val', save_drop_ids + ) elif node.name == 'save/SaveV2/tensor_names': # tensor_names may not have the same order as save/SaveV2/shape_and_slices tmp_drop_ids = [ - tmp_id for tmp_id, tmp_val in enumerate( - node.attr['value'].tensor.string_val) - if 'embedding_weights' in self.bytes2str(tmp_val) + tmp_id for tmp_id, tmp_val in + enumerate(node.attr['value'].tensor.string_val) + if 'embedding_weights' in self.bytes2str(tmp_val) ] # attr['value'].tensor.string_val # tensor_shape # size assert len(save_drop_ids) == len(save_drop_ids) node.attr['_output_shapes'].list.shape[0].dim[0].size -= len( - tmp_drop_ids) + tmp_drop_ids + ) node.attr['value'].tensor.tensor_shape.dim[0].size -= len(tmp_drop_ids) - self._drop_by_ids(node.attr['value'].tensor, 'string_val', tmp_drop_ids) + self._drop_by_ids( + node.attr['value'].tensor, 'string_val', tmp_drop_ids + ) def clear_initialize(self): """Clear initialization ops. @@ -792,7 +835,8 @@ def clear_initialize(self): elif 'embedding_weights' in node.name and node.op == 'VariableV2': self._all_graph_node_flags[tid] = False elif 'embedding_weights' in node.name and node.name.endswith( - '/read') and node.op == 'Identity': + '/read' + ) and node.op == 'Identity': self._all_graph_node_flags[tid] = False elif 'embedding_weights' in node.name and node.op == 'Identity': node_toks = node.name.split('/') @@ -811,16 +855,15 @@ def clear_embedding_variable(self): if not self._all_graph_node_flags[tid]: continue if node.op in [ - 'ReadKvVariableOp', 'KvVarIsInitializedOp', 'KvVarHandleOp' + 'ReadKvVariableOp', 'KvVarIsInitializedOp', 'KvVarHandleOp' ]: self._all_graph_node_flags[tid] = False # there maybe some nodes depend on the dropped nodes, they are dropped as well def drop_dependent_nodes(self): drop_names = [ - tmp_node.name - for tid, tmp_node in enumerate(self._all_graph_nodes) - if not self._all_graph_node_flags[tid] + tmp_node.name for tid, tmp_node in enumerate(self._all_graph_nodes) + if not self._all_graph_node_flags[tid] ] while True: more_drop_names = [] @@ -828,8 +871,10 @@ def drop_dependent_nodes(self): if not self._all_graph_node_flags[tid]: continue if len(tmp_node.input) > 0 and tmp_node.input[0] in drop_names: - logging.info('drop dependent node: %s depend on %s' % - (tmp_node.name, tmp_node.input[0])) + logging.info( + 'drop dependent node: %s depend on %s' % + (tmp_node.name, tmp_node.input[0]) + ) self._all_graph_node_flags[tid] = False more_drop_names.append(tmp_node.name) drop_names = more_drop_names @@ -842,10 +887,10 @@ def edit_graph(self): lookup_input_weights = self.find_lookup_inputs() # add lookup op to the graph - self._meta_graph_def = self.add_lookup_op(lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights) + self._meta_graph_def = self.add_lookup_op( + lookup_input_indices, lookup_input_values, lookup_input_shapes, + lookup_input_weights + ) self.clear_meta_graph_embeding(self._meta_graph_def) @@ -871,10 +916,12 @@ def edit_graph(self): self.drop_dependent_nodes() self._meta_graph_def.graph_def.ClearField('node') - self._meta_graph_def.graph_def.node.extend([ + self._meta_graph_def.graph_def.node.extend( + [ x for tid, x in enumerate(self._all_graph_nodes) if self._all_graph_node_flags[tid] - ]) + ] + ) logging.info('old node number = %d' % self._old_node_num) logging.info('node number = %d' % len(self._meta_graph_def.graph_def.node)) @@ -886,7 +933,8 @@ def edit_graph(self): debug_dump_path = os.path.join(self._debug_dir, 'meta_graph.txt') with GFile(debug_dump_path, 'w') as fout: fout.write( - text_format.MessageToString(self._meta_graph_def, as_utf8=True)) + text_format.MessageToString(self._meta_graph_def, as_utf8=True) + ) def edit_graph_for_oss(self): # the main entrance @@ -894,10 +942,10 @@ def edit_graph_for_oss(self): lookup_input_weights = self.find_lookup_inputs() # add lookup op to the graph - self._meta_graph_def = self.add_oss_lookup_op(lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights) + self._meta_graph_def = self.add_oss_lookup_op( + lookup_input_indices, lookup_input_values, lookup_input_shapes, + lookup_input_weights + ) self.clear_meta_graph_embeding(self._meta_graph_def) @@ -923,10 +971,12 @@ def edit_graph_for_oss(self): self.drop_dependent_nodes() self._meta_graph_def.graph_def.ClearField('node') - self._meta_graph_def.graph_def.node.extend([ + self._meta_graph_def.graph_def.node.extend( + [ x for tid, x in enumerate(self._all_graph_nodes) if self._all_graph_node_flags[tid] - ]) + ] + ) logging.info('old node number = %d' % self._old_node_num) logging.info('node number = %d' % len(self._meta_graph_def.graph_def.node)) @@ -938,4 +988,5 @@ def edit_graph_for_oss(self): debug_dump_path = os.path.join(self._debug_dir, 'meta_graph.txt') with GFile(debug_dump_path, 'w') as fout: fout.write( - text_format.MessageToString(self._meta_graph_def, as_utf8=True)) + text_format.MessageToString(self._meta_graph_def, as_utf8=True) + ) diff --git a/easy_rec/python/utils/multi_optimizer.py b/easy_rec/python/utils/multi_optimizer.py index c34c4abe0..fbfad1fff 100644 --- a/easy_rec/python/utils/multi_optimizer.py +++ b/easy_rec/python/utils/multi_optimizer.py @@ -24,7 +24,8 @@ def compute_gradients(self, loss, variables, **kwargs): grad_and_vars = [] for gid, opt in enumerate(self._opts): grad_and_vars.extend( - opt.compute_gradients(loss, self._grouped_vars[gid], **kwargs)) + opt.compute_gradients(loss, self._grouped_vars[gid], **kwargs) + ) return grad_and_vars def apply_gradients(self, grads_and_vars, global_step=None, name=None): diff --git a/easy_rec/python/utils/numpy_utils.py b/easy_rec/python/utils/numpy_utils.py index cfda857d2..84b9d8469 100644 --- a/easy_rec/python/utils/numpy_utils.py +++ b/easy_rec/python/utils/numpy_utils.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import json - import numpy as np diff --git a/easy_rec/python/utils/odps_util.py b/easy_rec/python/utils/odps_util.py index 47a9d9870..4ad4d8981 100644 --- a/easy_rec/python/utils/odps_util.py +++ b/easy_rec/python/utils/odps_util.py @@ -9,9 +9,9 @@ def is_type_compatiable(odps_type, input_type): """Check that odps_type are compatiable with input_type.""" type_map = { - 'bigint': DatasetConfig.INT64, - 'string': DatasetConfig.STRING, - 'double': DatasetConfig.DOUBLE + 'bigint': DatasetConfig.INT64, + 'string': DatasetConfig.STRING, + 'double': DatasetConfig.DOUBLE } tmp_type = type_map[odps_type] if tmp_type == input_type: @@ -30,9 +30,9 @@ def is_type_compatiable(odps_type, input_type): def odps_type_to_input_type(odps_type): """Check that odps_type are compatiable with input_type.""" odps_type_map = { - 'bigint': DatasetConfig.INT64, - 'string': DatasetConfig.STRING, - 'double': DatasetConfig.DOUBLE + 'bigint': DatasetConfig.INT64, + 'string': DatasetConfig.STRING, + 'double': DatasetConfig.DOUBLE } assert odps_type in odps_type_map, 'only support [bigint, string, double]' input_type = odps_type_map[odps_type] diff --git a/easy_rec/python/utils/pai_util.py b/easy_rec/python/utils/pai_util.py index de7ce99aa..d1f44aa90 100644 --- a/easy_rec/python/utils/pai_util.py +++ b/easy_rec/python/utils/pai_util.py @@ -4,15 +4,14 @@ import logging import os import sys -import traceback - import tensorflow as tf +import traceback if sys.version_info.major == 2: - from urllib2 import urlopen, Request, HTTPError + from urllib2 import HTTPError, Request, urlopen else: - from urllib.request import urlopen, Request from urllib.error import HTTPError + from urllib.request import Request, urlopen def is_on_pai(): @@ -77,7 +76,7 @@ def process_config(configs, task_index=0, worker_num=1): def test(): f = download( - 'https://easy-rec.oss-cn-hangzhou.aliyuncs.com/config/MultiTower/dwd_avazu_ctr_deepmodel.config' + 'https://easy-rec.oss-cn-hangzhou.aliyuncs.com/config/MultiTower/dwd_avazu_ctr_deepmodel.config' ) assert f == 'dwd_avazu_ctr_deepmodel.config' diff --git a/easy_rec/python/utils/proto_util.py b/easy_rec/python/utils/proto_util.py index c96d41a78..80e819a91 100644 --- a/easy_rec/python/utils/proto_util.py +++ b/easy_rec/python/utils/proto_util.py @@ -85,6 +85,8 @@ def is_cache_from_redis(name, redis_cache_names): for y in redis_cache_names: for k in tok: if k.startswith(y): - logging.info('embedding %s will be cached[specified by %s]' % (name, y)) + logging.info( + 'embedding %s will be cached[specified by %s]' % (name, y) + ) return True return False diff --git a/easy_rec/python/utils/restore_filter.py b/easy_rec/python/utils/restore_filter.py index b11934725..55ed7378c 100644 --- a/easy_rec/python/utils/restore_filter.py +++ b/easy_rec/python/utils/restore_filter.py @@ -2,8 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Define filters for restore.""" -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from enum import Enum diff --git a/easy_rec/python/utils/shape_utils.py b/easy_rec/python/utils/shape_utils.py index f54521513..5d6c17518 100644 --- a/easy_rec/python/utils/shape_utils.py +++ b/easy_rec/python/utils/shape_utils.py @@ -14,9 +14,7 @@ # limitations under the License. # ============================================================================== """Utils used to manipulate tensor shapes.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import six import tensorflow as tf @@ -64,7 +62,8 @@ def merge_shape(t, shape_list): """ t_shape = t.get_shape().as_list() assert len(shape_list) == len( - t_shape), 'input shape size should be the same of the tensor' + t_shape + ), 'input shape size should be the same of the tensor' for idx, size in enumerate(shape_list): if size is not None: t_shape[idx] = size @@ -90,8 +89,9 @@ def pad_tensor(t, length): t_d0 = t_shape[0] pad_d0 = tf.expand_dims(length - t_d0, 0) pad_shape = tf.cond( - tf.greater(t_rank, 1), lambda: tf.concat([pad_d0, t_shape[1:]], 0), - lambda: tf.expand_dims(length - t_d0, 0)) + tf.greater(t_rank, 1), lambda: tf.concat([pad_d0, t_shape[1:]], 0), + lambda: tf.expand_dims(length - t_d0, 0) + ) padded_t = tf.concat([t, tf.zeros(pad_shape, dtype=t.dtype)], 0) if not _is_tensor(length): padded_t = _set_dim_0(padded_t, length) @@ -147,15 +147,16 @@ def pad_nd(tensor, output_shape): tensor_shape = tf.shape(tensor) trailing_paddings = [ - shape - tensor_shape[i] if shape is not None else 0 - for i, shape in enumerate(output_shape) + shape - tensor_shape[i] if shape is not None else 0 + for i, shape in enumerate(output_shape) ] paddings = tf.stack( - [tf.zeros(len(trailing_paddings), dtype=tf.int32), trailing_paddings], - axis=1) + [tf.zeros(len(trailing_paddings), dtype=tf.int32), trailing_paddings], + axis=1 + ) padded_tensor = tf.pad(tensor, paddings=paddings) output_static_shape = [ - dim if not isinstance(dim, tf.Tensor) else None for dim in output_shape + dim if not isinstance(dim, tf.Tensor) else None for dim in output_shape ] padded_tensor.set_shape(output_static_shape) return padded_tensor @@ -174,11 +175,13 @@ def pad_or_clip_nd(tensor, output_shape): """ tensor_shape = tf.shape(tensor) clip_size = [ - tf.where(tensor_shape[i] - shape > 0, shape, -1) - if shape is not None else -1 for i, shape in enumerate(output_shape) + tf.where(tensor_shape[i] - + shape > 0, shape, -1) if shape is not None else -1 + for i, shape in enumerate(output_shape) ] clipped_tensor = tf.slice( - tensor, begin=tf.zeros(len(clip_size), dtype=tf.int32), size=clip_size) + tensor, begin=tf.zeros(len(clip_size), dtype=tf.int32), size=clip_size + ) # Pad tensor if the shape of clipped tensor is smaller than the expected # shape. @@ -233,17 +236,20 @@ def check_min_image_dim(min_dim, image_tensor): image_width = static_shape.get_width(image_shape) if image_height is None or image_width is None: shape_assert = tf.Assert( - tf.logical_and( - tf.greater_equal(tf.shape(image_tensor)[1], min_dim), - tf.greater_equal(tf.shape(image_tensor)[2], min_dim)), - ['image size must be >= {} in both height and width.'.format(min_dim)]) + tf.logical_and( + tf.greater_equal(tf.shape(image_tensor)[1], min_dim), + tf.greater_equal(tf.shape(image_tensor)[2], min_dim) + ), + ['image size must be >= {} in both height and width.'.format(min_dim)] + ) with tf.control_dependencies([shape_assert]): return tf.identity(image_tensor) if image_height < min_dim or image_width < min_dim: raise ValueError( - 'image size must be >= %d in both height and width; image dim = %d,%d' % - (min_dim, image_height, image_width)) + 'image size must be >= %d in both height and width; image dim = %d,%d' % + (min_dim, image_height, image_width) + ) return image_tensor @@ -268,8 +274,10 @@ def assert_shape_equal(shape_a, shape_b): Raises: ValueError: When shapes are both static and unequal. """ - if (all(isinstance(dim, int) for dim in shape_a) and - all(isinstance(dim, int) for dim in shape_b)): + if ( + all(isinstance(dim, int) for dim in shape_a) + and all(isinstance(dim, int) for dim in shape_b) + ): if shape_a != shape_b: raise ValueError('Unequal shapes {}, {}'.format(shape_a, shape_b)) else: @@ -300,8 +308,9 @@ def assert_shape_equal_along_first_dimension(shape_a, shape_b): """ if isinstance(shape_a[0], int) and isinstance(shape_b[0], int): if shape_a[0] != shape_b[0]: - raise ValueError('Unequal first dimension {}, {}'.format( - shape_a[0], shape_b[0])) + raise ValueError( + 'Unequal first dimension {}, {}'.format(shape_a[0], shape_b[0]) + ) else: return tf.no_op() else: @@ -325,9 +334,11 @@ def assert_box_normalized(boxes, maximum_normalized_coordinate=1.1): box_minimum = tf.reduce_min(boxes) box_maximum = tf.reduce_max(boxes) return tf.Assert( - tf.logical_and( - tf.less_equal(box_maximum, maximum_normalized_coordinate), - tf.greater_equal(box_minimum, 0)), [boxes]) + tf.logical_and( + tf.less_equal(box_maximum, maximum_normalized_coordinate), + tf.greater_equal(box_minimum, 0) + ), [boxes] + ) def get_shape_list(tensor, expected_rank=None, name=None): @@ -385,19 +396,22 @@ def assert_rank(tensor, expected_rank, name=None): if actual_rank not in expected_rank_dict: scope_name = tf.get_variable_scope().name raise ValueError( - 'For the tensor `%s` in scope `%s`, the actual rank ' - '`%d` (shape = %s) is not equal to the expected rank `%s`' % - (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank))) + 'For the tensor `%s` in scope `%s`, the actual rank ' + '`%d` (shape = %s) is not equal to the expected rank `%s`' % + (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank)) + ) def truncate_sequence(seq_emb, seq_len, limited_len): def truncate(seq_embed, seq_length): - seq_embed = tf.slice(seq_embed, [0, 0, 0], - [shape[0], limited_len, shape[2]]) + seq_embed = tf.slice( + seq_embed, [0, 0, 0], [shape[0], limited_len, shape[2]] + ) seq_length = tf.where( - tf.greater(seq_length, limited_len), - tf.ones_like(seq_length) * limited_len, seq_length) + tf.greater(seq_length, limited_len), + tf.ones_like(seq_length) * limited_len, seq_length + ) return seq_embed, seq_length def keep(seq_embed, seq_length): @@ -406,8 +420,10 @@ def keep(seq_embed, seq_length): shape = get_shape_list(seq_emb) max_seq_len = shape[1] - return tf.cond(max_seq_len > limited_len, lambda: truncate(seq_emb, seq_len), - lambda: keep(seq_emb, seq_len)) + return tf.cond( + max_seq_len > limited_len, lambda: truncate(seq_emb, seq_len), + lambda: keep(seq_emb, seq_len) + ) def pad_or_truncate_sequence(seq_emb, seq_len, fixed_len): @@ -420,13 +436,16 @@ def padding(): def truncate(): sliced = tf.slice(seq_emb, [0, 0, 0], [-1, fixed_len, -1]) - length = tf.where(seq_len < fixed_len, seq_len, - tf.ones_like(seq_len) * - fixed_len) if seq_len is not None else None + length = tf.where( + seq_len < fixed_len, seq_len, + tf.ones_like(seq_len) * fixed_len + ) if seq_len is not None else None return sliced, length def keep(): return seq_emb, seq_len - return tf.cond(padding_length > 0, padding, - lambda: tf.cond(padding_length < 0, truncate, keep)) + return tf.cond( + padding_length > 0, padding, + lambda: tf.cond(padding_length < 0, truncate, keep) + ) diff --git a/easy_rec/python/utils/test_utils.py b/easy_rec/python/utils/test_utils.py index b71249ac8..abe7c4284 100644 --- a/easy_rec/python/utils/test_utils.py +++ b/easy_rec/python/utils/test_utils.py @@ -5,6 +5,7 @@ isort:skip_file """ from future import standard_library + standard_library.install_aliases() import yaml import glob @@ -36,7 +37,8 @@ def get_hdfs_tmp_dir(test_dir): """Create a randomly of directory in HDFS.""" tmp_name = ''.join( - [random.choice(string.ascii_letters + string.digits) for i in range(8)]) + [random.choice(string.ascii_letters + string.digits) for i in range(8)] + ) assert isinstance(test_dir, str) test_rand_dir = os.path.join(test_dir, tmp_name) gfile.MkDir(test_rand_dir) @@ -48,8 +50,9 @@ def proc_wait(proc, timeout=1200): while proc.poll() is None and time.time() - t0 < timeout: time.sleep(1) if proc.poll() is None: - logging.warning('proc[pid=%d] timeout[%d], will kill the proc' % - (proc.pid, timeout)) + logging.warning( + 'proc[pid=%d] timeout[%d], will kill the proc' % (proc.pid, timeout) + ) proc.terminate() while proc.poll() is None: time.sleep(1) @@ -58,9 +61,9 @@ def proc_wait(proc, timeout=1200): def get_tmp_dir(): max_retry = 5 while max_retry > 0: - tmp_name = ''.join([ - random.choice(string.ascii_letters + string.digits) for i in range(12) - ]) + tmp_name = ''.join( + [random.choice(string.ascii_letters + string.digits) for i in range(12)] + ) if os.environ.get('TEST_DIR', '') != '': global TEST_DIR TEST_DIR = os.environ['TEST_DIR'] @@ -101,7 +104,8 @@ def run_cmd(cmd_str, log_file, env=None): logging.info('RUNCMD: %s > %s 2>&1 ' % (cmd_str, log_file)) with open(log_file, 'w') as lfile: proc = subprocess.Popen( - cmd_str, stdout=lfile, stderr=subprocess.STDOUT, shell=True, env=env) + cmd_str, stdout=lfile, stderr=subprocess.STDOUT, shell=True, env=env + ) if six.PY2: # for debug purpose proc.args = cmd_str @@ -160,12 +164,12 @@ def _replace_data_for_test(data_path): return data_path -def _load_config_for_test(pipeline_config_path, - test_dir, - total_steps=50, - num_epochs=0): +def _load_config_for_test( + pipeline_config_path, test_dir, total_steps=50, num_epochs=0 +): pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path) + pipeline_config_path + ) train_config = pipeline_config.train_config eval_config = pipeline_config.eval_config data_config = pipeline_config.data_config @@ -181,18 +185,21 @@ def _load_config_for_test(pipeline_config_path, def _load_config_for_distribute_eval(pipeline_config_path, test_dir): pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path) + pipeline_config_path + ) pipeline_config.model_dir = test_dir logging.info('test_model_dir %s' % pipeline_config.model_dir) return pipeline_config -def test_datahub_train_eval(pipeline_config_path, - odps_oss_config, - test_dir, - process_pipeline_func=None, - total_steps=50, - post_check_func=None): +def test_datahub_train_eval( + pipeline_config_path, + odps_oss_config, + test_dir, + process_pipeline_func=None, + total_steps=50, + post_check_func=None +): gpus = get_available_gpus() if len(gpus) > 0: set_gpu_id(gpus[0]) @@ -207,8 +214,9 @@ def test_datahub_train_eval(pipeline_config_path, if isinstance(pipeline_config_path, EasyRecConfig): pipeline_config = pipeline_config_path else: - pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, - total_steps) + pipeline_config = _load_config_for_test( + pipeline_config_path, test_dir, total_steps + ) pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -237,8 +245,9 @@ def test_datahub_train_eval(pipeline_config_path, proc_wait(proc, timeout=TEST_TIME_OUT) if proc.returncode != 0: logging.warning( - 'train %s failed[pid=%d][code=%d][args=%s]' % - (test_pipeline_config_path, proc.pid, proc.returncode, proc.args)) + 'train %s failed[pid=%d][code=%d][args=%s]' % + (test_pipeline_config_path, proc.pid, proc.returncode, proc.args) + ) return False if post_check_func: return post_check_func(pipeline_config) @@ -247,20 +256,23 @@ def test_datahub_train_eval(pipeline_config_path, def _Load_config_for_test_eval(pipeline_config_path): pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path) + pipeline_config_path + ) return pipeline_config -def test_single_train_eval(pipeline_config_path, - test_dir, - process_pipeline_func=None, - hyperparam_str='', - total_steps=50, - post_check_func=None, - check_mode=False, - fine_tune_checkpoint=None, - extra_cmd_args=None, - timeout=-1): +def test_single_train_eval( + pipeline_config_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='', + total_steps=50, + post_check_func=None, + check_mode=False, + fine_tune_checkpoint=None, + extra_cmd_args=None, + timeout=-1 +): gpus = get_available_gpus() if len(gpus) > 0: set_gpu_id(gpus[0]) @@ -275,8 +287,9 @@ def test_single_train_eval(pipeline_config_path, if isinstance(pipeline_config_path, EasyRecConfig): pipeline_config = pipeline_config_path else: - pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, - total_steps) + pipeline_config = _load_config_for_test( + pipeline_config_path, test_dir, total_steps + ) pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -330,7 +343,8 @@ def test_single_pre_check(pipeline_config_path, test_dir): config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') train_cmd = 'python -m easy_rec.python.tools.pre_check --pipeline_config_path %s ' % ( - test_pipeline_config_path) + test_pipeline_config_path + ) proc = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, 'master')) proc_wait(proc, timeout=TEST_TIME_OUT) @@ -348,7 +362,8 @@ def test_single_predict(test_dir, input_path, output_path, saved_model_dir): set_gpu_id(None) predict_cmd = 'python -m easy_rec.python.predict --input_path %s --output_path %s --saved_model_dir %s' % ( - input_path, output_path, saved_model_dir) + input_path, output_path, saved_model_dir + ) proc = run_cmd(predict_cmd, '%s/log_%s.txt' % (test_dir, 'master')) proc_wait(proc, timeout=TEST_TIME_OUT) @@ -372,33 +387,38 @@ def test_feature_selection(pipeline_config): return True -def yaml_replace(train_yaml_path, - pipline_config_path, - test_pipeline_config_path, - test_export_dir=None): +def yaml_replace( + train_yaml_path, + pipline_config_path, + test_pipeline_config_path, + test_export_dir=None +): with open(train_yaml_path, 'r', encoding='utf-8') as _file: sample = _file.read() x = yaml.load(sample) _command = x['app']['command'] if test_export_dir is not None: - _command = _command.replace(pipline_config_path, - test_pipeline_config_path).replace( - '{EXPOERT_DIR}', test_export_dir) + _command = _command.replace( + pipline_config_path, test_pipeline_config_path + ).replace('{EXPOERT_DIR}', test_export_dir) else: - _command = _command.replace(pipline_config_path, - test_pipeline_config_path) + _command = _command.replace( + pipline_config_path, test_pipeline_config_path + ) x['app']['command'] = _command with open(train_yaml_path, 'w', encoding='utf-8') as _file: yaml.dump(x, _file) -def test_hdfs_train_eval(pipeline_config_path, - train_yaml_path, - test_dir, - process_pipeline_func=None, - hyperparam_str='', - total_steps=2000): +def test_hdfs_train_eval( + pipeline_config_path, + train_yaml_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='', + total_steps=2000 +): gpus = get_available_gpus() if len(gpus) > 0: @@ -409,8 +429,9 @@ def test_hdfs_train_eval(pipeline_config_path, logging.info('train_yaml_path %s' % train_yaml_path) if 'TF_CONFIG' in os.environ: del os.environ['TF_CONFIG'] - pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, - total_steps) + pipeline_config = _load_config_for_test( + pipeline_config_path, test_dir, total_steps + ) logging.info('model_dir in pipeline_config has been modified') pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -420,7 +441,9 @@ def test_hdfs_train_eval(pipeline_config_path, pipeline_config = process_pipeline_func(pipeline_config) config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') - yaml_replace(train_yaml_path, pipeline_config_path, test_pipeline_config_path) + yaml_replace( + train_yaml_path, pipeline_config_path, test_pipeline_config_path + ) logging.info('test_pipeline_config_path is %s' % test_pipeline_config_path) train_cmd = 'el_submit -yaml %s' % train_yaml_path proc = subprocess.Popen(train_cmd.split(), stderr=subprocess.STDOUT) @@ -431,11 +454,13 @@ def test_hdfs_train_eval(pipeline_config_path, return proc.returncode == 0 -def test_hdfs_eval(pipeline_config_path, - eval_yaml_path, - test_dir, - process_pipeline_func=None, - hyperparam_str=''): +def test_hdfs_eval( + pipeline_config_path, + eval_yaml_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='' +): gpus = get_available_gpus() if len(gpus) > 0: @@ -463,11 +488,13 @@ def test_hdfs_eval(pipeline_config_path, return proc.returncode == 0 -def test_hdfs_export(pipeline_config_path, - export_yaml_path, - test_dir, - process_pipeline_func=None, - hyperparam_str=''): +def test_hdfs_export( + pipeline_config_path, + export_yaml_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='' +): gpus = get_available_gpus() if len(gpus) > 0: @@ -485,8 +512,10 @@ def test_hdfs_export(pipeline_config_path, config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') test_export_path = os.path.join(test_dir, 'export_dir') - yaml_replace(export_yaml_path, pipeline_config_path, - test_pipeline_config_path, test_export_path) + yaml_replace( + export_yaml_path, pipeline_config_path, test_pipeline_config_path, + test_export_path + ) logging.info('test_pipeline_config_path is %s' % test_pipeline_config_path) eval_cmd = 'el_submit -yaml %s' % export_yaml_path proc = subprocess.Popen(eval_cmd.split(), stderr=subprocess.STDOUT) @@ -530,12 +559,14 @@ def _get_ports(num_worker): return get_ports_base(num_worker) -def _ps_worker_train(pipeline_config_path, - test_dir, - num_worker, - num_evaluator=0, - fit_on_eval=False, - fit_on_eval_steps=None): +def _ps_worker_train( + pipeline_config_path, + test_dir, + num_worker, + num_evaluator=0, + fit_on_eval=False, + fit_on_eval_steps=None +): gpus = get_available_gpus() # not enough gpus, run on cpu only if len(gpus) < num_worker: @@ -543,9 +574,9 @@ def _ps_worker_train(pipeline_config_path, ports = _get_ports(num_worker + 1) chief_or_master = 'master' if num_evaluator == 0 else 'chief' cluster = { - chief_or_master: ['localhost:%d' % ports[0]], - 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], - 'ps': ['localhost:%d' % ports[-1]] + chief_or_master: ['localhost:%d' % ports[0]], + 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], + 'ps': ['localhost:%d' % ports[-1]] } tf_config = {'cluster': cluster} procs = {} @@ -558,7 +589,8 @@ def _ps_worker_train(pipeline_config_path, if fit_on_eval_steps is not None: train_cmd += ' --fit_on_eval_steps ' + str(int(fit_on_eval_steps)) procs[chief_or_master] = run_cmd( - train_cmd, '%s/log_%s.txt' % (test_dir, chief_or_master)) + train_cmd, '%s/log_%s.txt' % (test_dir, chief_or_master) + ) tf_config['task'] = {'type': 'ps', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') @@ -569,23 +601,27 @@ def _ps_worker_train(pipeline_config_path, os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id(gpus[idx + 1]) worker_name = 'worker_%d' % idx - procs[worker_name] = run_cmd(train_cmd, - '%s/log_%s.txt' % (test_dir, worker_name)) + procs[worker_name] = run_cmd( + train_cmd, '%s/log_%s.txt' % (test_dir, worker_name) + ) if num_evaluator > 0: tf_config['task'] = {'type': 'evaluator', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') - procs['evaluator'] = run_cmd(train_cmd, - '%s/log_%s.txt' % (test_dir, 'evaluator')) + procs['evaluator'] = run_cmd( + train_cmd, '%s/log_%s.txt' % (test_dir, 'evaluator') + ) return procs -def _ps_worker_distribute_eval(pipeline_config_path, - checkpoint_path, - test_dir, - num_worker, - num_evaluator=0): +def _ps_worker_distribute_eval( + pipeline_config_path, + checkpoint_path, + test_dir, + num_worker, + num_evaluator=0 +): gpus = get_available_gpus() # not enough gpus, run on cpu only if len(gpus) < num_worker: @@ -593,9 +629,9 @@ def _ps_worker_distribute_eval(pipeline_config_path, ports = _get_ports(num_worker + 1) chief_or_master = 'master' if num_evaluator == 0 else 'chief' cluster = { - chief_or_master: ['localhost:%d' % ports[0]], - 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], - 'ps': ['localhost:%d' % ports[-1]] + chief_or_master: ['localhost:%d' % ports[0]], + 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], + 'ps': ['localhost:%d' % ports[-1]] } tf_config = {'cluster': cluster} procs = {} @@ -605,14 +641,17 @@ def _ps_worker_distribute_eval(pipeline_config_path, set_gpu_id(gpus[0]) train_cmd = 'python -m easy_rec.python.eval --pipeline_config_path {} --checkpoint_path {} \ --distribute_eval True --eval_result_path distribute_eval_result.txt'.format( - pipeline_config_path, checkpoint_path) + pipeline_config_path, checkpoint_path + ) procs[chief_or_master] = run_cmd( - train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, chief_or_master)) + train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, chief_or_master) + ) tf_config['task'] = {'type': 'ps', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') - procs['ps'] = run_cmd(train_cmd, - '%s/distribute_eval_log_%s.txt' % (test_dir, 'ps')) + procs['ps'] = run_cmd( + train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, 'ps') + ) for idx in range(num_worker - 1): tf_config['task'] = {'type': 'worker', 'index': idx} @@ -620,13 +659,15 @@ def _ps_worker_distribute_eval(pipeline_config_path, set_gpu_id(gpus[idx + 1]) worker_name = 'worker_%d' % idx procs[worker_name] = run_cmd( - train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, worker_name)) + train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, worker_name) + ) if num_evaluator > 0: tf_config['task'] = {'type': 'evaluator', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') procs['evaluator'] = run_cmd( - train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, 'evaluator')) + train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, 'evaluator') + ) return procs @@ -638,9 +679,9 @@ def _multi_worker_mirror_train(pipeline_config_path, test_dir, num_worker): gpus = [None] * num_worker ports = _get_ports(num_worker) tf_config = { - 'cluster': { - 'worker': ['localhost:%d' % ports[i] for i in range(num_worker)] - } + 'cluster': { + 'worker': ['localhost:%d' % ports[i] for i in range(num_worker)] + } } procs = {} train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path %s' % pipeline_config_path @@ -649,8 +690,9 @@ def _multi_worker_mirror_train(pipeline_config_path, test_dir, num_worker): os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id(gpus[idx]) worker_name = 'worker_%d' % idx - procs[worker_name] = run_cmd(train_cmd, - '%s/log_%s.txt' % (test_dir, worker_name)) + procs[worker_name] = run_cmd( + train_cmd, '%s/log_%s.txt' % (test_dir, worker_name) + ) return procs @@ -665,31 +707,35 @@ def _multi_worker_hvd_train(pipeline_config_path, test_dir, num_worker): ports = _get_ports(num_worker) hosts = ','.join(['localhost:%d' % ports[i] for i in range(num_worker)]) train_cmd = 'horovodrun -np %d --hosts %s python -m easy_rec.python.train_eval --pipeline_config_path %s' % ( - num_worker, hosts, pipeline_config_path) + num_worker, hosts, pipeline_config_path + ) proc = run_cmd(train_cmd, '%s/log_hvd.txt' % test_dir) proc_wait(proc, timeout=1200) return proc.returncode == 0 -def test_distributed_train_eval(pipeline_config_path, - test_dir, - total_steps=50, - num_evaluator=0, - edit_config_json=None, - use_hvd=False, - fit_on_eval=False, - num_epoch=0): +def test_distributed_train_eval( + pipeline_config_path, + test_dir, + total_steps=50, + num_evaluator=0, + edit_config_json=None, + use_hvd=False, + fit_on_eval=False, + num_epoch=0 +): logging.info('testing pipeline config %s' % pipeline_config_path) - pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, - total_steps, num_epoch) + pipeline_config = _load_config_for_test( + pipeline_config_path, test_dir, total_steps, num_epoch + ) if edit_config_json is not None: config_util.edit_config(pipeline_config, edit_config_json) if use_hvd: pipeline_config.train_config.sync_replicas = False if pipeline_config.train_config.train_distribute not in [ - DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy ]: pipeline_config.train_config.train_distribute =\ DistributionStrategy.HorovodStrategy @@ -706,16 +752,18 @@ def test_distributed_train_eval(pipeline_config_path, if train_config.train_distribute == DistributionStrategy.NoStrategy: num_worker = 2 procs = _ps_worker_train( - test_pipeline_config_path, - test_dir, - num_worker, - num_evaluator, - fit_on_eval, - fit_on_eval_steps=int(total_steps // 2)) + test_pipeline_config_path, + test_dir, + num_worker, + num_evaluator, + fit_on_eval, + fit_on_eval_steps=int(total_steps // 2) + ) elif train_config.train_distribute == DistributionStrategy.MultiWorkerMirroredStrategy: num_worker = 2 - procs = _multi_worker_mirror_train(test_pipeline_config_path, test_dir, - num_worker) + procs = _multi_worker_mirror_train( + test_pipeline_config_path, test_dir, num_worker + ) else: raise NotImplementedError @@ -775,31 +823,35 @@ def test_distribute_eval_test(cur_eval_path, test_dir): single_data = read_data_from_json_path(single_work_eval_path) distribute_data = read_data_from_json_path(distribute_eval_path) single_ret = { - k: single_data[k] - for k in single_data.keys() - if 'loss' not in k and 'step' not in k + k: single_data[k] + for k in single_data.keys() if 'loss' not in k and 'step' not in k } distribute_ret = { - k: distribute_data[k] for k in distribute_data.keys() if 'loss' not in k + k: distribute_data[k] + for k in distribute_data.keys() if 'loss' not in k } difference_num = 0.00001 for k in single_ret.keys(): if (abs(single_ret[k] - distribute_ret[k]) > difference_num): logging.error( - 'distribute_eval difference[%.8f] large than threshold[%.8f]' % - (abs(single_ret[k] - distribute_ret[k]), difference_num)) + 'distribute_eval difference[%.8f] large than threshold[%.8f]' % + (abs(single_ret[k] - distribute_ret[k]), difference_num) + ) return False return True -def test_distributed_eval(pipeline_config_path, - checkpoint_path, - test_dir, - total_steps=50, - num_evaluator=0): +def test_distributed_eval( + pipeline_config_path, + checkpoint_path, + test_dir, + total_steps=50, + num_evaluator=0 +): logging.info('testing pipeline config %s' % pipeline_config_path) - pipeline_config = _load_config_for_distribute_eval(pipeline_config_path, - test_dir) + pipeline_config = _load_config_for_distribute_eval( + pipeline_config_path, test_dir + ) train_config = pipeline_config.train_config config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') @@ -810,9 +862,10 @@ def test_distributed_eval(pipeline_config_path, try: if train_config.train_distribute == DistributionStrategy.NoStrategy: num_worker = 2 - procs = _ps_worker_distribute_eval(test_pipeline_config_path, - checkpoint_path, test_dir, num_worker, - num_evaluator) + procs = _ps_worker_distribute_eval( + test_pipeline_config_path, checkpoint_path, test_dir, num_worker, + num_evaluator + ) else: raise NotImplementedError diff --git a/easy_rec/python/utils/tf_utils.py b/easy_rec/python/utils/tf_utils.py index 24f47a94a..8602cb0ce 100644 --- a/easy_rec/python/utils/tf_utils.py +++ b/easy_rec/python/utils/tf_utils.py @@ -3,7 +3,6 @@ """Common functions used for odps input.""" import json import os - import tensorflow as tf from easy_rec.python.protos.dataset_pb2 import DatasetConfig @@ -24,12 +23,12 @@ def get_ps_num_from_tf_config(): def get_tf_type(field_type): type_map = { - DatasetConfig.INT32: tf.int32, - DatasetConfig.INT64: tf.int64, - DatasetConfig.STRING: tf.string, - DatasetConfig.BOOL: tf.bool, - DatasetConfig.FLOAT: tf.float32, - DatasetConfig.DOUBLE: tf.double + DatasetConfig.INT32: tf.int32, + DatasetConfig.INT64: tf.int64, + DatasetConfig.STRING: tf.string, + DatasetConfig.BOOL: tf.bool, + DatasetConfig.FLOAT: tf.float32, + DatasetConfig.DOUBLE: tf.double } assert field_type in type_map, 'invalid type: %s' % field_type return type_map[field_type] @@ -37,12 +36,12 @@ def get_tf_type(field_type): def get_col_type(tf_type): type_map = { - tf.int32: 'BIGINT', - tf.int64: 'BIGINT', - tf.string: 'STRING', - tf.float32: 'FLOAT', - tf.double: 'DOUBLE', - tf.bool: 'BOOLEAN' + tf.int32: 'BIGINT', + tf.int64: 'BIGINT', + tf.string: 'STRING', + tf.float32: 'FLOAT', + tf.double: 'DOUBLE', + tf.bool: 'BOOLEAN' } assert tf_type in type_map, 'invalid type: %s' % tf_type return type_map[tf_type] diff --git a/examples/data/amazon_books_data/process_amazon.py b/examples/data/amazon_books_data/process_amazon.py index ba1fa7f4c..b59a88d57 100644 --- a/examples/data/amazon_books_data/process_amazon.py +++ b/examples/data/amazon_books_data/process_amazon.py @@ -1,25 +1,26 @@ -import random - import pandas as pd +import random print('Start reading data...') title = ['UserID', 'BookID', 'Time'] print('Reading train data...') train = pd.read_table( - 'AmazonBooksData/book_train.txt', - sep=',', - header=None, - names=title, - engine='python', - encoding='ISO-8859-1') + 'AmazonBooksData/book_train.txt', + sep=',', + header=None, + names=title, + engine='python', + encoding='ISO-8859-1' +) print('Reading test data...') test = pd.read_table( - 'AmazonBooksData/book_test.txt', - sep=',', - header=None, - names=title, - engine='python', - encoding='ISO-8859-1') + 'AmazonBooksData/book_test.txt', + sep=',', + header=None, + names=title, + engine='python', + encoding='ISO-8859-1' +) print('Start processing train data...') train_set = [] @@ -88,19 +89,23 @@ def gen_neg(): print('Start writing amazon_train_data...') train_set_df.to_csv( - r'amazon_train_data', index=False, sep='\t', mode='a', header=False) + r'amazon_train_data', index=False, sep='\t', mode='a', header=False +) print('Start writing amazon_test_data...') test_set_df.to_csv( - r'amazon_test_data', index=False, sep='\t', mode='a', header=False) + r'amazon_test_data', index=False, sep='\t', mode='a', header=False +) print('Negative Sampling') train_book = train[['BookID']].drop_duplicates() test_book = test[['BookID']].drop_duplicates() negative_book = pd.concat([train_book, test_book]).drop_duplicates() df_ones = pd.DataFrame( - 1, index=negative_book.index, columns=negative_book.columns) + 1, index=negative_book.index, columns=negative_book.columns +) negative_book_data = pd.concat([negative_book, df_ones, negative_book], axis=1) new_header = ['id:int64', 'weight:float', 'feature:string'] negative_book_data.to_csv( - r'negative_book_data', index=False, sep='\t', mode='a', header=new_header) + r'negative_book_data', index=False, sep='\t', mode='a', header=new_header +) print('Done.') diff --git a/examples/data/criteo/process_criteo_kaggle.py b/examples/data/criteo/process_criteo_kaggle.py index 5b9cb4f34..68cb5aecf 100644 --- a/examples/data/criteo/process_criteo_kaggle.py +++ b/examples/data/criteo/process_criteo_kaggle.py @@ -6,14 +6,17 @@ columns = target_columns + dense_features + category_features data_train = pd.read_csv( - 'criteo_kaggle_display/train.txt', sep='\t', names=columns) + 'criteo_kaggle_display/train.txt', sep='\t', names=columns +) samples_num = data_train.shape[0] print('samples_num:', samples_num, round(samples_num * 0.9)) train_num = int(round(samples_num * 0.9)) data_train[:train_num].to_csv( - r'criteo_train_data', index=False, sep='\t', mode='a', header=False) + r'criteo_train_data', index=False, sep='\t', mode='a', header=False +) data_train[train_num:].to_csv( - r'criteo_test_data', index=False, sep='\t', mode='a', header=False) + r'criteo_test_data', index=False, sep='\t', mode='a', header=False +) print('Done.') diff --git a/examples/data/movielens_1m/process_ml_1m.py b/examples/data/movielens_1m/process_ml_1m.py index 8fca6292a..2da8141ba 100644 --- a/examples/data/movielens_1m/process_ml_1m.py +++ b/examples/data/movielens_1m/process_ml_1m.py @@ -1,6 +1,5 @@ -import re - import pandas as pd +import re from sklearn.utils import shuffle @@ -11,12 +10,13 @@ def process_data(): print('----User Data----') users_title = ['UserID', 'Gender', 'Age', 'JobID', 'ZipCode'] users = pd.read_table( - 'ml-1m/users.dat', - sep='::', - header=None, - names=users_title, - engine='python', - encoding='ISO-8859-1') + 'ml-1m/users.dat', + sep='::', + header=None, + names=users_title, + engine='python', + encoding='ISO-8859-1' + ) users = users.filter(regex='UserID|Gender|Age|JobID|ZipCode') # process the gender and age of user gender_map = {'F': 0, 'M': 1} @@ -29,23 +29,24 @@ def process_data(): print('----Movie Data----') movies_title = ['MovieID', 'Title', 'Genres'] movies = pd.read_table( - 'ml-1m/movies.dat', - sep='::', - header=None, - names=movies_title, - engine='python', - encoding='ISO-8859-1') + 'ml-1m/movies.dat', + sep='::', + header=None, + names=movies_title, + engine='python', + encoding='ISO-8859-1' + ) # split the title and year in Feature:'Title' pattern = re.compile(r'^(.*)\((\d+)\)$') title_map = { - val: pattern.match(val).group(1) - for ii, val in enumerate(set(movies['Title'])) + val: pattern.match(val).group(1) + for ii, val in enumerate(set(movies['Title'])) } year_map = { - val: pattern.match(val).group(2) - for ii, val in enumerate(set(movies['Title'])) + val: pattern.match(val).group(2) + for ii, val in enumerate(set(movies['Title'])) } movies['Year'] = movies['Title'].map(year_map) movies['Title'] = movies['Title'].map(title_map) @@ -54,12 +55,13 @@ def process_data(): print('----Rating Data----') ratings_title = ['UserID', 'MovieID', 'ratings', 'timestamps'] ratings = pd.read_table( - 'ml-1m/ratings.dat', - sep='::', - header=None, - names=ratings_title, - engine='python', - encoding='ISO-8859-1') + 'ml-1m/ratings.dat', + sep='::', + header=None, + names=ratings_title, + engine='python', + encoding='ISO-8859-1' + ) ratings = ratings.filter(regex='UserID|MovieID|ratings') # ratings of 4 and 5 are viewed as positive samples [label:1] # ratings of 0, 1 and 2 are viewed as negative samples [label:0] @@ -87,7 +89,9 @@ def process_data(): # split train set and test set, and write to file print('Start writing to file.') data_new[:665110].to_csv( - r'movies_train_data', index=False, sep='\t', mode='a', header=False) + r'movies_train_data', index=False, sep='\t', mode='a', header=False +) data_new[665110:].to_csv( - r'movies_test_data', index=False, sep='\t', mode='a', header=False) + r'movies_test_data', index=False, sep='\t', mode='a', header=False +) print('Done.') diff --git a/examples/match_model/dssm.md b/examples/match_model/dssm.md index 0fa909e94..a5e7bdbf6 100644 --- a/examples/match_model/dssm.md +++ b/examples/match_model/dssm.md @@ -61,7 +61,7 @@ model_config:{ - dnn: deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - id: 指定user_id/item_id列 -- simi_func: 向量相似度函数，包括\[COSINE, INNER_PRODUCT, EUCLID\]，默认COSINE，建议使用INNER_PRODUCT +- simi_func: 向量相似度函数，包括[COSINE, INNER_PRODUCT, EUCLID]，默认COSINE，建议使用INNER_PRODUCT - embedding_regularization: 对embedding部分加regularization，防止overfit 支持的metric_set包括: diff --git a/examples/match_model/dssm_negative_sample.md b/examples/match_model/dssm_negative_sample.md index 531081634..402915c3e 100644 --- a/examples/match_model/dssm_negative_sample.md +++ b/examples/match_model/dssm_negative_sample.md @@ -92,7 +92,7 @@ model_config:{ - dnn: deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - id: 指定user_id/item_id列 -- simi_func: 向量相似度函数，包括\[COSINE, INNER_PRODUCT, EUCLID\]，默认COSINE，建议使用INNER_PRODUCT +- simi_func: 向量相似度函数，包括[COSINE, INNER_PRODUCT, EUCLID]，默认COSINE，建议使用INNER_PRODUCT - scale_simi: 是否自动缩放相似度便于loss计算，建议设置成false - loss_type: 目前只支持SOFTMAX_CROSS_ENTROPY - embedding_regularization: 对embedding部分加regularization，防止overfit diff --git a/examples/match_model/readme.md b/examples/match_model/readme.md index a1e4d6b2e..b3630741a 100644 --- a/examples/match_model/readme.md +++ b/examples/match_model/readme.md @@ -4,7 +4,7 @@ # Amazon Books 数据集 -在此数据集中, 提供了2个模型及其负采样版的demo示例 [DSSM](dssm.md) / [DSSM-Negative-Sample](dssm_negative_sample.md) / [MIND](mind.md) / [MIND-Negative-Sample](mind_negative_sample.md)。更多模型可参考[models](../../docs/source/models/)。 +在此数据集中, 提供了2个模型及其负采样版的demo示例 [DSSM](dssm.md) / [DSSM-Negative-Sample](dssm_negative_sample.md) / [MIND](mind.md) / [MIND-Negative-Sample](mind_negative_sample.md)。更多模型可参考[models](../../docs/source/models/)。 - DSSM diff --git a/examples/rank_model/DeepFM.md b/examples/rank_model/DeepFM.md index c5321d0e8..ca97862df 100644 --- a/examples/rank_model/DeepFM.md +++ b/examples/rank_model/DeepFM.md @@ -46,7 +46,7 @@ model_config:{ 需要两个feature_group: wide group和deep group, **group name不能变** -- deepfm: deepfm相关的参数 +- deepfm: deepfm相关的参数 - dnn: deep part的参数配置 diff --git a/examples/rank_model/wide_and_deep.md b/examples/rank_model/wide_and_deep.md index 2ba30d3df..9144891df 100644 --- a/examples/rank_model/wide_and_deep.md +++ b/examples/rank_model/wide_and_deep.md @@ -51,7 +51,7 @@ model_config:{ 需要两个feature_group: wide group和deep group, **group name不能变** -- wide_and_deep: wide_and_deep 相关的参数 +- wide_and_deep: wide_and_deep 相关的参数 - dnn: deep part的参数配置 diff --git a/examples/readme.md b/examples/readme.md index dc2122d2e..8817aa0b8 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -74,21 +74,21 @@ sudo docker exec -it bash 下面分别是三种常用数据集的下载和预处理： -- MovieLens-1M (详细见:[data/movielens_1m/](data/movielens_1m/)。 也可跳过预处理，直接通过链接下载处理后的数据集： [movies_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/movielens_1m/movies_train_data)、[movies_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/movielens_1m/movies_test_data)) +- MovieLens-1M (详细见:[data/movielens_1m/](data/movielens_1m/)。 也可跳过预处理，直接通过链接下载处理后的数据集： [movies_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/movielens_1m/movies_train_data)、[movies_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/movielens_1m/movies_test_data)) ```bash cd examples/data/movielens_1m sh download_and_process.sh ``` -- Criteo-Research-Kaggle (详细见:[data/criteo/](data/criteo/)。也可跳过预处理，直接通过链接下载处理后的数据集： [criteo_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/criteo_kaggle/criteo_train_data)、[criteo_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/criteo_kaggle/criteo_test_data)) +- Criteo-Research-Kaggle (详细见:[data/criteo/](data/criteo/)。也可跳过预处理，直接通过链接下载处理后的数据集： [criteo_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/criteo_kaggle/criteo_train_data)、[criteo_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/criteo_kaggle/criteo_test_data)) ```bash cd examples/data/criteo sh download_and_process.sh ``` -- Amazon Books (详细见:[data/amazon_books_data/](data/amazon_books_data/)。也可跳过预处理，直接通过链接直接下载处理后的数据集： [amazon_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/amazon_train_data)、[amazon_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/amazon_test_data)、[negative_book_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/negative_book_data)) +- Amazon Books (详细见:[data/amazon_books_data/](data/amazon_books_data/)。也可跳过预处理，直接通过链接直接下载处理后的数据集： [amazon_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/amazon_train_data)、[amazon_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/amazon_test_data)、[negative_book_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/negative_book_data)) ```bash cd examples/data/amazon_books_data diff --git a/git-lfs/git_lfs.py b/git-lfs/git_lfs.py index 4eefc73e8..3be516028 100644 --- a/git-lfs/git_lfs.py +++ b/git-lfs/git_lfs.py @@ -12,14 +12,15 @@ blank_split = re.compile('[\t ]') logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s[%(lineno)d] : %(message)s', - level=logging.INFO) + format='[%(levelname)s] %(asctime)s %(filename)s[%(lineno)d] : %(message)s', + level=logging.INFO +) try: import oss2 except ImportError: logging.error( - 'please install python_oss from https://github.com/aliyun/aliyun-oss-python-sdk.git' + 'please install python_oss from https://github.com/aliyun/aliyun-oss-python-sdk.git' ) sys.exit(1) @@ -49,8 +50,8 @@ def load_git_url(): for line_str in fin: line_str = line_str.strip() line_json = json.loads(line_str) - git_bin_url_map[line_json['leaf_path']] = (line_json['sig'], - line_json['remote_path']) + git_bin_url_map[line_json['leaf_path'] + ] = (line_json['sig'], line_json['remote_path']) except Exception as ex: logging.warning('exception: %s' % str(ex)) pass @@ -64,7 +65,8 @@ def save_git_url(git_bin_url_map): for key in keys: val = git_bin_url_map[key] tmp_str = '{"leaf_path": "%s", "sig": "%s", "remote_path": "%s"}' % ( - key, val[0], val[1]) + key, val[0], val[1] + ) fout.write('%s\n' % tmp_str) @@ -104,8 +106,9 @@ def load_git_bin(): line_json = json.loads(line_str) file_arr[line_json['leaf_name']] = line_json['leaf_file'] except Exception as ex: - logging.warning('%s is corrupted : %s' % - (git_bin_path, traceback.format_exc(ex))) + logging.warning( + '%s is corrupted : %s' % (git_bin_path, traceback.format_exc(ex)) + ) return file_arr @@ -118,7 +121,8 @@ def save_git_bin(git_arr): leaf_files.sort() # make sure that leaf_name is in front of leaf_file tmp_str = '{"leaf_name": "%s", "leaf_file": %s}' % ( - leaf_path, json.dumps(leaf_files)) + leaf_path, json.dumps(leaf_files) + ) fout.write('%s\n' % tmp_str) @@ -221,7 +225,7 @@ def get_yes_no(msg): if __name__ == '__main__': if len(sys.argv) < 2: logging.error( - 'usage: python git_lfs.py [pull] [push] [add filename] [resolve_conflict]' + 'usage: python git_lfs.py [pull] [push] [add filename] [resolve_conflict]' ) sys.exit(1) home_directory = os.path.expanduser('~') @@ -239,8 +243,9 @@ def get_yes_no(msg): if line_str.startswith('#'): continue line_str = line_str.replace('~/', home_directory + '/') - line_str = line_str.replace('${TMPDIR}/', - os.environ.get('TMPDIR', '/tmp/')) + line_str = line_str.replace( + '${TMPDIR}/', os.environ.get('TMPDIR', '/tmp/') + ) line_str = line_str.replace('${PROJECT_NAME}', get_proj_name()) line_tok = [x.strip() for x in line_str.split('=') if x != ''] if line_tok[0] == 'host': @@ -252,15 +257,18 @@ def get_yes_no(msg): elif line_tok[0] == 'git_oss_private_config': git_oss_private_path = line_tok[1] if git_oss_private_path.startswith('~/'): - git_oss_private_path = os.path.join(home_directory, - git_oss_private_path[2:]) + git_oss_private_path = os.path.join( + home_directory, git_oss_private_path[2:] + ) elif line_tok[0] == 'git_oss_cache_dir': git_oss_cache_dir = line_tok[1] elif line_tok[0] == 'accl_endpoint': accl_endpoint = line_tok[1] - logging.info('git_oss_data_dir=%s, host=%s, bucket_name=%s' % - (git_oss_data_dir, host, bucket_name)) + logging.info( + 'git_oss_data_dir=%s, host=%s, bucket_name=%s' % + (git_oss_data_dir, host, bucket_name) + ) logging.info('git_oss_cache_dir: %s' % git_oss_cache_dir) @@ -281,8 +289,10 @@ def get_yes_no(msg): oss_auth = oss2.Auth(accessid, accesskey) oss_bucket = oss2.Bucket(oss_auth, host, bucket_name) else: - logging.info('git_oss_private_path[%s] is not found, read-only mode' % - git_oss_private_path) + logging.info( + 'git_oss_private_path[%s] is not found, read-only mode' % + git_oss_private_path + ) # pull only mode oss_auth = None oss_bucket = None @@ -353,14 +363,15 @@ def get_yes_no(msg): local_sig = '' update = False - if len(sys.argv) > 2 and (sys.argv[2] == '-f' or - sys.argv[2] == '--force'): + if len(sys.argv + ) > 2 and (sys.argv[2] == '-f' or sys.argv[2] == '--force'): update = True else: if has_conflict(leaf_path, leaf_files): update = get_yes_no( - 'update %s using remote file[remote_sig=%s local_sig=%s]?[N/Y]' % - (leaf_path, remote_sig, local_sig)) + 'update %s using remote file[remote_sig=%s local_sig=%s]?[N/Y]' % + (leaf_path, remote_sig, local_sig) + ) else: update = True if not update: @@ -382,9 +393,13 @@ def get_yes_no(msg): if sys.platform.startswith('linux'): subprocess.check_output(['wget', url, '-O', tar_tmp_path]) elif sys.platform.startswith('darwin'): - subprocess.check_output(['curl', url, '--output', tar_tmp_path]) + subprocess.check_output( + ['curl', url, '--output', tar_tmp_path] + ) elif sys.platform.startswith('win'): - subprocess.check_output(['curl', url, '--output', tar_tmp_path]) + subprocess.check_output( + ['curl', url, '--output', tar_tmp_path] + ) else: in_cache = True logging.info('%s is in cache' % file_name_with_sig) @@ -396,8 +411,10 @@ def get_yes_no(msg): logging.warning('cache invalid, will download from remote') os.remove(tar_tmp_path) continue - logging.warning('download failed, local_sig(%s) != remote_sig(%s)' % - (local_sig, remote_sig)) + logging.warning( + 'download failed, local_sig(%s) != remote_sig(%s)' % + (local_sig, remote_sig) + ) except subprocess.CalledProcessError as ex: logging.error('exception: %s' % str(ex)) except oss2.exceptions.RequestError as ex: @@ -503,8 +520,10 @@ def get_yes_no(msg): else: git_objs[leaf_name] = leaf_file else: - logging.warning('invalid state: merge_start = %d, line_str = %s' % - (merge_start, line_str)) + logging.warning( + 'invalid state: merge_start = %d, line_str = %s' % + (merge_start, line_str) + ) save_git_bin(git_objs) git_bin_url_map = {} @@ -520,15 +539,18 @@ def get_yes_no(msg): elif merge_start in [0, 1, 2]: line_json = json.loads(line_str) if line_json['leaf_path'] in git_objs: - git_bin_url_map[line_json['leaf_path']] = (line_json['sig'], - line_json['remote_path']) + git_bin_url_map[line_json['leaf_path'] + ] = (line_json['sig'], line_json['remote_path']) else: - logging.warning('invalid state: merge_start = %d, line_str = %s' % - (merge_start, line_str)) + logging.warning( + 'invalid state: merge_start = %d, line_str = %s' % + (merge_start, line_str) + ) save_git_url(git_bin_url_map) logging.info('all conflicts fixed.') else: logging.warning('invalid cmd: %s' % sys.argv[1]) logging.warning( - 'choices are: %s' % - ','.join(['push', 'pull', 'add', 'remove', 'resolve_conflict'])) + 'choices are: %s' % + ','.join(['push', 'pull', 'add', 'remove', 'resolve_conflict']) + ) diff --git a/pai_jobs/run.py b/pai_jobs/run.py index e3abc891d..1f9b91419 100644 --- a/pai_jobs/run.py +++ b/pai_jobs/run.py @@ -5,9 +5,8 @@ import logging # use few threads to avoid oss error import os -import time - import tensorflow as tf +import time import yaml from tensorflow.python.platform import gfile @@ -15,18 +14,13 @@ from easy_rec.python.inference.odps_predictor import ODPSPredictor from easy_rec.python.inference.vector_retrieve import VectorRetrieve from easy_rec.python.tools.pre_check import run_check -from easy_rec.python.utils import config_util -from easy_rec.python.utils import constant -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import fg_util -from easy_rec.python.utils import hpo_util -from easy_rec.python.utils import pai_util -from easy_rec.python.utils.distribution_utils import DistributionStrategyMap -from easy_rec.python.utils.distribution_utils import set_distribution_config +from easy_rec.python.utils import config_util, constant, estimator_utils, fg_util, hpo_util, pai_util # NOQA +from easy_rec.python.utils.distribution_utils import DistributionStrategyMap, set_distribution_config # NOQA os.environ['IS_ON_PAI'] = '1' from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_train_worker_num # NOQA + os.environ['OENV_MultiWriteThreadsNum'] = '4' os.environ['OENV_MultiCopyThreadsNum'] = '4' @@ -41,149 +35,199 @@ from easy_rec.python.main import _train_and_evaluate_impl as train_and_evaluate_impl # NOQA logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) -tf.app.flags.DEFINE_string('worker_hosts', '', - 'Comma-separated list of hostname:port pairs') -tf.app.flags.DEFINE_string('ps_hosts', '', - 'Comma-separated list of hostname:port pairs') +tf.app.flags.DEFINE_string( + 'worker_hosts', '', 'Comma-separated list of hostname:port pairs' +) +tf.app.flags.DEFINE_string( + 'ps_hosts', '', 'Comma-separated list of hostname:port pairs' +) tf.app.flags.DEFINE_string('job_name', '', 'task type, ps/worker') tf.app.flags.DEFINE_integer('task_index', 0, 'Index of task within the job') tf.app.flags.DEFINE_string('config', '', 'EasyRec config file path') -tf.app.flags.DEFINE_string('cmd', 'train', - 'command type, train/evaluate/export') +tf.app.flags.DEFINE_string( + 'cmd', 'train', 'command type, train/evaluate/export' +) tf.app.flags.DEFINE_string('tables', '', 'tables passed by pai command') # flags for train -tf.app.flags.DEFINE_integer('num_gpus_per_worker', 1, - 'number of gpu to use in training') -tf.app.flags.DEFINE_boolean('with_evaluator', False, - 'whether a evaluator is necessary') +tf.app.flags.DEFINE_integer( + 'num_gpus_per_worker', 1, 'number of gpu to use in training' +) +tf.app.flags.DEFINE_boolean( + 'with_evaluator', False, 'whether a evaluator is necessary' +) tf.app.flags.DEFINE_string( - 'eval_method', 'none', 'default to none, choices are [none: not evaluate,' + - 'master: evaluate on master, separate: evaluate on a separate task]') + 'eval_method', 'none', 'default to none, choices are [none: not evaluate,' + + 'master: evaluate on master, separate: evaluate on a separate task]' +) -tf.app.flags.DEFINE_string('distribute_strategy', '', - 'training distribute strategy') +tf.app.flags.DEFINE_string( + 'distribute_strategy', '', 'training distribute strategy' +) tf.app.flags.DEFINE_string('edit_config_json', '', 'edit config json string') tf.app.flags.DEFINE_string('train_tables', '', 'tables used for train') tf.app.flags.DEFINE_string('eval_tables', '', 'tables used for evaluation') tf.app.flags.DEFINE_string('boundary_table', '', 'tables used for boundary') tf.app.flags.DEFINE_string('sampler_table', '', 'tables used for sampler') -tf.app.flags.DEFINE_string('fine_tune_checkpoint', None, - 'finetune checkpoint path') -tf.app.flags.DEFINE_string('query_table', '', - 'table used for retrieve vector neighbours') -tf.app.flags.DEFINE_string('doc_table', '', - 'table used for be retrieved as indexed vectors') -tf.app.flags.DEFINE_enum('knn_distance', 'inner_product', - ['l2', 'inner_product'], 'type of knn distance') -tf.app.flags.DEFINE_integer('knn_num_neighbours', None, - 'top n neighbours to be retrieved') -tf.app.flags.DEFINE_integer('knn_feature_dims', None, - 'number of feature dimensions') +tf.app.flags.DEFINE_string( + 'fine_tune_checkpoint', None, 'finetune checkpoint path' +) +tf.app.flags.DEFINE_string( + 'query_table', '', 'table used for retrieve vector neighbours' +) +tf.app.flags.DEFINE_string( + 'doc_table', '', 'table used for be retrieved as indexed vectors' +) +tf.app.flags.DEFINE_enum( + 'knn_distance', 'inner_product', ['l2', 'inner_product'], + 'type of knn distance' +) +tf.app.flags.DEFINE_integer( + 'knn_num_neighbours', None, 'top n neighbours to be retrieved' +) +tf.app.flags.DEFINE_integer( + 'knn_feature_dims', None, 'number of feature dimensions' +) tf.app.flags.DEFINE_enum( - 'knn_index_type', 'ivfflat', - ['flat', 'ivfflat', 'ivfpq', 'gpu_flat', 'gpu_ivfflat', 'gpu_ivfpg'], - 'knn index type') -tf.app.flags.DEFINE_string('knn_feature_delimiter', ',', - 'delimiter for feature vectors') -tf.app.flags.DEFINE_integer('knn_nlist', 5, - 'number of split part on each worker') -tf.app.flags.DEFINE_integer('knn_nprobe', 2, - 'number of probe part on each worker') + 'knn_index_type', 'ivfflat', + ['flat', 'ivfflat', 'ivfpq', 'gpu_flat', 'gpu_ivfflat', 'gpu_ivfpg'], + 'knn index type' +) +tf.app.flags.DEFINE_string( + 'knn_feature_delimiter', ',', 'delimiter for feature vectors' +) +tf.app.flags.DEFINE_integer( + 'knn_nlist', 5, 'number of split part on each worker' +) tf.app.flags.DEFINE_integer( - 'knn_compress_dim', 8, - 'number of dimensions after compress for `ivfpq` and `gpu_ivfpq`') + 'knn_nprobe', 2, 'number of probe part on each worker' +) +tf.app.flags.DEFINE_integer( + 'knn_compress_dim', 8, + 'number of dimensions after compress for `ivfpq` and `gpu_ivfpq`' +) # flags used for evaluate & export tf.app.flags.DEFINE_string( - 'checkpoint_path', '', 'checkpoint to be evaluated or exported ' - 'if not specified, use the latest checkpoint ' - 'in train_config.model_dir') + 'checkpoint_path', '', 'checkpoint to be evaluated or exported ' + 'if not specified, use the latest checkpoint ' + 'in train_config.model_dir' +) # flags used for evaluate -tf.app.flags.DEFINE_string('eval_result_path', 'eval_result.txt', - 'eval result metric file') -tf.app.flags.DEFINE_bool('distribute_eval', False, - 'use distribute parameter server for train and eval.') +tf.app.flags.DEFINE_string( + 'eval_result_path', 'eval_result.txt', 'eval result metric file' +) +tf.app.flags.DEFINE_bool( + 'distribute_eval', False, + 'use distribute parameter server for train and eval.' +) # flags used for export -tf.app.flags.DEFINE_string('export_dir', '', - 'directory where model should be exported to') +tf.app.flags.DEFINE_string( + 'export_dir', '', 'directory where model should be exported to' +) tf.app.flags.DEFINE_bool('clear_export', False, 'remove export_dir if exists') -tf.app.flags.DEFINE_string('export_done_file', '', - 'a flag file to signal that export model is done') -tf.app.flags.DEFINE_integer('max_wait_ckpt_ts', 0, - 'max wait time in seconds for checkpoints') -tf.app.flags.DEFINE_boolean('continue_train', True, - 'use the same model to continue train or not') +tf.app.flags.DEFINE_string( + 'export_done_file', '', 'a flag file to signal that export model is done' +) +tf.app.flags.DEFINE_integer( + 'max_wait_ckpt_ts', 0, 'max wait time in seconds for checkpoints' +) +tf.app.flags.DEFINE_boolean( + 'continue_train', True, 'use the same model to continue train or not' +) # flags used for predict -tf.app.flags.DEFINE_string('saved_model_dir', '', - 'directory where saved_model.pb exists') +tf.app.flags.DEFINE_string( + 'saved_model_dir', '', 'directory where saved_model.pb exists' +) tf.app.flags.DEFINE_string('outputs', '', 'output tables') tf.app.flags.DEFINE_string( - 'all_cols', '', - 'union of (selected_cols, reserved_cols), separated with , ') + 'all_cols', '', 'union of (selected_cols, reserved_cols), separated with , ' +) tf.app.flags.DEFINE_string( - 'all_col_types', '', - 'column data types, for build record defaults, separated with ,') + 'all_col_types', '', + 'column data types, for build record defaults, separated with ,' +) tf.app.flags.DEFINE_string( - 'selected_cols', '', - 'columns to keep from input table, they are separated with ,') + 'selected_cols', '', + 'columns to keep from input table, they are separated with ,' +) tf.app.flags.DEFINE_string( - 'reserved_cols', '', - 'columns to keep from input table, they are separated with ,') + 'reserved_cols', '', + 'columns to keep from input table, they are separated with ,' +) tf.app.flags.DEFINE_string( - 'output_cols', None, - 'output columns, such as: score float. multiple columns are separated by ,') + 'output_cols', None, + 'output columns, such as: score float. multiple columns are separated by ,' +) tf.app.flags.DEFINE_integer('batch_size', 1024, 'predict batch size') tf.app.flags.DEFINE_string( - 'profiling_file', None, - 'time stat file which can be viewed using chrome tracing') + 'profiling_file', None, + 'time stat file which can be viewed using chrome tracing' +) tf.app.flags.DEFINE_string('redis_url', None, 'export to redis url, host:port') tf.app.flags.DEFINE_string('redis_passwd', None, 'export to redis passwd') tf.app.flags.DEFINE_integer('redis_threads', 5, 'export to redis threads') -tf.app.flags.DEFINE_integer('redis_batch_size', 1024, - 'export to redis batch_size') -tf.app.flags.DEFINE_integer('redis_timeout', 600, - 'export to redis time_out in seconds') -tf.app.flags.DEFINE_integer('redis_expire', 24, - 'export to redis expire time in hour') -tf.app.flags.DEFINE_string('redis_embedding_version', '', - 'redis embedding version') +tf.app.flags.DEFINE_integer( + 'redis_batch_size', 1024, 'export to redis batch_size' +) +tf.app.flags.DEFINE_integer( + 'redis_timeout', 600, 'export to redis time_out in seconds' +) +tf.app.flags.DEFINE_integer( + 'redis_expire', 24, 'export to redis expire time in hour' +) +tf.app.flags.DEFINE_string( + 'redis_embedding_version', '', 'redis embedding version' +) tf.app.flags.DEFINE_integer('redis_write_kv', 1, 'whether write kv ') tf.app.flags.DEFINE_string( - 'oss_path', None, 'write embed objects to oss folder, oss://bucket/folder') + 'oss_path', None, 'write embed objects to oss folder, oss://bucket/folder' +) tf.app.flags.DEFINE_string('oss_endpoint', None, 'oss endpoint') tf.app.flags.DEFINE_string('oss_ak', None, 'oss ak') tf.app.flags.DEFINE_string('oss_sk', None, 'oss sk') -tf.app.flags.DEFINE_integer('oss_threads', 10, - '# threads access oss at the same time') -tf.app.flags.DEFINE_integer('oss_timeout', 10, - 'connect to oss, time_out in seconds') +tf.app.flags.DEFINE_integer( + 'oss_threads', 10, '# threads access oss at the same time' +) +tf.app.flags.DEFINE_integer( + 'oss_timeout', 10, 'connect to oss, time_out in seconds' +) tf.app.flags.DEFINE_integer('oss_expire', 24, 'oss expire time in hours') -tf.app.flags.DEFINE_integer('oss_write_kv', 1, - 'whether to write embedding to oss') -tf.app.flags.DEFINE_string('oss_embedding_version', '', 'oss embedding version') +tf.app.flags.DEFINE_integer( + 'oss_write_kv', 1, 'whether to write embedding to oss' +) +tf.app.flags.DEFINE_string( + 'oss_embedding_version', '', 'oss embedding version' +) tf.app.flags.DEFINE_bool('verbose', False, 'print more debug information') -tf.app.flags.DEFINE_bool('place_embedding_on_cpu', False, - 'whether to place embedding variables on cpu') +tf.app.flags.DEFINE_bool( + 'place_embedding_on_cpu', False, + 'whether to place embedding variables on cpu' +) # for automl hyper parameter tuning tf.app.flags.DEFINE_string('model_dir', None, 'model directory') -tf.app.flags.DEFINE_bool('clear_model', False, - 'remove model directory if exists') -tf.app.flags.DEFINE_string('hpo_param_path', None, - 'hyperparameter tuning param path') -tf.app.flags.DEFINE_string('hpo_metric_save_path', None, - 'hyperparameter save metric path') +tf.app.flags.DEFINE_bool( + 'clear_model', False, 'remove model directory if exists' +) +tf.app.flags.DEFINE_string( + 'hpo_param_path', None, 'hyperparameter tuning param path' +) +tf.app.flags.DEFINE_string( + 'hpo_metric_save_path', None, 'hyperparameter save metric path' +) tf.app.flags.DEFINE_string('asset_files', None, 'extra files to add to export') tf.app.flags.DEFINE_bool('check_mode', False, 'is use check mode') tf.app.flags.DEFINE_string('fg_json_path', None, '') -tf.app.flags.DEFINE_bool('enable_avx_str_split', False, - 'enable avx str split to speedup') +tf.app.flags.DEFINE_bool( + 'enable_avx_str_split', False, 'enable avx str split to speedup' +) FLAGS = tf.app.flags.FLAGS @@ -201,17 +245,22 @@ def set_selected_cols(pipeline_config, selected_cols, all_cols, all_col_types): all_cols_arr = all_cols.split(',') all_col_types_arr = all_col_types.split(',') all_col_types_map = { - x.strip(): y.strip() for x, y in zip(all_cols_arr, all_col_types_arr) + x.strip(): y.strip() + for x, y in zip(all_cols_arr, all_col_types_arr) } selected_cols_arr = [x.strip() for x in selected_cols.split(',')] selected_col_types = [all_col_types_map[x] for x in selected_cols_arr] selected_col_types = ','.join(selected_col_types) pipeline_config.data_config.selected_col_types = selected_col_types - print('[run.py] data_config.selected_cols = "%s"' % - pipeline_config.data_config.selected_cols) - print('[run.py] data_config.selected_col_types = "%s"' % - pipeline_config.data_config.selected_col_types) + print( + '[run.py] data_config.selected_cols = "%s"' % + pipeline_config.data_config.selected_cols + ) + print( + '[run.py] data_config.selected_col_types = "%s"' % + pipeline_config.data_config.selected_col_types + ) def _wait_ckpt(ckpt_path, max_wait_ts): @@ -224,8 +273,9 @@ def _wait_ckpt(ckpt_path, max_wait_ts): logging.info('wait for checkpoint in directory[%s]' % ckpt_path) time.sleep(30) else: - logging.info('find checkpoint[%s] in directory[%s]' % - (tmp_ckpt, ckpt_path)) + logging.info( + 'find checkpoint[%s] in directory[%s]' % (tmp_ckpt, ckpt_path) + ) break else: while time.time() - start_ts < max_wait_ts: @@ -241,8 +291,9 @@ def main(argv): pai_util.set_on_pai() if FLAGS.enable_avx_str_split: constant.enable_avx_str_split() - logging.info('will enable avx str split: %s' % - constant.is_avx_str_split_enabled()) + logging.info( + 'will enable avx str split: %s' % constant.is_avx_str_split_enabled() + ) if FLAGS.distribute_eval: os.environ['distribute_eval'] = 'True' @@ -262,8 +313,9 @@ def main(argv): FLAGS.distribute_strategy, ','.join(DistributionStrategyMap.keys())) if FLAGS.config: - config = pai_util.process_config(FLAGS.config, FLAGS.task_index, - len(FLAGS.worker_hosts.split(','))) + config = pai_util.process_config( + FLAGS.config, FLAGS.task_index, len(FLAGS.worker_hosts.split(',')) + ) pipeline_config = config_util.get_configs_from_pipeline_file(config, False) # should be in front of edit_config_json step @@ -281,19 +333,21 @@ def main(argv): pipeline_config.model_dir = pipeline_config.model_dir.strip() print('[run.py] update model_dir to %s' % pipeline_config.model_dir) assert pipeline_config.model_dir.startswith( - 'oss://'), 'invalid model_dir format: %s' % pipeline_config.model_dir + 'oss://' + ), 'invalid model_dir format: %s' % pipeline_config.model_dir if FLAGS.asset_files: pipeline_config.export_config.asset_files.extend( - FLAGS.asset_files.split(',')) + FLAGS.asset_files.split(',') + ) if FLAGS.config: if not pipeline_config.model_dir.endswith('/'): pipeline_config.model_dir += '/' if FLAGS.clear_model: - if gfile.IsDirectory( - pipeline_config.model_dir) and estimator_utils.is_chief(): + if gfile.IsDirectory(pipeline_config.model_dir + ) and estimator_utils.is_chief(): gfile.DeleteRecursively(pipeline_config.model_dir) if FLAGS.max_wait_ckpt_ts > 0: @@ -308,9 +362,10 @@ def main(argv): if not FLAGS.train_tables and FLAGS.tables: tables = FLAGS.tables.split(',') assert len( - tables + tables ) >= 2, 'at least 2 tables must be specified, but only[%d]: %s' % ( - len(tables), FLAGS.tables) + len(tables), FLAGS.tables + ) if FLAGS.train_tables: pipeline_config.train_input_path = FLAGS.train_tables @@ -330,20 +385,24 @@ def main(argv): if pipeline_config.train_config.HasField('fine_tune_checkpoint'): pipeline_config.train_config.fine_tune_checkpoint = estimator_utils.get_latest_checkpoint_from_checkpoint_path( - pipeline_config.train_config.fine_tune_checkpoint, False) + pipeline_config.train_config.fine_tune_checkpoint, False + ) if FLAGS.boundary_table: logging.info('Load boundary_table: %s' % FLAGS.boundary_table) - config_util.add_boundaries_to_config(pipeline_config, - FLAGS.boundary_table) + config_util.add_boundaries_to_config( + pipeline_config, FLAGS.boundary_table + ) if FLAGS.sampler_table: pipeline_config.data_config.negative_sampler.input_path = FLAGS.sampler_table if FLAGS.train_tables or FLAGS.tables: # parse selected_cols - set_selected_cols(pipeline_config, FLAGS.selected_cols, FLAGS.all_cols, - FLAGS.all_col_types) + set_selected_cols( + pipeline_config, FLAGS.selected_cols, FLAGS.all_cols, + FLAGS.all_col_types + ) else: pipeline_config.data_config.selected_cols = '' pipeline_config.data_config.selected_col_types = '' @@ -362,7 +421,7 @@ def main(argv): print('[run.py] with_evaluator %s' % str(FLAGS.with_evaluator)) print('[run.py] eval_method %s' % FLAGS.eval_method) assert FLAGS.eval_method in [ - 'none', 'master', 'separate' + 'none', 'master', 'separate' ], 'invalid evalaute_method: %s' % FLAGS.eval_method # with_evaluator is depreciated, keeped for compatibility @@ -370,32 +429,37 @@ def main(argv): FLAGS.eval_method = 'separate' num_worker = set_tf_config_and_get_train_worker_num( - FLAGS.ps_hosts, - FLAGS.worker_hosts, - FLAGS.task_index, - FLAGS.job_name, - distribute_strategy=distribute_strategy, - eval_method=FLAGS.eval_method) - set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, - distribute_strategy) + FLAGS.ps_hosts, + FLAGS.worker_hosts, + FLAGS.task_index, + FLAGS.job_name, + distribute_strategy=distribute_strategy, + eval_method=FLAGS.eval_method + ) + set_distribution_config( + pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy + ) logging.info('run.py check_mode: %s .' % FLAGS.check_mode) train_and_evaluate_impl( - pipeline_config, - continue_train=FLAGS.continue_train, - check_mode=FLAGS.check_mode) + pipeline_config, + continue_train=FLAGS.continue_train, + check_mode=FLAGS.check_mode + ) if FLAGS.hpo_metric_save_path: hpo_util.save_eval_metrics( - pipeline_config.model_dir, - metric_save_path=FLAGS.hpo_metric_save_path, - has_evaluator=(FLAGS.eval_method == 'separate')) + pipeline_config.model_dir, + metric_save_path=FLAGS.hpo_metric_save_path, + has_evaluator=(FLAGS.eval_method == 'separate') + ) elif FLAGS.cmd == 'evaluate': check_param('config') # TODO: support multi-worker evaluation if not FLAGS.distribute_eval: assert len( - FLAGS.worker_hosts.split(',')) == 1, 'evaluate only need 1 worker' + FLAGS.worker_hosts.split(',') + ) == 1, 'evaluate only need 1 worker' config_util.auto_expand_share_feature_configs(pipeline_config) if FLAGS.eval_tables: @@ -405,18 +469,22 @@ def main(argv): distribute_strategy = DistributionStrategyMap[FLAGS.distribute_strategy] set_tf_config_and_get_train_worker_num( - FLAGS.ps_hosts, - FLAGS.worker_hosts, - FLAGS.task_index, - FLAGS.job_name, - eval_method='none') - set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, - distribute_strategy) + FLAGS.ps_hosts, + FLAGS.worker_hosts, + FLAGS.task_index, + FLAGS.job_name, + eval_method='none' + ) + set_distribution_config( + pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy + ) if FLAGS.eval_tables or FLAGS.tables: # parse selected_cols - set_selected_cols(pipeline_config, FLAGS.selected_cols, FLAGS.all_cols, - FLAGS.all_col_types) + set_selected_cols( + pipeline_config, FLAGS.selected_cols, FLAGS.all_cols, + FLAGS.all_col_types + ) else: pipeline_config.data_config.selected_cols = '' pipeline_config.data_config.selected_col_types = '' @@ -426,15 +494,17 @@ def main(argv): logging.info('will_use_distribute_eval') distribute_eval = os.environ.get('distribute_eval') logging.info('distribute_eval = {}'.format(distribute_eval)) - easy_rec.distribute_evaluate(pipeline_config, FLAGS.checkpoint_path, None, - FLAGS.eval_result_path) + easy_rec.distribute_evaluate( + pipeline_config, FLAGS.checkpoint_path, None, FLAGS.eval_result_path + ) else: os.environ['distribute_eval'] = 'False' logging.info('will_use_eval') distribute_eval = os.environ.get('distribute_eval') logging.info('distribute_eval = {}'.format(distribute_eval)) - easy_rec.evaluate(pipeline_config, FLAGS.checkpoint_path, None, - FLAGS.eval_result_path) + easy_rec.evaluate( + pipeline_config, FLAGS.checkpoint_path, None, FLAGS.eval_result_path + ) elif FLAGS.cmd == 'export': check_param('export_dir') check_param('config') @@ -479,11 +549,12 @@ def main(argv): oss_params['oss_write_kv'] = True if FLAGS.oss_write_kv == 1 else False set_tf_config_and_get_train_worker_num( - FLAGS.ps_hosts, - FLAGS.worker_hosts, - FLAGS.task_index, - FLAGS.job_name, - eval_method='none') + FLAGS.ps_hosts, + FLAGS.worker_hosts, + FLAGS.task_index, + FLAGS.job_name, + eval_method='none' + ) assert len(FLAGS.worker_hosts.split(',')) == 1, 'export only need 1 worker' config_util.auto_expand_share_feature_configs(pipeline_config) @@ -497,9 +568,10 @@ def main(argv): extra_params = redis_params extra_params.update(oss_params) - export_out_dir = easy_rec.export(export_dir, pipeline_config, - FLAGS.checkpoint_path, FLAGS.asset_files, - FLAGS.verbose, **extra_params) + export_out_dir = easy_rec.export( + export_dir, pipeline_config, FLAGS.checkpoint_path, FLAGS.asset_files, + FLAGS.verbose, **extra_params + ) if FLAGS.export_done_file: flag_file = os.path.join(export_out_dir, FLAGS.export_done_file) logging.info('create export done file: %s' % flag_file) @@ -508,49 +580,56 @@ def main(argv): elif FLAGS.cmd == 'predict': check_param('tables') check_param('saved_model_dir') - logging.info('will use the following columns as model input: %s' % - FLAGS.selected_cols) - logging.info('will copy the following columns to output: %s' % - FLAGS.reserved_cols) + logging.info( + 'will use the following columns as model input: %s' % FLAGS.selected_cols + ) + logging.info( + 'will copy the following columns to output: %s' % FLAGS.reserved_cols + ) profiling_file = FLAGS.profiling_file if FLAGS.task_index == 0 else None if profiling_file is not None: print('profiling_file = %s ' % profiling_file) predictor = ODPSPredictor( - FLAGS.saved_model_dir, - fg_json_path=FLAGS.fg_json_path, - profiling_file=profiling_file, - all_cols=FLAGS.all_cols, - all_col_types=FLAGS.all_col_types) + FLAGS.saved_model_dir, + fg_json_path=FLAGS.fg_json_path, + profiling_file=profiling_file, + all_cols=FLAGS.all_cols, + all_col_types=FLAGS.all_col_types + ) input_table, output_table = FLAGS.tables, FLAGS.outputs - logging.info('input_table = %s, output_table = %s' % - (input_table, output_table)) + logging.info( + 'input_table = %s, output_table = %s' % (input_table, output_table) + ) worker_num = len(FLAGS.worker_hosts.split(',')) predictor.predict_impl( - input_table, - output_table, - reserved_cols=FLAGS.reserved_cols, - output_cols=FLAGS.output_cols, - batch_size=FLAGS.batch_size, - slice_id=FLAGS.task_index, - slice_num=worker_num) + input_table, + output_table, + reserved_cols=FLAGS.reserved_cols, + output_cols=FLAGS.output_cols, + batch_size=FLAGS.batch_size, + slice_id=FLAGS.task_index, + slice_num=worker_num + ) elif FLAGS.cmd == 'export_checkpoint': check_param('export_dir') check_param('config') set_tf_config_and_get_train_worker_num( - FLAGS.ps_hosts, - FLAGS.worker_hosts, - FLAGS.task_index, - FLAGS.job_name, - eval_method='none') + FLAGS.ps_hosts, + FLAGS.worker_hosts, + FLAGS.task_index, + FLAGS.job_name, + eval_method='none' + ) assert len(FLAGS.worker_hosts.split(',')) == 1, 'export only need 1 woker' config_util.auto_expand_share_feature_configs(pipeline_config) easy_rec.export_checkpoint( - pipeline_config, - export_path=FLAGS.export_dir + '/model', - checkpoint_path=FLAGS.checkpoint_path, - asset_files=FLAGS.asset_files, - verbose=FLAGS.verbose) + pipeline_config, + export_path=FLAGS.export_dir + '/model', + checkpoint_path=FLAGS.checkpoint_path, + asset_files=FLAGS.asset_files, + verbose=FLAGS.verbose + ) elif FLAGS.cmd == 'vector_retrieve': check_param('knn_distance') assert FLAGS.knn_feature_dims is not None, '`knn_feature_dims` should not be None' @@ -560,31 +639,33 @@ def main(argv): if not query_table: tables = FLAGS.tables.split(',') assert len( - tables + tables ) >= 1, 'at least 1 tables must be specified, but only[%d]: %s' % ( - len(tables), FLAGS.tables) + len(tables), FLAGS.tables + ) query_table = tables[0] doc_table = tables[1] if len(tables) > 1 else query_table knn = VectorRetrieve( - query_table, - doc_table, - output_table, - ndim=FLAGS.knn_feature_dims, - distance=1 if FLAGS.knn_distance == 'inner_product' else 0, - delimiter=FLAGS.knn_feature_delimiter, - batch_size=FLAGS.batch_size, - index_type=FLAGS.knn_index_type, - nlist=FLAGS.knn_nlist, - nprobe=FLAGS.knn_nprobe, - m=FLAGS.knn_compress_dim) + query_table, + doc_table, + output_table, + ndim=FLAGS.knn_feature_dims, + distance=1 if FLAGS.knn_distance == 'inner_product' else 0, + delimiter=FLAGS.knn_feature_delimiter, + batch_size=FLAGS.batch_size, + index_type=FLAGS.knn_index_type, + nlist=FLAGS.knn_nlist, + nprobe=FLAGS.knn_nprobe, + m=FLAGS.knn_compress_dim + ) worker_hosts = FLAGS.worker_hosts.split(',') knn(FLAGS.knn_num_neighbours, FLAGS.task_index, len(worker_hosts)) elif FLAGS.cmd == 'check': run_check(pipeline_config, FLAGS.tables) else: raise ValueError( - 'cmd should be one of train/evaluate/export/predict/export_checkpoint/vector_retrieve' + 'cmd should be one of train/evaluate/export/predict/export_checkpoint/vector_retrieve' ) diff --git a/scripts/ci_test_change_files.py b/scripts/ci_test_change_files.py index 0502f90c1..509f5913f 100644 --- a/scripts/ci_test_change_files.py +++ b/scripts/ci_test_change_files.py @@ -12,14 +12,17 @@ sys.exit(2) logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' +) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--pull_request_num', type=int, default=None, help='pull request number') + '--pull_request_num', type=int, default=None, help='pull request number' + ) parser.add_argument( - '--exclude_dirs', nargs='*', type=str, help='the directory to be ignored') + '--exclude_dirs', nargs='*', type=str, help='the directory to be ignored' + ) args = parser.parse_args() diff --git a/scripts/pre-commit b/scripts/pre-commit index fbd34dfde..88a3e7804 100755 --- a/scripts/pre-commit +++ b/scripts/pre-commit @@ -19,7 +19,7 @@ os.environ.pop('__PYVENV_LAUNCHER__', None) # start templated INSTALL_PYTHON = '/apsarapangu/disk2/yancheng.lgq/miniconda3/envs/tf_1_15/bin/python' ARGS = [ - 'hook-impl', '--config=.pre-commit-config.yaml', '--hook-type=pre-commit' + 'hook-impl', '--config=.pre-commit-config.yaml', '--hook-type=pre-commit' ] # end templated ARGS.extend(('--hook-dir', os.path.realpath(os.path.dirname(__file__)))) diff --git a/setup.cfg b/setup.cfg index f0223c47a..aebb3437c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,7 +7,7 @@ universal = 1 [isort] line_length = 79 multi_line_output = 7 -force_single_line = true +# force_single_line = true 这与 multi_line_output 冲突 known_standard_library = setuptools known_first_party = easy_rec known_third_party = absl,common_io,distutils,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,scipy,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,tensorflow_probability,yaml @@ -16,14 +16,20 @@ default_section = THIRDPARTY skip = easy_rec/python/protos [yapf] -BASED_ON_STYLE = yapf -ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT = true +based_on_style = pep8 +column_limit = 79 +indent_width = 2 +CONTINUATION_INDENT_WIDTH = 2 +CONTINUATION_ALIGN_STYLE = SPACE +DEDENT_CLOSING_BRACKETS = true +ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT = false +SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN = false [flake8] select = B,C,D,E,F,P,T4,W,B9 max-line-length = 120 ignore = - E111,E114,E125,E129,W291,W503,W504, + E111,E114,E121,E124,E125,E126,E129,E251,W291,W503,W504, # docstring missing error should be used when all docstrings are completed D100,D101,D102,D103,D104,D105,D106,D107 per-file-ignores = diff --git a/setup.py b/setup.py index 26b94f707..d6aea8ca1 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. +from setuptools import find_packages, setup + import codecs import os -from setuptools import find_packages -from setuptools import setup def readme(): @@ -50,22 +50,23 @@ def parse_require_file(fpath): setup( - name='easy-rec', - version=get_version(), - description='An easy-to-use framework for Recommendation', - doc=readme(), - author='EasyRec Team', - author_email='easy_rec@alibaba-inc.com', - url='https://github.com/alibaba/EasyRec', - packages=find_packages(), - include_package_data=True, - classifiers=[ - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3' - ], - tests_require=parse_requirements('requirements/tests.txt'), - install_requires=parse_requirements('requirements/runtime.txt'), - extras_require={ - 'all': parse_requirements('requirements.txt'), - 'tests': parse_requirements('requirements/tests.txt'), - }) + name='easy-rec', + version=get_version(), + description='An easy-to-use framework for Recommendation', + doc=readme(), + author='EasyRec Team', + author_email='easy_rec@alibaba-inc.com', + url='https://github.com/alibaba/EasyRec', + packages=find_packages(), + include_package_data=True, + classifiers=[ + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3' + ], + tests_require=parse_requirements('requirements/tests.txt'), + install_requires=parse_requirements('requirements/runtime.txt'), + extras_require={ + 'all': parse_requirements('requirements.txt'), + 'tests': parse_requirements('requirements/tests.txt'), + } +) From a6fefaa4bb5ba6a94b990bf01c941e06c671a9a3 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Mon, 10 Nov 2025 15:29:14 +0800 Subject: [PATCH 43/51] upgrade zero inflated lognormal loss, support export structure path, upgrade pre-commit hooks --- .github/workflows/code_style.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 7a7a79d83..4ff9bfc03 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -15,6 +15,16 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} submodules: recursive + + - name: Update pre-commit + run: | + source ~/.bashrc + source activate tf25_py3 + pip install --upgrade pre-commit>=3.0.0 + pre-commit --version + pre-commit clean + pre-commit install-hooks + - name: RunCiTest id: run_ci_test env: From e5521f611727f8ecf14c75fdb77678ebfe5db9e7 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Mon, 10 Nov 2025 15:48:48 +0800 Subject: [PATCH 44/51] upgrade zero inflated lognormal loss, support export structure path, upgrade pre-commit hooks --- .github/workflows/code_style.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 4ff9bfc03..2a4617cc7 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -16,14 +16,11 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} submodules: recursive - - name: Update pre-commit + - name: Clean pre-commit run: | - source ~/.bashrc - source activate tf25_py3 - pip install --upgrade pre-commit>=3.0.0 pre-commit --version - pre-commit clean - pre-commit install-hooks + pre-commit clean || true + pre-commit gc || true - name: RunCiTest id: run_ci_test From f8fde64fc6d967f968fafe3d9eb279cb362c2cda Mon Sep 17 00:00:00 2001 From: yangxudong Date: Mon, 10 Nov 2025 20:13:42 +0800 Subject: [PATCH 45/51] upgrade zero inflated lognormal loss, support export structure path, upgrade pre-commit hooks --- .pre-commit-config.yaml | 4 ++-- .../python/compat/embedding_parallel_saver.py | 4 ++-- easy_rec/python/compat/optimizers.py | 17 +++++++++-------- easy_rec/python/main.py | 8 ++++---- easy_rec/python/model/deepfm.py | 5 +++++ easy_rec/python/model/easy_rec_estimator.py | 3 +-- easy_rec/python/utils/shape_utils.py | 5 ++--- 7 files changed, 25 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa89dc5c1..9ec014920 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,8 +10,8 @@ repos: rev: 5.12.0 hooks: - id: isort - - repo: https://github.com/google/yapf - rev: v0.43.0 + - repo: https://github.com/pre-commit/mirrors-yapf + rev: v0.32.0 hooks: - id: yapf - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/easy_rec/python/compat/embedding_parallel_saver.py b/easy_rec/python/compat/embedding_parallel_saver.py index 406a641c1..f3db9afa4 100644 --- a/easy_rec/python/compat/embedding_parallel_saver.py +++ b/easy_rec/python/compat/embedding_parallel_saver.py @@ -171,8 +171,8 @@ def _load_embed( embed_ids_o = part_id_o + embed_ids_o * len(embed_files) sel_ids = np.where( np.logical_and( - (embed_ids_o % part_num) == part_id, embed_ids_o - < embed_part_size * part_num + (embed_ids_o % part_num) == part_id, + embed_ids_o < embed_part_size * part_num ) )[0] part_update_cnt += len(sel_ids) diff --git a/easy_rec/python/compat/optimizers.py b/easy_rec/python/compat/optimizers.py index 35d0fd58f..3828f323e 100644 --- a/easy_rec/python/compat/optimizers.py +++ b/easy_rec/python/compat/optimizers.py @@ -497,20 +497,21 @@ def _get_grad_norm(grads_and_vars, embedding_parallel=False): sparse_norms.append(gen_nn_ops.l2_loss(grad.values)) else: dense_norms.append(gen_nn_ops.l2_loss(grad)) - reduced_norms = hvd.grouped_allreduce( - part_norms, op=hvd.Sum, compression=hvd.compression.NoneCompressor - ) - sparse_norms = sparse_norms + reduced_norms - all_norms = reduced_norms + dense_norms + if hvd is not None and part_norms: + reduced_norms = hvd.grouped_allreduce( + part_norms, op=hvd.Sum, compression=hvd.compression.NoneCompressor + ) + sparse_norms = sparse_norms + reduced_norms + all_norms = sparse_norms + dense_norms sparse_norm = math_ops.sqrt( math_ops.reduce_sum(array_ops.stack(sparse_norms) * 2.0) - ) + ) if sparse_norms else tf.constant(0.0) dense_norm = math_ops.sqrt( math_ops.reduce_sum(array_ops.stack(dense_norms) * 2.0) - ) + ) if dense_norms else tf.constant(0.0) grad_norm = math_ops.sqrt( math_ops.reduce_sum(array_ops.stack(all_norms)) * 2.0 - ) + ) if all_norms else tf.constant(0.0) return sparse_norm, dense_norm, grad_norm diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index 45119bb8e..442c01324 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -209,13 +209,13 @@ def _metric_cmp_fn(best_eval_result, current_eval_result): ) if export_config.metric_bigger: return ( - best_eval_result[export_config.best_exporter_metric] - < current_eval_result[export_config.best_exporter_metric] + best_eval_result[export_config.best_exporter_metric] < + current_eval_result[export_config.best_exporter_metric] ) else: return ( - best_eval_result[export_config.best_exporter_metric] - > current_eval_result[export_config.best_exporter_metric] + best_eval_result[export_config.best_exporter_metric] > + current_eval_result[export_config.best_exporter_metric] ) exporters = [ diff --git a/easy_rec/python/model/deepfm.py b/easy_rec/python/model/deepfm.py index c6d851b87..1d38ad0be 100644 --- a/easy_rec/python/model/deepfm.py +++ b/easy_rec/python/model/deepfm.py @@ -1,5 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. +import logging import tensorflow as tf from easy_rec.python.layers import dnn, fm @@ -47,6 +48,10 @@ def build_input_layer(self, model_config, feature_configs): if not has_final: assert model_config.deepfm.wide_output_dim == model_config.num_class self._wide_output_dim = model_config.deepfm.wide_output_dim + if self._wide_output_dim != 1: + logging.warning( + 'wide_output_dim not equal to 1, it is not a standard model' + ) super(DeepFM, self).build_input_layer(model_config, feature_configs) def build_predict_graph(self): diff --git a/easy_rec/python/model/easy_rec_estimator.py b/easy_rec/python/model/easy_rec_estimator.py index 587c9ec0f..16e6e6e0a 100644 --- a/easy_rec/python/model/easy_rec_estimator.py +++ b/easy_rec/python/model/easy_rec_estimator.py @@ -322,8 +322,7 @@ def _train_model_fn(self, features, labels, run_config): 'embedding_learning_rate_multiplier' ): gradient_multipliers = { - var: - self.train_config.optimizer_config[0]. + var: self.train_config.optimizer_config[0]. embedding_learning_rate_multiplier for var in tf.trainable_variables() if 'embedding_weights:' in var.name or '/embedding_weights/part_' in var.name diff --git a/easy_rec/python/utils/shape_utils.py b/easy_rec/python/utils/shape_utils.py index 5d6c17518..438fb4331 100644 --- a/easy_rec/python/utils/shape_utils.py +++ b/easy_rec/python/utils/shape_utils.py @@ -175,9 +175,8 @@ def pad_or_clip_nd(tensor, output_shape): """ tensor_shape = tf.shape(tensor) clip_size = [ - tf.where(tensor_shape[i] - - shape > 0, shape, -1) if shape is not None else -1 - for i, shape in enumerate(output_shape) + tf.where(tensor_shape[i] - shape > 0, shape, -1) + if shape is not None else -1 for i, shape in enumerate(output_shape) ] clipped_tensor = tf.slice( tensor, begin=tf.zeros(len(clip_size), dtype=tf.int32), size=clip_size From 76b9f108d8886bc54fb00d3660539c8c30640552 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Tue, 11 Nov 2025 09:58:14 +0800 Subject: [PATCH 46/51] upgrade zero inflated lognormal loss, support export structure path, upgrade pre-commit hooks --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ec014920..1ff72a56b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: hooks: - id: yapf - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 + rev: v4.4.0 hooks: - id: trailing-whitespace args: ["--no-markdown-linebreak-ext"] From 96a93aecbb60c1252b2355df470e868da45d9919 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Tue, 11 Nov 2025 10:08:36 +0800 Subject: [PATCH 47/51] upgrade zero inflated lognormal loss, support export structure path, upgrade pre-commit hooks --- .pre-commit-config.yaml | 8 ++++++-- docs/source/_ext/post_process.py | 1 + docs/source/conf.py | 3 ++- easy_rec/python/builders/loss_builder.py | 6 ++++-- easy_rec/python/builders/optimizer_builder.py | 1 + easy_rec/python/compat/adam_s.py | 3 ++- easy_rec/python/compat/dynamic_variable.py | 2 ++ easy_rec/python/compat/early_stopping.py | 6 ++++-- easy_rec/python/compat/embedding_parallel_saver.py | 10 ++++++---- easy_rec/python/compat/estimator_train.py | 6 ++++-- easy_rec/python/compat/exporter.py | 1 + .../python/compat/feature_column/feature_column.py | 8 +++++--- .../compat/feature_column/feature_column_v2.py | 11 +++++++---- .../feature_column/sequence_feature_column.py | 4 +++- easy_rec/python/compat/layers.py | 1 + easy_rec/python/compat/optimizers.py | 6 ++++-- easy_rec/python/compat/queues.py | 3 ++- easy_rec/python/compat/regularizers.py | 1 + easy_rec/python/compat/sok_optimizer.py | 5 +++-- easy_rec/python/compat/sync_replicas_optimizer.py | 5 +++-- easy_rec/python/compat/weight_decay_optimizers.py | 3 ++- easy_rec/python/core/easyrec_metrics/__init__.py | 1 + .../easyrec_metrics/distribute_metrics_impl_pai.py | 3 ++- .../easyrec_metrics/distribute_metrics_impl_tf.py | 3 ++- easy_rec/python/core/metrics.py | 11 +++++++---- easy_rec/python/core/sampler.py | 7 ++++--- easy_rec/python/eval.py | 2 ++ easy_rec/python/export.py | 1 + easy_rec/python/feature_column/feature_column.py | 4 +++- easy_rec/python/hpo/emr_hpo.py | 1 + easy_rec/python/hpo/pai_hpo.py | 1 + easy_rec/python/inference/csv_predictor.py | 4 +++- easy_rec/python/inference/hive_parquet_predictor.py | 5 +++-- easy_rec/python/inference/hive_predictor.py | 3 ++- easy_rec/python/inference/parquet_predictor.py | 3 ++- easy_rec/python/inference/parquet_predictor_v2.py | 3 ++- easy_rec/python/inference/predictor.py | 8 +++++--- easy_rec/python/inference/processor/test.py | 3 ++- easy_rec/python/inference/vector_retrieve.py | 5 +++-- easy_rec/python/input/batch_tfrecord_input.py | 1 + easy_rec/python/input/criteo_binary_reader.py | 3 ++- easy_rec/python/input/criteo_input.py | 1 + easy_rec/python/input/csv_input.py | 1 + easy_rec/python/input/datahub_input.py | 3 ++- easy_rec/python/input/hive_input.py | 1 + easy_rec/python/input/hive_parquet_input.py | 3 ++- easy_rec/python/input/hive_rtp_input.py | 1 + easy_rec/python/input/input.py | 10 ++++++---- easy_rec/python/input/kafka_dataset.py | 1 + easy_rec/python/input/kafka_input.py | 3 ++- easy_rec/python/input/load_parquet.py | 3 ++- easy_rec/python/input/odps_input_v2.py | 1 + easy_rec/python/input/odps_input_v3.py | 1 + easy_rec/python/input/odps_rtp_input.py | 1 + easy_rec/python/input/odps_rtp_input_v2.py | 1 + easy_rec/python/input/parquet_input.py | 3 ++- easy_rec/python/input/parquet_input_v2.py | 1 + easy_rec/python/input/parquet_input_v3.py | 1 + easy_rec/python/input/rtp_input.py | 4 +++- easy_rec/python/input/rtp_input_v2.py | 1 + easy_rec/python/input/tfrecord_input.py | 1 + easy_rec/python/layers/backbone.py | 1 + easy_rec/python/layers/capsule_layer.py | 1 + easy_rec/python/layers/dnn.py | 1 + easy_rec/python/layers/input_layer.py | 8 +++++--- easy_rec/python/layers/keras/__init__.py | 5 +++-- easy_rec/python/layers/keras/auxiliary_loss.py | 1 + easy_rec/python/layers/keras/blocks.py | 1 + easy_rec/python/layers/keras/custom_ops.py | 1 + easy_rec/python/layers/keras/din.py | 1 + easy_rec/python/layers/keras/einsum_dense.py | 4 +++- easy_rec/python/layers/keras/fibinet.py | 1 + easy_rec/python/layers/keras/mask_net.py | 1 + .../python/layers/keras/multi_head_attention.py | 3 ++- easy_rec/python/layers/keras/multi_task.py | 1 + easy_rec/python/layers/keras/numerical_embedding.py | 1 + easy_rec/python/layers/keras/ppnet.py | 1 + easy_rec/python/layers/keras/transformer.py | 1 + easy_rec/python/layers/mmoe.py | 1 + easy_rec/python/layers/multihead_cross_attention.py | 1 + easy_rec/python/layers/seq_input_layer.py | 1 + easy_rec/python/layers/sequence_feature_layer.py | 1 + easy_rec/python/layers/utils.py | 1 + easy_rec/python/layers/variational_dropout_layer.py | 1 + easy_rec/python/loss/focal_loss.py | 1 + easy_rec/python/loss/jrc_loss.py | 1 + easy_rec/python/loss/listwise_loss.py | 1 + easy_rec/python/loss/pairwise_loss.py | 1 + easy_rec/python/loss/zero_inflated_lognormal.py | 1 + easy_rec/python/main.py | 4 +++- easy_rec/python/model/autoint.py | 2 ++ easy_rec/python/model/cmbf.py | 1 + .../python/model/collaborative_metric_learning.py | 5 +++-- easy_rec/python/model/dcn.py | 1 + easy_rec/python/model/deepfm.py | 1 + easy_rec/python/model/dlrm.py | 2 ++ easy_rec/python/model/dropoutnet.py | 5 +++-- easy_rec/python/model/dssm_senet.py | 3 ++- easy_rec/python/model/easy_rec_estimator.py | 10 ++++++---- easy_rec/python/model/easy_rec_model.py | 3 ++- easy_rec/python/model/esmm.py | 1 + easy_rec/python/model/match_model.py | 1 + easy_rec/python/model/mind.py | 1 + easy_rec/python/model/multi_task_model.py | 3 ++- easy_rec/python/model/multi_tower.py | 1 + easy_rec/python/model/multi_tower_bst.py | 2 ++ easy_rec/python/model/multi_tower_din.py | 2 ++ easy_rec/python/model/multi_tower_recall.py | 1 + easy_rec/python/model/rank_model.py | 4 +++- easy_rec/python/model/rocket_launching.py | 3 ++- easy_rec/python/model/simple_multi_task.py | 1 + easy_rec/python/model/uniter.py | 1 + easy_rec/python/model/wide_and_deep.py | 2 ++ easy_rec/python/ops/gen_kafka_ops.py | 3 ++- easy_rec/python/ops/gen_str_avx_op.py | 1 + easy_rec/python/ops/incr_record.py | 1 + easy_rec/python/predict.py | 4 +++- easy_rec/python/test/csv_input_test.py | 3 ++- easy_rec/python/test/dh_local_run.py | 4 +++- easy_rec/python/test/embed_test.py | 1 + easy_rec/python/test/emr_run.py | 4 +++- easy_rec/python/test/eval_metric_test.py | 1 + easy_rec/python/test/excel_convert_test.py | 1 + easy_rec/python/test/export_test.py | 5 +++-- easy_rec/python/test/fg_test.py | 3 ++- easy_rec/python/test/hive_input_test.py | 3 ++- easy_rec/python/test/hpo_test.py | 5 +++-- easy_rec/python/test/kafka_test.py | 7 ++++--- easy_rec/python/test/local_incr_test.py | 5 +++-- easy_rec/python/test/odps_local_run.py | 4 +++- easy_rec/python/test/odps_run.py | 6 ++++-- easy_rec/python/test/odps_test_prepare.py | 3 ++- easy_rec/python/test/odps_test_util.py | 3 ++- easy_rec/python/test/pre_check_test.py | 1 + easy_rec/python/test/predictor_test.py | 3 ++- easy_rec/python/test/rtp_convert_test.py | 1 + easy_rec/python/test/run.py | 3 ++- easy_rec/python/test/train_eval_test.py | 8 +++++--- easy_rec/python/tools/add_boundaries_to_config.py | 3 ++- easy_rec/python/tools/add_feature_info_to_config.py | 1 + easy_rec/python/tools/convert_config_format.py | 1 + easy_rec/python/tools/convert_rtp_data.py | 1 + easy_rec/python/tools/convert_rtp_fg.py | 1 + easy_rec/python/tools/create_config_from_excel.py | 3 ++- easy_rec/python/tools/criteo/convert_data.py | 5 +++-- easy_rec/python/tools/edit_lookup_graph.py | 1 + easy_rec/python/tools/faiss_index_pai.py | 5 +++-- easy_rec/python/tools/feature_selection.py | 7 ++++--- easy_rec/python/tools/hit_rate_ds.py | 6 ++++-- easy_rec/python/tools/hit_rate_pai.py | 2 ++ easy_rec/python/tools/pre_check.py | 2 ++ easy_rec/python/tools/predict_and_chk.py | 3 ++- easy_rec/python/tools/read_kafka.py | 1 + easy_rec/python/tools/split_model_pai.py | 1 + easy_rec/python/tools/split_pdn_model_pai.py | 1 + easy_rec/python/tools/test_saved_model.py | 3 ++- easy_rec/python/tools/view_saved_model.py | 1 + easy_rec/python/tools/write_kafka.py | 1 + easy_rec/python/train_eval.py | 2 ++ easy_rec/python/utils/config_util.py | 5 +++-- easy_rec/python/utils/convert_rtp_fg.py | 6 ++++-- easy_rec/python/utils/distribution_utils.py | 2 ++ easy_rec/python/utils/ds_util.py | 1 + easy_rec/python/utils/embedding_utils.py | 1 + easy_rec/python/utils/estimator_utils.py | 13 ++++++++----- easy_rec/python/utils/export_big_model.py | 11 +++++++---- easy_rec/python/utils/fg_util.py | 2 ++ easy_rec/python/utils/hit_rate_utils.py | 3 ++- easy_rec/python/utils/hpo_util.py | 1 + easy_rec/python/utils/hvd_utils.py | 1 + easy_rec/python/utils/load_class.py | 5 +++-- easy_rec/python/utils/meta_graph_editor.py | 3 ++- easy_rec/python/utils/numpy_utils.py | 1 + easy_rec/python/utils/pai_util.py | 3 ++- easy_rec/python/utils/tf_utils.py | 1 + examples/data/amazon_books_data/process_amazon.py | 3 ++- examples/data/movielens_1m/process_ml_1m.py | 3 ++- pai_jobs/run.py | 4 +++- setup.py | 3 +-- 179 files changed, 372 insertions(+), 146 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ff72a56b..d88700aed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,8 +6,12 @@ repos: additional_dependencies: [ 'flake8-docstrings==1.5.0' ] - - repo: https://github.com/pycqa/isort - rev: 5.12.0 + - repo: https://github.com/asottile/seed-isort-config + rev: v2.2.0 + hooks: + - id: seed-isort-config + - repo: https://github.com/timothycrosley/isort + rev: 4.3.21 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-yapf diff --git a/docs/source/_ext/post_process.py b/docs/source/_ext/post_process.py index 221dcdaa0..13bda9b34 100644 --- a/docs/source/_ext/post_process.py +++ b/docs/source/_ext/post_process.py @@ -1,4 +1,5 @@ import logging + from docutils import nodes from docutils.transforms import Transform diff --git a/docs/source/conf.py b/docs/source/conf.py index 5cc1d0f5b..8eac72396 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,9 +13,10 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # import os -import sphinx_rtd_theme import sys +import sphinx_rtd_theme + import easy_rec # sys.path.insert(0, os.path.abspath('.')) diff --git a/easy_rec/python/builders/loss_builder.py b/easy_rec/python/builders/loss_builder.py index 157d1cbfa..f561be235 100644 --- a/easy_rec/python/builders/loss_builder.py +++ b/easy_rec/python/builders/loss_builder.py @@ -1,16 +1,18 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import numpy as np import tensorflow as tf -from easy_rec.python.loss.f1_reweight_loss import f1_reweight_sigmoid_cross_entropy # NOQA from easy_rec.python.loss.focal_loss import sigmoid_focal_loss_with_logits from easy_rec.python.loss.jrc_loss import jrc_loss +from easy_rec.python.protos.loss_pb2 import LossType + +from easy_rec.python.loss.f1_reweight_loss import f1_reweight_sigmoid_cross_entropy # NOQA from easy_rec.python.loss.listwise_loss import listwise_distill_loss, listwise_rank_loss # NOQA from easy_rec.python.loss.pairwise_loss import pairwise_focal_loss, pairwise_hinge_loss, pairwise_logistic_loss, pairwise_loss # NOQA from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_loss # NOQA -from easy_rec.python.protos.loss_pb2 import LossType if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/builders/optimizer_builder.py b/easy_rec/python/builders/optimizer_builder.py index de9b5efb4..990647e65 100644 --- a/easy_rec/python/builders/optimizer_builder.py +++ b/easy_rec/python/builders/optimizer_builder.py @@ -15,6 +15,7 @@ # ============================================================================== """Functions to build training optimizers.""" import logging + import tensorflow as tf from easy_rec.python.compat import weight_decay_optimizers diff --git a/easy_rec/python/compat/adam_s.py b/easy_rec/python/compat/adam_s.py index 3fa067732..d65b0c108 100644 --- a/easy_rec/python/compat/adam_s.py +++ b/easy_rec/python/compat/adam_s.py @@ -17,9 +17,10 @@ from tensorflow.python.eager import context from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops, control_flow_ops, math_ops, resource_variable_ops, state_ops # NOQA from tensorflow.python.training import optimizer, training_ops +from tensorflow.python.ops import array_ops, control_flow_ops, math_ops, resource_variable_ops, state_ops # NOQA + class AdamOptimizerS(optimizer.Optimizer): """Optimizer that implements the Adam algorithm. diff --git a/easy_rec/python/compat/dynamic_variable.py b/easy_rec/python/compat/dynamic_variable.py index 9bd7f43dd..d41e7a716 100644 --- a/easy_rec/python/compat/dynamic_variable.py +++ b/easy_rec/python/compat/dynamic_variable.py @@ -15,6 +15,7 @@ # import json + import tensorflow as tf from sparse_operation_kit.experiment import raw_ops as dynamic_variable_ops from sparse_operation_kit.experiment.communication import num_gpus @@ -22,6 +23,7 @@ from tensorflow.python.framework import ops # from tensorflow.python.ops import array_ops from tensorflow.python.ops import resource_variable_ops + from tensorflow.python.ops.resource_variable_ops import ResourceVariable, variable_accessed # NOQA # from tensorflow.python.util import object_identity diff --git a/easy_rec/python/compat/early_stopping.py b/easy_rec/python/compat/early_stopping.py index 7d9bde4b0..2a69f6d88 100644 --- a/easy_rec/python/compat/early_stopping.py +++ b/easy_rec/python/compat/early_stopping.py @@ -19,19 +19,21 @@ import logging import operator import os -import tensorflow as tf import threading import time + +import tensorflow as tf from distutils.version import LooseVersion from tensorflow.python.framework import dtypes, ops from tensorflow.python.ops import init_ops, state_ops, variable_scope from tensorflow.python.platform import gfile, tf_logging from tensorflow.python.summary import summary_iterator -from tensorflow.python.training import basic_session_run_hooks, session_run_hook, training_util # NOQA from easy_rec.python.utils.config_util import parse_time from easy_rec.python.utils.load_class import load_by_path +from tensorflow.python.training import basic_session_run_hooks, session_run_hook, training_util # NOQA + if LooseVersion(tf.__version__) >= LooseVersion('2.12.0'): from tensorflow_estimator.python.estimator.estimator_export import estimator_export # NOQA else: diff --git a/easy_rec/python/compat/embedding_parallel_saver.py b/easy_rec/python/compat/embedding_parallel_saver.py index f3db9afa4..2b5646185 100644 --- a/easy_rec/python/compat/embedding_parallel_saver.py +++ b/easy_rec/python/compat/embedding_parallel_saver.py @@ -1,18 +1,20 @@ # -*- encoding:utf-8 -*- import logging -import numpy as np import os + +import numpy as np from tensorflow.core.protobuf import saver_pb2 from tensorflow.python.framework import dtypes, ops -# from tensorflow.python.ops import math_ops -# from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops, control_flow_ops, script_ops, state_ops # NOQA from tensorflow.python.platform import gfile from tensorflow.python.training import saver from easy_rec.python.utils import constant +# from tensorflow.python.ops import math_ops +# from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import array_ops, control_flow_ops, script_ops, state_ops # NOQA + try: import horovod.tensorflow as hvd from sparse_operation_kit.experiment import raw_ops as dynamic_variable_ops diff --git a/easy_rec/python/compat/estimator_train.py b/easy_rec/python/compat/estimator_train.py index 72ffec9ef..917c451da 100644 --- a/easy_rec/python/compat/estimator_train.py +++ b/easy_rec/python/compat/estimator_train.py @@ -2,15 +2,17 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os + import tensorflow as tf -from tensorflow.python.distribute import estimator_training as distribute_coordinator_training # NOQA from tensorflow.python.estimator import run_config as run_config_lib -from tensorflow.python.estimator.training import _assert_eval_spec, _ContinuousEvalListener, _TrainingExecutor # NOQA from tensorflow.python.util import compat from easy_rec.python.compat.exporter import FinalExporter from easy_rec.python.utils import estimator_utils +from tensorflow.python.distribute import estimator_training as distribute_coordinator_training # NOQA +from tensorflow.python.estimator.training import _assert_eval_spec, _ContinuousEvalListener, _TrainingExecutor # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 gfile = tf.gfile diff --git a/easy_rec/python/compat/exporter.py b/easy_rec/python/compat/exporter.py index 3bbd4af62..f25c525e0 100644 --- a/easy_rec/python/compat/exporter.py +++ b/easy_rec/python/compat/exporter.py @@ -18,6 +18,7 @@ from __future__ import absolute_import, division, print_function import os + from tensorflow.python.estimator import gc, util from tensorflow.python.estimator.canned import metric_keys from tensorflow.python.estimator.exporter import Exporter, _SavedModelExporter diff --git a/easy_rec/python/compat/feature_column/feature_column.py b/easy_rec/python/compat/feature_column/feature_column.py index b2acd612d..0345aeb6d 100644 --- a/easy_rec/python/compat/feature_column/feature_column.py +++ b/easy_rec/python/compat/feature_column/feature_column.py @@ -133,8 +133,9 @@ import abc import collections import math -import numpy as np import os + +import numpy as np import six from tensorflow.python.eager import context from tensorflow.python.framework import dtypes, ops @@ -142,8 +143,6 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.keras.engine import training from tensorflow.python.layers import base -# from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops, check_ops, control_flow_ops, data_flow_ops, embedding_ops, init_ops, lookup_ops, math_ops, nn_ops, parsing_ops, resource_variable_ops, sparse_ops, string_ops, template, variable_scope, variables # NOQA # from tensorflow.python.ops.ragged import ragged_tensor # from tensorflow.python.ops.ragged import ragged_util from tensorflow.python.platform import gfile @@ -154,6 +153,9 @@ from easy_rec.python.compat.feature_column import utils as fc_utils from easy_rec.python.utils import conditional, constant, embedding_utils +# from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import array_ops, check_ops, control_flow_ops, data_flow_ops, embedding_ops, init_ops, lookup_ops, math_ops, nn_ops, parsing_ops, resource_variable_ops, sparse_ops, string_ops, template, variable_scope, variables # NOQA + try: from easy_rec.python.compat import dynamic_variable except Exception: diff --git a/easy_rec/python/compat/feature_column/feature_column_v2.py b/easy_rec/python/compat/feature_column/feature_column_v2.py index d121846ad..2bb8aa2e2 100644 --- a/easy_rec/python/compat/feature_column/feature_column_v2.py +++ b/easy_rec/python/compat/feature_column/feature_column_v2.py @@ -129,10 +129,11 @@ import abc import collections import math -import numpy as np import os -import six import sys + +import numpy as np +import six import tensorflow as tf from tensorflow.python.eager import context from tensorflow.python.framework import dtypes, ops @@ -143,7 +144,6 @@ from tensorflow.python.keras import utils from tensorflow.python.keras.engine import training from tensorflow.python.keras.engine.base_layer import Layer -from tensorflow.python.ops import array_ops, check_ops, control_flow_ops, embedding_ops, init_ops, lookup_ops, math_ops, nn_ops, parsing_ops, sparse_ops, string_ops, variable_scope, variables # NOQA from tensorflow.python.platform import gfile from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import checkpoint_utils @@ -153,9 +153,12 @@ from easy_rec.python.compat import ops as compat_ops from easy_rec.python.compat.feature_column import feature_column as fc_old from easy_rec.python.compat.feature_column import utils as fc_utils -from easy_rec.python.compat.feature_column.feature_column import embedding_lookup_ragged # NOQA from easy_rec.python.layers import utils as layer_utils +from tensorflow.python.ops import array_ops, check_ops, control_flow_ops, embedding_ops, init_ops, lookup_ops, math_ops, nn_ops, parsing_ops, sparse_ops, string_ops, variable_scope, variables # NOQA + +from easy_rec.python.compat.feature_column.feature_column import embedding_lookup_ragged # NOQA + _FEATURE_COLUMN_DEPRECATION_DATE = None _FEATURE_COLUMN_DEPRECATION = ( 'The old _FeatureColumn APIs are being ' diff --git a/easy_rec/python/compat/feature_column/sequence_feature_column.py b/easy_rec/python/compat/feature_column/sequence_feature_column.py index 5bf97107f..9c7f3b9b2 100644 --- a/easy_rec/python/compat/feature_column/sequence_feature_column.py +++ b/easy_rec/python/compat/feature_column/sequence_feature_column.py @@ -21,13 +21,15 @@ from __future__ import absolute_import, division, print_function import collections + from tensorflow.python.framework import dtypes, ops, tensor_shape -from tensorflow.python.ops import array_ops, check_ops, math_ops, parsing_ops, sparse_ops # NOQA from easy_rec.python.compat.feature_column import feature_column as fc_v1 from easy_rec.python.compat.feature_column import feature_column_v2 as fc from easy_rec.python.compat.feature_column import utils as fc_utils +from tensorflow.python.ops import array_ops, check_ops, math_ops, parsing_ops, sparse_ops # NOQA + # pylint: disable=protected-access diff --git a/easy_rec/python/compat/layers.py b/easy_rec/python/compat/layers.py index 0311f8cf5..75b170ce4 100644 --- a/easy_rec/python/compat/layers.py +++ b/easy_rec/python/compat/layers.py @@ -18,6 +18,7 @@ from __future__ import absolute_import, division, print_function import functools + from tensorflow.python.framework import dtypes, ops from tensorflow.python.ops import init_ops, nn, variable_scope diff --git a/easy_rec/python/compat/optimizers.py b/easy_rec/python/compat/optimizers.py index 3828f323e..eaf45975b 100644 --- a/easy_rec/python/compat/optimizers.py +++ b/easy_rec/python/compat/optimizers.py @@ -18,12 +18,11 @@ from __future__ import absolute_import, division, print_function import logging + import six import tensorflow as tf # from tensorflow.contrib import framework as contrib_framework from tensorflow.python.framework import dtypes, ops -# from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops, clip_ops, control_flow_ops, gen_nn_ops, init_ops, math_ops, random_ops # NOQA from tensorflow.python.ops import variable_scope as vs from tensorflow.python.ops import variables as vars_ from tensorflow.python.summary import summary @@ -34,6 +33,9 @@ from easy_rec.python.ops.incr_record import set_sparse_indices from easy_rec.python.utils import constant, estimator_utils +# from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import array_ops, clip_ops, control_flow_ops, gen_nn_ops, init_ops, math_ops, random_ops # NOQA + try: from tensorflow.python.framework import indexed_slices except Exception: diff --git a/easy_rec/python/compat/queues.py b/easy_rec/python/compat/queues.py index 1be904921..b28f73135 100644 --- a/easy_rec/python/compat/queues.py +++ b/easy_rec/python/compat/queues.py @@ -11,7 +11,6 @@ import errno import logging import os -import six import sys import threading import time @@ -20,6 +19,8 @@ from multiprocessing.util import Finalize, is_exiting, register_after_fork from queue import Empty, Full +import six + try: from multiprocessing import context except ImportError: diff --git a/easy_rec/python/compat/regularizers.py b/easy_rec/python/compat/regularizers.py index 840c35671..f49cb87dc 100644 --- a/easy_rec/python/compat/regularizers.py +++ b/easy_rec/python/compat/regularizers.py @@ -19,6 +19,7 @@ from __future__ import absolute_import, division, print_function import numbers + from tensorflow.python.framework import constant_op, ops from tensorflow.python.ops import math_ops, nn, standard_ops from tensorflow.python.platform import tf_logging as logging diff --git a/easy_rec/python/compat/sok_optimizer.py b/easy_rec/python/compat/sok_optimizer.py index 470bbb471..e8f5a98c1 100644 --- a/easy_rec/python/compat/sok_optimizer.py +++ b/easy_rec/python/compat/sok_optimizer.py @@ -18,11 +18,12 @@ from tensorflow.python.eager import context # from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -# from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import array_ops, gradients, resource_variable_ops, state_ops # NOQA from easy_rec.python.compat.dynamic_variable import DynamicVariable +# from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import array_ops, gradients, resource_variable_ops, state_ops # NOQA + def OptimizerWrapper(optimizer): """Abbreviated as ``sok.experiment.OptimizerWrapper``. diff --git a/easy_rec/python/compat/sync_replicas_optimizer.py b/easy_rec/python/compat/sync_replicas_optimizer.py index 8245f26db..634d589c2 100644 --- a/easy_rec/python/compat/sync_replicas_optimizer.py +++ b/easy_rec/python/compat/sync_replicas_optimizer.py @@ -17,11 +17,12 @@ from tensorflow.core.framework import types_pb2 from tensorflow.python.framework import errors_impl, ops -from tensorflow.python.ops import array_ops, control_flow_ops, data_flow_ops, state_ops, variable_scope, variables # NOQA from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import optimizer, queue_runner, session_manager, session_run_hook # NOQA from tensorflow.python.util.tf_export import tf_export +from tensorflow.python.ops import array_ops, control_flow_ops, data_flow_ops, state_ops, variable_scope, variables # NOQA +from tensorflow.python.training import optimizer, queue_runner, session_manager, session_run_hook # NOQA + # Please note that the gradients from replicas are averaged instead of summed # (as in the old sync_replicas_optimizer) so you need to increase the learning diff --git a/easy_rec/python/compat/weight_decay_optimizers.py b/easy_rec/python/compat/weight_decay_optimizers.py index 5a5e6b538..664328f99 100755 --- a/easy_rec/python/compat/weight_decay_optimizers.py +++ b/easy_rec/python/compat/weight_decay_optimizers.py @@ -16,12 +16,13 @@ from __future__ import absolute_import, division, print_function from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops, control_flow_ops, resource_variable_ops, state_ops # NOQA from tensorflow.python.training import adam from tensorflow.python.training import momentum as momentum_opt from tensorflow.python.training import optimizer from tensorflow.python.util.tf_export import tf_export +from tensorflow.python.ops import array_ops, control_flow_ops, resource_variable_ops, state_ops # NOQA + class DecoupledWeightDecayExtension(object): """This class allows to extend optimizers with decoupled weight decay. diff --git a/easy_rec/python/core/easyrec_metrics/__init__.py b/easy_rec/python/core/easyrec_metrics/__init__.py index 2783b9e27..f5d82c865 100644 --- a/easy_rec/python/core/easyrec_metrics/__init__.py +++ b/easy_rec/python/core/easyrec_metrics/__init__.py @@ -1,5 +1,6 @@ import logging import os + import tensorflow as tf from easy_rec.python.utils import pai_util diff --git a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py index d30f36d8b..938f65c47 100644 --- a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py +++ b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py @@ -18,12 +18,13 @@ from tensorflow.python.eager import context from tensorflow.python.framework import dtypes, ops, sparse_tensor -from tensorflow.python.ops import array_ops, check_ops, confusion_matrix, control_flow_ops, math_ops, nn, sets, sparse_ops, state_ops, variable_scope, weights_broadcast_ops # NOQA from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import distribution_strategy_context from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.tf_export import tf_export +from tensorflow.python.ops import array_ops, check_ops, confusion_matrix, control_flow_ops, math_ops, nn, sets, sparse_ops, state_ops, variable_scope, weights_broadcast_ops # NOQA + def metric_variable(shape, dtype, validate_shape=True, name=None): """Create variable in `GraphKeys.(LOCAL|METRIC_VARIABLES)` collections. diff --git a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py index 4c2c3a7b1..641fbe362 100644 --- a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py +++ b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py @@ -18,11 +18,12 @@ from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.eager import context from tensorflow.python.framework import dtypes, ops, sparse_tensor -from tensorflow.python.ops import array_ops, check_ops, confusion_matrix, control_flow_ops, math_ops, nn, sets, sparse_ops, state_ops, variable_scope, weights_broadcast_ops # NOQA from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.tf_export import tf_export +from tensorflow.python.ops import array_ops, check_ops, confusion_matrix, control_flow_ops, math_ops, nn, sets, sparse_ops, state_ops, variable_scope, weights_broadcast_ops # NOQA + # from tensorflow.python.training import distribution_strategy_context diff --git a/easy_rec/python/core/metrics.py b/easy_rec/python/core/metrics.py index 2169c9960..e1dea67a9 100644 --- a/easy_rec/python/core/metrics.py +++ b/easy_rec/python/core/metrics.py @@ -2,17 +2,20 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import json import logging -import numpy as np import os -import tensorflow as tf from collections import defaultdict + +import numpy as np +import tensorflow as tf from sklearn import metrics as sklearn_metrics -from tensorflow.python.ops import array_ops, math_ops, state_ops, variable_scope # NOQA from easy_rec.python.utils.estimator_utils import get_task_index_and_num -from easy_rec.python.utils.io_util import read_data_from_json_path, save_data_to_json_path # NOQA from easy_rec.python.utils.shape_utils import get_shape_list +from tensorflow.python.ops import array_ops, math_ops, state_ops, variable_scope # NOQA + +from easy_rec.python.utils.io_util import read_data_from_json_path, save_data_to_json_path # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/core/sampler.py b/easy_rec/python/core/sampler.py index 59fb82655..62193b289 100644 --- a/easy_rec/python/core/sampler.py +++ b/easy_rec/python/core/sampler.py @@ -5,13 +5,14 @@ import json import logging import math -import numpy as np import os -import six import sys -import tensorflow as tf import threading +import numpy as np +import six +import tensorflow as tf + from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils import ds_util from easy_rec.python.utils.config_util import process_multi_file_input_path diff --git a/easy_rec/python/eval.py b/easy_rec/python/eval.py index f0b80abc5..491df0f48 100644 --- a/easy_rec/python/eval.py +++ b/easy_rec/python/eval.py @@ -3,6 +3,7 @@ import logging import os + import six import tensorflow as tf from tensorflow.python.lib.io import file_io @@ -10,6 +11,7 @@ from easy_rec.python.main import distribute_evaluate, evaluate from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import config_util, ds_util, estimator_utils + from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_distribute_eval_worker_num_on_ds # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/export.py b/easy_rec/python/export.py index 0768cd4bd..72726af34 100644 --- a/easy_rec/python/export.py +++ b/easy_rec/python/export.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os + import tensorflow as tf from tensorflow.python.lib.io import file_io diff --git a/easy_rec/python/feature_column/feature_column.py b/easy_rec/python/feature_column/feature_column.py index 0bcc595c8..bea5f20d3 100644 --- a/easy_rec/python/feature_column/feature_column.py +++ b/easy_rec/python/feature_column/feature_column.py @@ -3,16 +3,18 @@ import collections import logging import sys + import tensorflow as tf from tensorflow.python.ops import partitioned_variables from tensorflow.python.platform import gfile from easy_rec.python.builders import hyperparams_builder -from easy_rec.python.compat.feature_column import feature_column_v2 as feature_column # NOQA from easy_rec.python.compat.feature_column import sequence_feature_column from easy_rec.python.protos.feature_config_pb2 import FeatureConfig, WideOrDeep from easy_rec.python.utils.proto_util import copy_obj +from easy_rec.python.compat.feature_column import feature_column_v2 as feature_column # NOQA + MAX_HASH_BUCKET_SIZE = 9223372036854775807 diff --git a/easy_rec/python/hpo/emr_hpo.py b/easy_rec/python/hpo/emr_hpo.py index fca116c64..5ec95d748 100644 --- a/easy_rec/python/hpo/emr_hpo.py +++ b/easy_rec/python/hpo/emr_hpo.py @@ -7,6 +7,7 @@ import os import shutil import time + from pai.automl.hpo.autotuner import AutoTuner from easy_rec.python.utils import hpo_util diff --git a/easy_rec/python/hpo/pai_hpo.py b/easy_rec/python/hpo/pai_hpo.py index 11fb5c6ab..d2c9e3f3c 100644 --- a/easy_rec/python/hpo/pai_hpo.py +++ b/easy_rec/python/hpo/pai_hpo.py @@ -6,6 +6,7 @@ import os import shutil import time + from pai.automl import hpo from easy_rec.python.utils import hpo_util diff --git a/easy_rec/python/inference/csv_predictor.py b/easy_rec/python/inference/csv_predictor.py index 9053ff3f2..83dbb5f2a 100644 --- a/easy_rec/python/inference/csv_predictor.py +++ b/easy_rec/python/inference/csv_predictor.py @@ -4,13 +4,15 @@ import logging import os + import tensorflow as tf from tensorflow.python.platform import gfile -from easy_rec.python.inference.predictor import SINGLE_PLACEHOLDER_FEATURE_KEY, Predictor # NOQA from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils.check_utils import check_split +from easy_rec.python.inference.predictor import SINGLE_PLACEHOLDER_FEATURE_KEY, Predictor # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/inference/hive_parquet_predictor.py b/easy_rec/python/inference/hive_parquet_predictor.py index e6c878309..f2d7976de 100644 --- a/easy_rec/python/inference/hive_parquet_predictor.py +++ b/easy_rec/python/inference/hive_parquet_predictor.py @@ -2,11 +2,12 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from __future__ import absolute_import, division, print_function -import numpy as np import os +import time + +import numpy as np import pandas as pd import tensorflow as tf -import time from tensorflow.python.platform import gfile from easy_rec.python.inference.predictor import Predictor diff --git a/easy_rec/python/inference/hive_predictor.py b/easy_rec/python/inference/hive_predictor.py index cf21062fc..a00b4b98e 100644 --- a/easy_rec/python/inference/hive_predictor.py +++ b/easy_rec/python/inference/hive_predictor.py @@ -3,8 +3,9 @@ from __future__ import absolute_import, division, print_function import os -import tensorflow as tf import time + +import tensorflow as tf from tensorflow.python.platform import gfile from easy_rec.python.inference.predictor import Predictor diff --git a/easy_rec/python/inference/parquet_predictor.py b/easy_rec/python/inference/parquet_predictor.py index 3d6f8dac1..d808b0c9b 100644 --- a/easy_rec/python/inference/parquet_predictor.py +++ b/easy_rec/python/inference/parquet_predictor.py @@ -3,8 +3,9 @@ from __future__ import absolute_import, division, print_function import logging -import numpy as np import os + +import numpy as np import pandas as pd import tensorflow as tf from tensorflow.python.platform import gfile diff --git a/easy_rec/python/inference/parquet_predictor_v2.py b/easy_rec/python/inference/parquet_predictor_v2.py index 2ece02cea..9b77fb3ad 100644 --- a/easy_rec/python/inference/parquet_predictor_v2.py +++ b/easy_rec/python/inference/parquet_predictor_v2.py @@ -3,8 +3,9 @@ from __future__ import absolute_import, division, print_function import logging -import numpy as np import os + +import numpy as np import pandas as pd import tensorflow as tf from tensorflow.python.platform import gfile diff --git a/easy_rec/python/inference/predictor.py b/easy_rec/python/inference/predictor.py index ba9b4204d..e77351a88 100644 --- a/easy_rec/python/inference/predictor.py +++ b/easy_rec/python/inference/predictor.py @@ -6,21 +6,23 @@ import json import logging import math -import numpy as np import os +import time + +import numpy as np import six import tensorflow as tf -import time from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.python.platform import gfile from tensorflow.python.saved_model import constants, signature_constants import easy_rec from easy_rec.python.utils import numpy_utils -from easy_rec.python.utils.config_util import get_configs_from_pipeline_file, get_input_name_from_fg_json, search_fg_json # NOQA from easy_rec.python.utils.input_utils import get_type_defaults from easy_rec.python.utils.load_class import get_register_class_meta +from easy_rec.python.utils.config_util import get_configs_from_pipeline_file, get_input_name_from_fg_json, search_fg_json # NOQA + try: tf.load_op_library(os.path.join(easy_rec.ops_dir, 'libcustom_ops.so')) except Exception as ex: diff --git a/easy_rec/python/inference/processor/test.py b/easy_rec/python/inference/processor/test.py index 741f744bd..af3391644 100644 --- a/easy_rec/python/inference/processor/test.py +++ b/easy_rec/python/inference/processor/test.py @@ -5,10 +5,11 @@ import glob import json import logging -import numpy as np import os import subprocess import time + +import numpy as np from google.protobuf import text_format from easy_rec.python.protos import dataset_pb2, pipeline_pb2, tf_predict_pb2 diff --git a/easy_rec/python/inference/vector_retrieve.py b/easy_rec/python/inference/vector_retrieve.py index 53f2368e4..4fc6a268e 100644 --- a/easy_rec/python/inference/vector_retrieve.py +++ b/easy_rec/python/inference/vector_retrieve.py @@ -2,11 +2,12 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from __future__ import absolute_import, division, print_function -import common_io import logging +from datetime import datetime + +import common_io import numpy as np import tensorflow as tf -from datetime import datetime try: import graphlearn as gl diff --git a/easy_rec/python/input/batch_tfrecord_input.py b/easy_rec/python/input/batch_tfrecord_input.py index 5e0e27ba8..74d7ea192 100644 --- a/easy_rec/python/input/batch_tfrecord_input.py +++ b/easy_rec/python/input/batch_tfrecord_input.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/criteo_binary_reader.py b/easy_rec/python/input/criteo_binary_reader.py index ba69f6195..26b1c8d95 100644 --- a/easy_rec/python/input/criteo_binary_reader.py +++ b/easy_rec/python/input/criteo_binary_reader.py @@ -5,11 +5,12 @@ import concurrent.futures import glob import logging -import numpy as np import os import queue import time +import numpy as np + class BinaryDataset: diff --git a/easy_rec/python/input/criteo_input.py b/easy_rec/python/input/criteo_input.py index 9d438ad60..ae345daf4 100644 --- a/easy_rec/python/input/criteo_input.py +++ b/easy_rec/python/input/criteo_input.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.input.criteo_binary_reader import BinaryDataset diff --git a/easy_rec/python/input/csv_input.py b/easy_rec/python/input/csv_input.py index 4c130d67d..ad25dcb09 100644 --- a/easy_rec/python/input/csv_input.py +++ b/easy_rec/python/input/csv_input.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/datahub_input.py b/easy_rec/python/input/datahub_input.py index e9a4111a2..706a1906d 100644 --- a/easy_rec/python/input/datahub_input.py +++ b/easy_rec/python/input/datahub_input.py @@ -2,8 +2,9 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import json import logging -import tensorflow as tf import traceback + +import tensorflow as tf from tensorflow.python.framework import dtypes from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/hive_input.py b/easy_rec/python/input/hive_input.py index 5a462de05..b970d157d 100644 --- a/easy_rec/python/input/hive_input.py +++ b/easy_rec/python/input/hive_input.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import logging import os + import tensorflow as tf from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/hive_parquet_input.py b/easy_rec/python/input/hive_parquet_input.py index 07847bbf0..931187394 100644 --- a/easy_rec/python/input/hive_parquet_input.py +++ b/easy_rec/python/input/hive_parquet_input.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- import logging -import numpy as np import os + +import numpy as np import pandas as pd import tensorflow as tf diff --git a/easy_rec/python/input/hive_rtp_input.py b/easy_rec/python/input/hive_rtp_input.py index df71444c5..e1a1b46b6 100644 --- a/easy_rec/python/input/hive_rtp_input.py +++ b/easy_rec/python/input/hive_rtp_input.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import logging import os + import tensorflow as tf from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/input.py b/easy_rec/python/input/input.py index 2aeff4ed6..cc025c213 100644 --- a/easy_rec/python/input/input.py +++ b/easy_rec/python/input/input.py @@ -2,10 +2,11 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os -import six -import tensorflow as tf from abc import abstractmethod from collections import OrderedDict + +import six +import tensorflow as tf from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops, sparse_ops, string_ops from tensorflow.python.platform import gfile @@ -13,12 +14,13 @@ from easy_rec.python.core import sampler as sampler_lib from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils import conditional, config_util, constant -from easy_rec.python.utils.check_utils import check_split, check_string_to_number # NOQA from easy_rec.python.utils.expr_util import get_expression from easy_rec.python.utils.input_utils import get_type_defaults -from easy_rec.python.utils.load_class import get_register_class_meta, load_by_path # NOQA from easy_rec.python.utils.tf_utils import get_tf_type +from easy_rec.python.utils.check_utils import check_split, check_string_to_number # NOQA +from easy_rec.python.utils.load_class import get_register_class_meta, load_by_path # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/input/kafka_dataset.py b/easy_rec/python/input/kafka_dataset.py index e381553d2..b8f39f83d 100644 --- a/easy_rec/python/input/kafka_dataset.py +++ b/easy_rec/python/input/kafka_dataset.py @@ -16,6 +16,7 @@ import logging import traceback + from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes, ops, tensor_shape diff --git a/easy_rec/python/input/kafka_input.py b/easy_rec/python/input/kafka_input.py index aa6f5b78f..d2bb60dbf 100644 --- a/easy_rec/python/input/kafka_input.py +++ b/easy_rec/python/input/kafka_input.py @@ -2,9 +2,10 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import json import logging +import traceback + import six import tensorflow as tf -import traceback from easy_rec.python.input.input import Input from easy_rec.python.input.kafka_dataset import KafkaDataset diff --git a/easy_rec/python/input/load_parquet.py b/easy_rec/python/input/load_parquet.py index c8940a126..8154669d6 100644 --- a/easy_rec/python/input/load_parquet.py +++ b/easy_rec/python/input/load_parquet.py @@ -1,8 +1,9 @@ import logging import multiprocessing +import queue + import numpy as np import pandas as pd -import queue def start_data_proc( diff --git a/easy_rec/python/input/odps_input_v2.py b/easy_rec/python/input/odps_input_v2.py index 916ab6f25..bd6bac28e 100644 --- a/easy_rec/python/input/odps_input_v2.py +++ b/easy_rec/python/input/odps_input_v2.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/odps_input_v3.py b/easy_rec/python/input/odps_input_v3.py index 85d32bd62..93d273bf7 100644 --- a/easy_rec/python/input/odps_input_v3.py +++ b/easy_rec/python/input/odps_input_v3.py @@ -3,6 +3,7 @@ import logging import sys + import tensorflow as tf from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/odps_rtp_input.py b/easy_rec/python/input/odps_rtp_input.py index 4e175f4b0..359938be5 100644 --- a/easy_rec/python/input/odps_rtp_input.py +++ b/easy_rec/python/input/odps_rtp_input.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/odps_rtp_input_v2.py b/easy_rec/python/input/odps_rtp_input_v2.py index 7ac7b234c..8f070c3d0 100644 --- a/easy_rec/python/input/odps_rtp_input_v2.py +++ b/easy_rec/python/input/odps_rtp_input_v2.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import json import logging + import tensorflow as tf from easy_rec.python.input.odps_rtp_input import OdpsRTPInput diff --git a/easy_rec/python/input/parquet_input.py b/easy_rec/python/input/parquet_input.py index b7446409f..3fdc84a3e 100644 --- a/easy_rec/python/input/parquet_input.py +++ b/easy_rec/python/input/parquet_input.py @@ -3,8 +3,9 @@ import logging import multiprocessing import queue -import tensorflow as tf import time + +import tensorflow as tf from tensorflow.python.ops import array_ops from easy_rec.python.compat import queues diff --git a/easy_rec/python/input/parquet_input_v2.py b/easy_rec/python/input/parquet_input_v2.py index 92968ef56..af2c2414d 100644 --- a/easy_rec/python/input/parquet_input_v2.py +++ b/easy_rec/python/input/parquet_input_v2.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. # import logging import os + # import numpy as np # import pandas as pd import tensorflow as tf diff --git a/easy_rec/python/input/parquet_input_v3.py b/easy_rec/python/input/parquet_input_v3.py index 6a970522d..f779c3249 100644 --- a/easy_rec/python/input/parquet_input_v3.py +++ b/easy_rec/python/input/parquet_input_v3.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/rtp_input.py b/easy_rec/python/input/rtp_input.py index 322719bcc..1c677f414 100644 --- a/easy_rec/python/input/rtp_input.py +++ b/easy_rec/python/input/rtp_input.py @@ -1,14 +1,16 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.input.input import Input from easy_rec.python.ops.gen_str_avx_op import str_split_by_chr -from easy_rec.python.utils.check_utils import check_split, check_string_to_number # NOQA from easy_rec.python.utils.input_utils import string_to_number from easy_rec.python.utils.tf_utils import get_tf_type +from easy_rec.python.utils.check_utils import check_split, check_string_to_number # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/input/rtp_input_v2.py b/easy_rec/python/input/rtp_input_v2.py index 60b792013..d427c3e0b 100644 --- a/easy_rec/python/input/rtp_input_v2.py +++ b/easy_rec/python/input/rtp_input_v2.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.input.input import Input diff --git a/easy_rec/python/input/tfrecord_input.py b/easy_rec/python/input/tfrecord_input.py index a3c130653..010eec649 100644 --- a/easy_rec/python/input/tfrecord_input.py +++ b/easy_rec/python/input/tfrecord_input.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.input.input import Input diff --git a/easy_rec/python/layers/backbone.py b/easy_rec/python/layers/backbone.py index 927aa19f8..c82fd3454 100644 --- a/easy_rec/python/layers/backbone.py +++ b/easy_rec/python/layers/backbone.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import six import tensorflow as tf from google.protobuf import struct_pb2 diff --git a/easy_rec/python/layers/capsule_layer.py b/easy_rec/python/layers/capsule_layer.py index 1fac8e797..a710a9e49 100644 --- a/easy_rec/python/layers/capsule_layer.py +++ b/easy_rec/python/layers/capsule_layer.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import numpy as np import tensorflow as tf diff --git a/easy_rec/python/layers/dnn.py b/easy_rec/python/layers/dnn.py index 78290ba11..1d33e0628 100644 --- a/easy_rec/python/layers/dnn.py +++ b/easy_rec/python/layers/dnn.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.utils.activation import get_activation diff --git a/easy_rec/python/layers/input_layer.py b/easy_rec/python/layers/input_layer.py index 452009ff1..092ce19c5 100644 --- a/easy_rec/python/layers/input_layer.py +++ b/easy_rec/python/layers/input_layer.py @@ -2,22 +2,24 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os -import tensorflow as tf from collections import OrderedDict + +import tensorflow as tf from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops, variable_scope from easy_rec.python.compat import regularizers from easy_rec.python.compat.feature_column import feature_column -from easy_rec.python.compat.feature_column.feature_column_v2 import is_embedding_column # NOQA from easy_rec.python.feature_column.feature_column import FeatureColumnParser from easy_rec.python.feature_column.feature_group import FeatureGroup -from easy_rec.python.layers import sequence_feature_layer, variational_dropout_layer # NOQA from easy_rec.python.layers.keras import TextCNN from easy_rec.python.layers.utils import Parameter from easy_rec.python.protos.feature_config_pb2 import WideOrDeep from easy_rec.python.utils import conditional, shape_utils +from easy_rec.python.compat.feature_column.feature_column_v2 import is_embedding_column # NOQA +from easy_rec.python.layers import sequence_feature_layer, variational_dropout_layer # NOQA + class InputLayer(object): """Input Layer for generate input features. diff --git a/easy_rec/python/layers/keras/__init__.py b/easy_rec/python/layers/keras/__init__.py index ede06379e..ab5e50324 100644 --- a/easy_rec/python/layers/keras/__init__.py +++ b/easy_rec/python/layers/keras/__init__.py @@ -2,7 +2,6 @@ from .auxiliary_loss import AuxiliaryLoss from .blocks import MLP, Gate, Highway, TextCNN from .bst import BST -from .custom_ops import EditDistance, MappedDotProduct, OverlapFeature, SeqAugmentOps, TextNormalize # NOQA from .data_augment import SeqAugment from .din import DIN from .embedding import EmbeddingLayer @@ -11,6 +10,8 @@ from .mask_net import MaskBlock, MaskNet from .multi_head_attention import MultiHeadAttention from .multi_task import AITMTower, MMoE -from .numerical_embedding import AutoDisEmbedding, NaryDisEmbedding, PeriodicEmbedding # NOQA from .ppnet import PPNet from .transformer import TextEncoder, TransformerBlock, TransformerEncoder + +from .custom_ops import EditDistance, MappedDotProduct, OverlapFeature, SeqAugmentOps, TextNormalize # NOQA +from .numerical_embedding import AutoDisEmbedding, NaryDisEmbedding, PeriodicEmbedding # NOQA diff --git a/easy_rec/python/layers/keras/auxiliary_loss.py b/easy_rec/python/layers/keras/auxiliary_loss.py index abe116c09..8371bfdff 100644 --- a/easy_rec/python/layers/keras/auxiliary_loss.py +++ b/easy_rec/python/layers/keras/auxiliary_loss.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.loss import contrastive_loss diff --git a/easy_rec/python/layers/keras/blocks.py b/easy_rec/python/layers/keras/blocks.py index 0f8d1fd20..85cc83dc2 100644 --- a/easy_rec/python/layers/keras/blocks.py +++ b/easy_rec/python/layers/keras/blocks.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Convenience blocks for building models.""" import logging + import tensorflow as tf from tensorflow.python.keras.initializers import Constant from tensorflow.python.keras.layers import Dense, Dropout, Lambda, Layer diff --git a/easy_rec/python/layers/keras/custom_ops.py b/easy_rec/python/layers/keras/custom_ops.py index decf6deed..feeacb4f7 100644 --- a/easy_rec/python/layers/keras/custom_ops.py +++ b/easy_rec/python/layers/keras/custom_ops.py @@ -3,6 +3,7 @@ """Convenience blocks for using custom ops.""" import logging import os + import tensorflow as tf from tensorflow.python.framework import ops from tensorflow.python.keras.layers import Layer diff --git a/easy_rec/python/layers/keras/din.py b/easy_rec/python/layers/keras/din.py index f45be564a..d3bded0bb 100644 --- a/easy_rec/python/layers/keras/din.py +++ b/easy_rec/python/layers/keras/din.py @@ -1,6 +1,7 @@ # -*- encoding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from tensorflow.python.keras.layers import Layer diff --git a/easy_rec/python/layers/keras/einsum_dense.py b/easy_rec/python/layers/keras/einsum_dense.py index 86fe185c6..dc53a3317 100644 --- a/easy_rec/python/layers/keras/einsum_dense.py +++ b/easy_rec/python/layers/keras/einsum_dense.py @@ -2,10 +2,12 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import re import string + import tensorflow as tf -from tensorflow.python.keras import activations, constraints, initializers, regularizers # NOQA from tensorflow.python.keras.layers import Layer +from tensorflow.python.keras import activations, constraints, initializers, regularizers # NOQA + class EinsumDense(Layer): """A layer that uses `einsum` as the backing computation. diff --git a/easy_rec/python/layers/keras/fibinet.py b/easy_rec/python/layers/keras/fibinet.py index 2b667c790..d58f4d154 100644 --- a/easy_rec/python/layers/keras/fibinet.py +++ b/easy_rec/python/layers/keras/fibinet.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import itertools import logging + import tensorflow as tf from tensorflow.python.keras.layers import Dense, Layer diff --git a/easy_rec/python/layers/keras/mask_net.py b/easy_rec/python/layers/keras/mask_net.py index 1611f2941..7ae12f20e 100644 --- a/easy_rec/python/layers/keras/mask_net.py +++ b/easy_rec/python/layers/keras/mask_net.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from tensorflow.python.keras.layers import Activation, Dense, Layer diff --git a/easy_rec/python/layers/keras/multi_head_attention.py b/easy_rec/python/layers/keras/multi_head_attention.py index a08d1fcab..33386dd8a 100644 --- a/easy_rec/python/layers/keras/multi_head_attention.py +++ b/easy_rec/python/layers/keras/multi_head_attention.py @@ -1,8 +1,9 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import math -import numpy as np import string + +import numpy as np import tensorflow as tf from tensorflow.python.keras import constraints, initializers, regularizers from tensorflow.python.keras.layers import Dropout, Layer, Softmax diff --git a/easy_rec/python/layers/keras/multi_task.py b/easy_rec/python/layers/keras/multi_task.py index 24ec973f5..133d53558 100644 --- a/easy_rec/python/layers/keras/multi_task.py +++ b/easy_rec/python/layers/keras/multi_task.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from tensorflow.python.keras.layers import Dense, Layer diff --git a/easy_rec/python/layers/keras/numerical_embedding.py b/easy_rec/python/layers/keras/numerical_embedding.py index fef1c0f0b..031878595 100644 --- a/easy_rec/python/layers/keras/numerical_embedding.py +++ b/easy_rec/python/layers/keras/numerical_embedding.py @@ -3,6 +3,7 @@ import logging import math import os + import tensorflow as tf from tensorflow.python.framework import ops from tensorflow.python.keras.layers import Layer diff --git a/easy_rec/python/layers/keras/ppnet.py b/easy_rec/python/layers/keras/ppnet.py index 204b700b2..29c063954 100644 --- a/easy_rec/python/layers/keras/ppnet.py +++ b/easy_rec/python/layers/keras/ppnet.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Convenience blocks for building models.""" import logging + import tensorflow as tf from easy_rec.python.layers.keras.activation import activation_layer diff --git a/easy_rec/python/layers/keras/transformer.py b/easy_rec/python/layers/keras/transformer.py index c2cf8fd84..a30eb45f3 100644 --- a/easy_rec/python/layers/keras/transformer.py +++ b/easy_rec/python/layers/keras/transformer.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import numpy as np import tensorflow as tf from tensorflow.python.keras.layers import Dense, Dropout, Embedding, Layer diff --git a/easy_rec/python/layers/mmoe.py b/easy_rec/python/layers/mmoe.py index bfac751a1..42e96a64c 100644 --- a/easy_rec/python/layers/mmoe.py +++ b/easy_rec/python/layers/mmoe.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.layers import dnn diff --git a/easy_rec/python/layers/multihead_cross_attention.py b/easy_rec/python/layers/multihead_cross_attention.py index 6b62ce449..a616ab346 100644 --- a/easy_rec/python/layers/multihead_cross_attention.py +++ b/easy_rec/python/layers/multihead_cross_attention.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, division, print_function import math + import tensorflow as tf from easy_rec.python.compat.layers import layer_norm as tf_layer_norm diff --git a/easy_rec/python/layers/seq_input_layer.py b/easy_rec/python/layers/seq_input_layer.py index 140278876..fd5d92519 100644 --- a/easy_rec/python/layers/seq_input_layer.py +++ b/easy_rec/python/layers/seq_input_layer.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from tensorflow.python.framework import ops from tensorflow.python.ops import variable_scope diff --git a/easy_rec/python/layers/sequence_feature_layer.py b/easy_rec/python/layers/sequence_feature_layer.py index 48e6a94b1..b40d7bc08 100644 --- a/easy_rec/python/layers/sequence_feature_layer.py +++ b/easy_rec/python/layers/sequence_feature_layer.py @@ -1,5 +1,6 @@ import logging import os + import tensorflow as tf from tensorflow.python.framework import ops diff --git a/easy_rec/python/layers/utils.py b/easy_rec/python/layers/utils.py index df7a52e5e..c2f05fa61 100644 --- a/easy_rec/python/layers/utils.py +++ b/easy_rec/python/layers/utils.py @@ -16,6 +16,7 @@ from __future__ import absolute_import, division, print_function import json + from google.protobuf import struct_pb2 from google.protobuf.descriptor import FieldDescriptor from tensorflow.python.framework import ops, sparse_tensor diff --git a/easy_rec/python/layers/variational_dropout_layer.py b/easy_rec/python/layers/variational_dropout_layer.py index 04312bb7d..bd11828f6 100644 --- a/easy_rec/python/layers/variational_dropout_layer.py +++ b/easy_rec/python/layers/variational_dropout_layer.py @@ -1,6 +1,7 @@ # -*- encoding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import json + import numpy as np import tensorflow as tf diff --git a/easy_rec/python/loss/focal_loss.py b/easy_rec/python/loss/focal_loss.py index 29d15ec91..5037380f5 100644 --- a/easy_rec/python/loss/focal_loss.py +++ b/easy_rec/python/loss/focal_loss.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf if tf.__version__ >= '2.0': diff --git a/easy_rec/python/loss/jrc_loss.py b/easy_rec/python/loss/jrc_loss.py index 69c335ef0..368cc319a 100644 --- a/easy_rec/python/loss/jrc_loss.py +++ b/easy_rec/python/loss/jrc_loss.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import numpy as np import tensorflow as tf diff --git a/easy_rec/python/loss/listwise_loss.py b/easy_rec/python/loss/listwise_loss.py index 5f70a6051..3120e9d2e 100644 --- a/easy_rec/python/loss/listwise_loss.py +++ b/easy_rec/python/loss/listwise_loss.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.utils.load_class import load_by_path diff --git a/easy_rec/python/loss/pairwise_loss.py b/easy_rec/python/loss/pairwise_loss.py index f9a674f19..ee3f4a0e3 100644 --- a/easy_rec/python/loss/pairwise_loss.py +++ b/easy_rec/python/loss/pairwise_loss.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from tensorflow.python.ops.losses.losses_impl import compute_weighted_loss diff --git a/easy_rec/python/loss/zero_inflated_lognormal.py b/easy_rec/python/loss/zero_inflated_lognormal.py index 8a4da34e1..c4897cf07 100644 --- a/easy_rec/python/loss/zero_inflated_lognormal.py +++ b/easy_rec/python/loss/zero_inflated_lognormal.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Zero-inflated lognormal loss for lifetime value prediction.""" import logging + import tensorflow as tf import tensorflow_probability as tfp diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index 442c01324..8edebcc01 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -7,9 +7,10 @@ import logging import math import os +import time + import six import tensorflow as tf -import time from tensorflow.core.protobuf import saved_model_pb2 import easy_rec @@ -19,6 +20,7 @@ from easy_rec.python.model.easy_rec_estimator import EasyRecEstimator from easy_rec.python.model.easy_rec_model import EasyRecModel from easy_rec.python.protos.train_pb2 import DistributionStrategy + from easy_rec.python.utils import config_util, constant, estimator_utils, fg_util, load_class # NOQA from easy_rec.python.utils.config_util import get_eval_input_path, get_model_dir_path, get_train_input_path, set_eval_input_path # NOQA from easy_rec.python.utils.export_big_model import export_big_model, export_big_model_to_oss # NOQA diff --git a/easy_rec/python/model/autoint.py b/easy_rec/python/model/autoint.py index bab8565d5..958eb760e 100644 --- a/easy_rec/python/model/autoint.py +++ b/easy_rec/python/model/autoint.py @@ -1,10 +1,12 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.layers import multihead_attention from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.autoint_pb2 import AutoInt as AutoIntConfig # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/model/cmbf.py b/easy_rec/python/model/cmbf.py index 32bf596e5..2dcd18ac1 100644 --- a/easy_rec/python/model/cmbf.py +++ b/easy_rec/python/model/cmbf.py @@ -4,6 +4,7 @@ from easy_rec.python.layers import cmbf, dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.cmbf_pb2 import CMBF as CMBFConfig # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/model/collaborative_metric_learning.py b/easy_rec/python/model/collaborative_metric_learning.py index dde65a615..2015f01d9 100644 --- a/easy_rec/python/model/collaborative_metric_learning.py +++ b/easy_rec/python/model/collaborative_metric_learning.py @@ -1,16 +1,17 @@ import tensorflow as tf -from easy_rec.python.core.metrics import metric_learning_average_precision_at_k, metric_learning_recall_at_k # NOQA from easy_rec.python.layers import dnn from easy_rec.python.layers.common_layers import highway from easy_rec.python.loss.circle_loss import circle_loss from easy_rec.python.loss.multi_similarity import ms_loss from easy_rec.python.model.easy_rec_model import EasyRecModel -from easy_rec.python.protos.collaborative_metric_learning_pb2 import CoMetricLearningI2I as MetricLearningI2IConfig # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.utils.activation import gelu from easy_rec.python.utils.proto_util import copy_obj +from easy_rec.python.core.metrics import metric_learning_average_precision_at_k, metric_learning_recall_at_k # NOQA +from easy_rec.python.protos.collaborative_metric_learning_pb2 import CoMetricLearningI2I as MetricLearningI2IConfig # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/model/dcn.py b/easy_rec/python/model/dcn.py index d172b5c4c..25b0c5b09 100644 --- a/easy_rec/python/model/dcn.py +++ b/easy_rec/python/model/dcn.py @@ -5,6 +5,7 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.dcn_pb2 import DCN as DCNConfig # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/model/deepfm.py b/easy_rec/python/model/deepfm.py index 1d38ad0be..8d0693475 100644 --- a/easy_rec/python/model/deepfm.py +++ b/easy_rec/python/model/deepfm.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.layers import dnn, fm diff --git a/easy_rec/python/model/dlrm.py b/easy_rec/python/model/dlrm.py index 548e7bea1..00ad46287 100755 --- a/easy_rec/python/model/dlrm.py +++ b/easy_rec/python/model/dlrm.py @@ -1,10 +1,12 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.dlrm_pb2 import DLRM as DLRMConfig # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/model/dropoutnet.py b/easy_rec/python/model/dropoutnet.py index 3d95cac66..98b4437c2 100644 --- a/easy_rec/python/model/dropoutnet.py +++ b/easy_rec/python/model/dropoutnet.py @@ -4,12 +4,13 @@ from easy_rec.python.layers import dnn from easy_rec.python.loss.pairwise_loss import pairwise_loss -from easy_rec.python.loss.softmax_loss_with_negative_mining import softmax_loss_with_negative_mining # NOQA from easy_rec.python.model.easy_rec_model import EasyRecModel -from easy_rec.python.protos.dropoutnet_pb2 import DropoutNet as DropoutNetConfig # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.utils.proto_util import copy_obj +from easy_rec.python.loss.softmax_loss_with_negative_mining import softmax_loss_with_negative_mining # NOQA +from easy_rec.python.protos.dropoutnet_pb2 import DropoutNet as DropoutNetConfig # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses diff --git a/easy_rec/python/model/dssm_senet.py b/easy_rec/python/model/dssm_senet.py index cef6f709e..626b4f02c 100644 --- a/easy_rec/python/model/dssm_senet.py +++ b/easy_rec/python/model/dssm_senet.py @@ -5,11 +5,12 @@ from easy_rec.python.layers import dnn, senet from easy_rec.python.model.dssm import DSSM from easy_rec.python.model.match_model import MatchModel -from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.protos.simi_pb2 import Similarity from easy_rec.python.utils.proto_util import copy_obj +from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses diff --git a/easy_rec/python/model/easy_rec_estimator.py b/easy_rec/python/model/easy_rec_estimator.py index 16e6e6e0a..a9d20d137 100644 --- a/easy_rec/python/model/easy_rec_estimator.py +++ b/easy_rec/python/model/easy_rec_estimator.py @@ -6,9 +6,10 @@ import logging import os import re -import tensorflow as tf import time from collections import OrderedDict + +import tensorflow as tf from tensorflow.python.client import session as tf_session from tensorflow.python.eager import context from tensorflow.python.framework import ops @@ -19,16 +20,17 @@ from easy_rec.python.builders import optimizer_builder from easy_rec.python.compat import optimizers, sync_replicas_optimizer -from easy_rec.python.compat.early_stopping import custom_early_stop_hook, deadline_stop_hook, find_early_stop_var, oss_stop_hook, stop_if_no_decrease_hook, stop_if_no_increase_hook # NOQA -from easy_rec.python.compat.embedding_parallel_saver import EmbeddingParallelSaver # NOQA from easy_rec.python.compat.ops import GraphKeys from easy_rec.python.input.input import Input from easy_rec.python.layers.utils import _tensor_to_tensorinfo from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.utils import constant, embedding_utils, estimator_utils, hvd_utils, pai_util # NOQA from easy_rec.python.utils.multi_optimizer import MultiOptimizer +from easy_rec.python.compat.early_stopping import custom_early_stop_hook, deadline_stop_hook, find_early_stop_var, oss_stop_hook, stop_if_no_decrease_hook, stop_if_no_increase_hook # NOQA +from easy_rec.python.compat.embedding_parallel_saver import EmbeddingParallelSaver # NOQA +from easy_rec.python.utils import constant, embedding_utils, estimator_utils, hvd_utils, pai_util # NOQA + try: import horovod.tensorflow as hvd except Exception: diff --git a/easy_rec/python/model/easy_rec_model.py b/easy_rec/python/model/easy_rec_model.py index b98fbea46..df48e9371 100644 --- a/easy_rec/python/model/easy_rec_model.py +++ b/easy_rec/python/model/easy_rec_model.py @@ -4,9 +4,10 @@ import logging import os import re +from abc import abstractmethod + import six import tensorflow as tf -from abc import abstractmethod from tensorflow.python.framework import ops, tensor_shape from tensorflow.python.ops import variables from tensorflow.python.platform import gfile diff --git a/easy_rec/python/model/esmm.py b/easy_rec/python/model/esmm.py index adb2d884f..262b08b48 100644 --- a/easy_rec/python/model/esmm.py +++ b/easy_rec/python/model/esmm.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.layers import dnn diff --git a/easy_rec/python/model/match_model.py b/easy_rec/python/model/match_model.py index 7045fac5b..3b8bd9ebe 100644 --- a/easy_rec/python/model/match_model.py +++ b/easy_rec/python/model/match_model.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os + import tensorflow as tf from easy_rec.python.builders import loss_builder diff --git a/easy_rec/python/model/mind.py b/easy_rec/python/model/mind.py index 943f6f620..b86babb6d 100644 --- a/easy_rec/python/model/mind.py +++ b/easy_rec/python/model/mind.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.compat import regularizers diff --git a/easy_rec/python/model/multi_task_model.py b/easy_rec/python/model/multi_task_model.py index 83c1be50d..07a54a6ba 100644 --- a/easy_rec/python/model/multi_task_model.py +++ b/easy_rec/python/model/multi_task_model.py @@ -1,9 +1,10 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging -import tensorflow as tf from collections import OrderedDict +import tensorflow as tf + from easy_rec.python.builders import loss_builder from easy_rec.python.layers.dnn import DNN from easy_rec.python.model.rank_model import RankModel diff --git a/easy_rec/python/model/multi_tower.py b/easy_rec/python/model/multi_tower.py index 702a74374..aa4c7ecd0 100644 --- a/easy_rec/python/model/multi_tower.py +++ b/easy_rec/python/model/multi_tower.py @@ -5,6 +5,7 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/model/multi_tower_bst.py b/easy_rec/python/model/multi_tower_bst.py index 7e28c0c4f..67f620341 100644 --- a/easy_rec/python/model/multi_tower_bst.py +++ b/easy_rec/python/model/multi_tower_bst.py @@ -2,11 +2,13 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import math + import tensorflow as tf from easy_rec.python.compat import regularizers from easy_rec.python.layers import dnn, layer_norm, seq_input_layer from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/model/multi_tower_din.py b/easy_rec/python/model/multi_tower_din.py index 6e50b0f51..9736caf93 100644 --- a/easy_rec/python/model/multi_tower_din.py +++ b/easy_rec/python/model/multi_tower_din.py @@ -1,11 +1,13 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.compat import regularizers from easy_rec.python.layers import dnn, seq_input_layer from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/model/multi_tower_recall.py b/easy_rec/python/model/multi_tower_recall.py index 45de92329..bc12b4fc3 100644 --- a/easy_rec/python/model/multi_tower_recall.py +++ b/easy_rec/python/model/multi_tower_recall.py @@ -5,6 +5,7 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.multi_tower_recall_pb2 import MultiTowerRecall as MultiTowerRecallConfig # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/model/rank_model.py b/easy_rec/python/model/rank_model.py index ee01b533e..a5194ba5b 100644 --- a/easy_rec/python/model/rank_model.py +++ b/easy_rec/python/model/rank_model.py @@ -1,14 +1,16 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from tensorflow.python.ops import math_ops from easy_rec.python.builders import loss_builder -from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_pred # NOQA from easy_rec.python.model.easy_rec_model import EasyRecModel from easy_rec.python.protos.loss_pb2 import LossType +from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_pred # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/model/rocket_launching.py b/easy_rec/python/model/rocket_launching.py index 970a5e6d6..62f7cff46 100755 --- a/easy_rec/python/model/rocket_launching.py +++ b/easy_rec/python/model/rocket_launching.py @@ -6,9 +6,10 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel from easy_rec.python.protos.loss_pb2 import LossType -from easy_rec.python.protos.rocket_launching_pb2 import RocketLaunching as RocketLaunchingConfig # NOQA from easy_rec.python.protos.simi_pb2 import Similarity +from easy_rec.python.protos.rocket_launching_pb2 import RocketLaunching as RocketLaunchingConfig # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/model/simple_multi_task.py b/easy_rec/python/model/simple_multi_task.py index ec2207e48..a02d294f4 100644 --- a/easy_rec/python/model/simple_multi_task.py +++ b/easy_rec/python/model/simple_multi_task.py @@ -4,6 +4,7 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.multi_task_model import MultiTaskModel + from easy_rec.python.protos.simple_multi_task_pb2 import SimpleMultiTask as SimpleMultiTaskConfig # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/model/uniter.py b/easy_rec/python/model/uniter.py index 3aaf72e66..a38179e22 100644 --- a/easy_rec/python/model/uniter.py +++ b/easy_rec/python/model/uniter.py @@ -4,6 +4,7 @@ from easy_rec.python.layers import dnn, uniter from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.uniter_pb2 import Uniter as UNITERConfig # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/model/wide_and_deep.py b/easy_rec/python/model/wide_and_deep.py index bd826416d..40f8d1e00 100755 --- a/easy_rec/python/model/wide_and_deep.py +++ b/easy_rec/python/model/wide_and_deep.py @@ -1,10 +1,12 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.wide_and_deep_pb2 import WideAndDeep as WideAndDeepConfig # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/ops/gen_kafka_ops.py b/easy_rec/python/ops/gen_kafka_ops.py index bcf99608f..bf2618e5a 100644 --- a/easy_rec/python/ops/gen_kafka_ops.py +++ b/easy_rec/python/ops/gen_kafka_ops.py @@ -6,9 +6,10 @@ import logging import os +import traceback + import six as _six import tensorflow as tf -import traceback from tensorflow.python import pywrap_tensorflow as _pywrap_tensorflow from tensorflow.python.eager import context as _context from tensorflow.python.eager import core as _core diff --git a/easy_rec/python/ops/gen_str_avx_op.py b/easy_rec/python/ops/gen_str_avx_op.py index a42293e01..2710bae02 100644 --- a/easy_rec/python/ops/gen_str_avx_op.py +++ b/easy_rec/python/ops/gen_str_avx_op.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os + import tensorflow as tf from tensorflow.python.ops import string_ops diff --git a/easy_rec/python/ops/incr_record.py b/easy_rec/python/ops/incr_record.py index cc0c49e30..dd62a0a06 100644 --- a/easy_rec/python/ops/incr_record.py +++ b/easy_rec/python/ops/incr_record.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import os + import tensorflow as tf import easy_rec diff --git a/easy_rec/python/predict.py b/easy_rec/python/predict.py index 2a21c68ea..ae60e8c99 100644 --- a/easy_rec/python/predict.py +++ b/easy_rec/python/predict.py @@ -4,11 +4,11 @@ import json import logging import os + import tensorflow as tf from tensorflow.python.lib.io import file_io from easy_rec.python.inference.csv_predictor import CSVPredictor -from easy_rec.python.inference.hive_parquet_predictor import HiveParquetPredictor # NOQA from easy_rec.python.inference.hive_predictor import HivePredictor from easy_rec.python.inference.parquet_predictor import ParquetPredictor from easy_rec.python.inference.parquet_predictor_v2 import ParquetPredictorV2 @@ -17,6 +17,8 @@ from easy_rec.python.utils import config_util, numpy_utils from easy_rec.python.utils.hive_utils import HiveUtils +from easy_rec.python.inference.hive_parquet_predictor import HiveParquetPredictor # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/test/csv_input_test.py b/easy_rec/python/test/csv_input_test.py index 68048cce6..fc57b7331 100644 --- a/easy_rec/python/test/csv_input_test.py +++ b/easy_rec/python/test/csv_input_test.py @@ -3,8 +3,9 @@ """Define cv_input, the base class for cv tasks.""" import os -import tensorflow as tf import unittest + +import tensorflow as tf from google.protobuf import text_format from easy_rec.python.input.csv_input import CSVInput diff --git a/easy_rec/python/test/dh_local_run.py b/easy_rec/python/test/dh_local_run.py index 28f2b7a51..14ead43d6 100644 --- a/easy_rec/python/test/dh_local_run.py +++ b/easy_rec/python/test/dh_local_run.py @@ -3,13 +3,15 @@ import os import shutil import sys + import tensorflow as tf from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA from easy_rec.python.utils import test_utils +from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA + logging.basicConfig( level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' ) diff --git a/easy_rec/python/test/embed_test.py b/easy_rec/python/test/embed_test.py index 3d2990006..61b429bbc 100644 --- a/easy_rec/python/test/embed_test.py +++ b/easy_rec/python/test/embed_test.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import numpy as np import tensorflow as tf from google.protobuf import text_format diff --git a/easy_rec/python/test/emr_run.py b/easy_rec/python/test/emr_run.py index ab3bb5a24..f258d63f4 100644 --- a/easy_rec/python/test/emr_run.py +++ b/easy_rec/python/test/emr_run.py @@ -6,13 +6,15 @@ import os import shutil import sys + import tensorflow as tf from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA from easy_rec.python.utils import test_utils +from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA + logging.basicConfig( level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' ) diff --git a/easy_rec/python/test/eval_metric_test.py b/easy_rec/python/test/eval_metric_test.py index 86fda46e2..cb310721b 100644 --- a/easy_rec/python/test/eval_metric_test.py +++ b/easy_rec/python/test/eval_metric_test.py @@ -3,6 +3,7 @@ from __future__ import division import logging + import tensorflow as tf from absl.testing import parameterized diff --git a/easy_rec/python/test/excel_convert_test.py b/easy_rec/python/test/excel_convert_test.py index 6f5f7d515..35f4ff84c 100644 --- a/easy_rec/python/test/excel_convert_test.py +++ b/easy_rec/python/test/excel_convert_test.py @@ -3,6 +3,7 @@ import logging import os + import tensorflow as tf from easy_rec.python.utils import test_utils diff --git a/easy_rec/python/test/export_test.py b/easy_rec/python/test/export_test.py index b41409b80..3c94b5ef3 100644 --- a/easy_rec/python/test/export_test.py +++ b/easy_rec/python/test/export_test.py @@ -5,10 +5,11 @@ import functools import json import logging -import numpy as np import os -import tensorflow as tf import unittest + +import numpy as np +import tensorflow as tf from tensorflow.python.platform import gfile import easy_rec diff --git a/easy_rec/python/test/fg_test.py b/easy_rec/python/test/fg_test.py index 65981500e..e29e38d4e 100644 --- a/easy_rec/python/test/fg_test.py +++ b/easy_rec/python/test/fg_test.py @@ -1,8 +1,9 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import logging -import tensorflow as tf import unittest + +import tensorflow as tf from google.protobuf import text_format from easy_rec.python.utils import config_util, fg_util, test_utils diff --git a/easy_rec/python/test/hive_input_test.py b/easy_rec/python/test/hive_input_test.py index 993b7fb98..8140820fd 100644 --- a/easy_rec/python/test/hive_input_test.py +++ b/easy_rec/python/test/hive_input_test.py @@ -3,8 +3,9 @@ """Define cv_input, the base class for cv tasks.""" import logging import os -import tensorflow as tf import unittest + +import tensorflow as tf from google.protobuf import text_format from easy_rec.python.input.hive_input import HiveInput diff --git a/easy_rec/python/test/hpo_test.py b/easy_rec/python/test/hpo_test.py index e2a2d52c4..19918a691 100644 --- a/easy_rec/python/test/hpo_test.py +++ b/easy_rec/python/test/hpo_test.py @@ -3,11 +3,12 @@ import json import logging -import numpy as np import os -import tensorflow as tf import time +import numpy as np +import tensorflow as tf + from easy_rec.python.protos.feature_config_pb2 import FeatureConfig from easy_rec.python.utils import config_util, hpo_util, test_utils diff --git a/easy_rec/python/test/kafka_test.py b/easy_rec/python/test/kafka_test.py index 00a5b00a9..630517e01 100644 --- a/easy_rec/python/test/kafka_test.py +++ b/easy_rec/python/test/kafka_test.py @@ -3,14 +3,15 @@ import json import logging -import numpy as np import os -import six -import tensorflow as tf import threading import time import traceback import unittest + +import numpy as np +import six +import tensorflow as tf from tensorflow.python.data.ops import iterator_ops from tensorflow.python.platform import gfile diff --git a/easy_rec/python/test/local_incr_test.py b/easy_rec/python/test/local_incr_test.py index c505e1cde..b4ffdfe4f 100644 --- a/easy_rec/python/test/local_incr_test.py +++ b/easy_rec/python/test/local_incr_test.py @@ -3,10 +3,11 @@ import json import logging -import numpy as np import os -import tensorflow as tf import unittest + +import numpy as np +import tensorflow as tf from tensorflow.python.platform import gfile from easy_rec.python.inference.predictor import Predictor diff --git a/easy_rec/python/test/odps_local_run.py b/easy_rec/python/test/odps_local_run.py index 1d8b9befc..d1804254d 100644 --- a/easy_rec/python/test/odps_local_run.py +++ b/easy_rec/python/test/odps_local_run.py @@ -6,13 +6,15 @@ import os import shutil import sys + import tensorflow as tf from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA from easy_rec.python.utils import test_utils +from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA + logging.basicConfig( level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' ) diff --git a/easy_rec/python/test/odps_run.py b/easy_rec/python/test/odps_run.py index 96b64d9dd..1af392943 100644 --- a/easy_rec/python/test/odps_run.py +++ b/easy_rec/python/test/odps_run.py @@ -4,16 +4,18 @@ import argparse import logging import os -import oss2 import shutil import sys + +import oss2 import tensorflow as tf from easy_rec.python.test.odps_test_cls import OdpsTest from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA from easy_rec.python.utils import config_util +from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA + logging.basicConfig( level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' ) diff --git a/easy_rec/python/test/odps_test_prepare.py b/easy_rec/python/test/odps_test_prepare.py index 60e6c3493..a52ee0983 100644 --- a/easy_rec/python/test/odps_test_prepare.py +++ b/easy_rec/python/test/odps_test_prepare.py @@ -4,10 +4,11 @@ import glob import logging import os -import oss2 import shutil import sys +import oss2 + from easy_rec.python.test.odps_test_util import OdpsOSSConfig, get_oss_bucket diff --git a/easy_rec/python/test/odps_test_util.py b/easy_rec/python/test/odps_test_util.py index 54946edbf..aac82167e 100644 --- a/easy_rec/python/test/odps_test_util.py +++ b/easy_rec/python/test/odps_test_util.py @@ -3,10 +3,11 @@ import logging import os -import oss2 import time import traceback +import oss2 + try: from datahub import DataHub from datahub.exceptions import InvalidOperationException, ResourceExistException # NOQA diff --git a/easy_rec/python/test/pre_check_test.py b/easy_rec/python/test/pre_check_test.py index 5a63f0948..3d9e56bfb 100644 --- a/easy_rec/python/test/pre_check_test.py +++ b/easy_rec/python/test/pre_check_test.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging + import tensorflow as tf from easy_rec.python.utils import test_utils diff --git a/easy_rec/python/test/predictor_test.py b/easy_rec/python/test/predictor_test.py index e68ed4a59..66710569a 100644 --- a/easy_rec/python/test/predictor_test.py +++ b/easy_rec/python/test/predictor_test.py @@ -3,9 +3,10 @@ import csv import json import logging -import numpy as np import os import shutil + +import numpy as np import tensorflow as tf from easy_rec.python.inference.csv_predictor import CSVPredictor diff --git a/easy_rec/python/test/rtp_convert_test.py b/easy_rec/python/test/rtp_convert_test.py index 718300355..f368181ce 100644 --- a/easy_rec/python/test/rtp_convert_test.py +++ b/easy_rec/python/test/rtp_convert_test.py @@ -3,6 +3,7 @@ import logging import os + import tensorflow as tf from easy_rec.python.utils import config_util, test_utils diff --git a/easy_rec/python/test/run.py b/easy_rec/python/test/run.py index 6ea41714a..e201db9cf 100644 --- a/easy_rec/python/test/run.py +++ b/easy_rec/python/test/run.py @@ -5,9 +5,10 @@ import logging import os import sys -import tensorflow as tf import unittest +import tensorflow as tf + from easy_rec.python.utils import test_utils if tf.__version__ >= '2.0': diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index f7bc77acc..6641673d0 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -3,17 +3,19 @@ import glob import logging -import numpy as np import os -import six -import tensorflow as tf import threading import time import unittest + +import numpy as np +import six +import tensorflow as tf from distutils.version import LooseVersion from tensorflow.python.platform import gfile from easy_rec.python.main import predict + from easy_rec.python.utils import config_util, constant, estimator_utils, test_utils # NOQA try: diff --git a/easy_rec/python/tools/add_boundaries_to_config.py b/easy_rec/python/tools/add_boundaries_to_config.py index 7e34300d9..c173ace82 100644 --- a/easy_rec/python/tools/add_boundaries_to_config.py +++ b/easy_rec/python/tools/add_boundaries_to_config.py @@ -1,10 +1,11 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -import common_io import json import logging import os import sys + +import common_io import tensorflow as tf from easy_rec.python.utils import config_util, io_util diff --git a/easy_rec/python/tools/add_feature_info_to_config.py b/easy_rec/python/tools/add_feature_info_to_config.py index d972da0c4..1c8c354b2 100644 --- a/easy_rec/python/tools/add_feature_info_to_config.py +++ b/easy_rec/python/tools/add_feature_info_to_config.py @@ -4,6 +4,7 @@ import logging import os import sys + import tensorflow as tf from easy_rec.python.utils import config_util, io_util diff --git a/easy_rec/python/tools/convert_config_format.py b/easy_rec/python/tools/convert_config_format.py index f8691c5df..a1b1906a5 100644 --- a/easy_rec/python/tools/convert_config_format.py +++ b/easy_rec/python/tools/convert_config_format.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import os + from google.protobuf import json_format, text_format from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig diff --git a/easy_rec/python/tools/convert_rtp_data.py b/easy_rec/python/tools/convert_rtp_data.py index a29f9752b..3edbcc94d 100644 --- a/easy_rec/python/tools/convert_rtp_data.py +++ b/easy_rec/python/tools/convert_rtp_data.py @@ -14,6 +14,7 @@ import json import logging import sys + import tensorflow as tf logging.basicConfig( diff --git a/easy_rec/python/tools/convert_rtp_fg.py b/easy_rec/python/tools/convert_rtp_fg.py index 91890f5f0..2315bd43b 100644 --- a/easy_rec/python/tools/convert_rtp_fg.py +++ b/easy_rec/python/tools/convert_rtp_fg.py @@ -4,6 +4,7 @@ import argparse import logging import sys + import tensorflow as tf from easy_rec.python.utils.config_util import save_message diff --git a/easy_rec/python/tools/create_config_from_excel.py b/easy_rec/python/tools/create_config_from_excel.py index b14d9c32a..201260615 100644 --- a/easy_rec/python/tools/create_config_from_excel.py +++ b/easy_rec/python/tools/create_config_from_excel.py @@ -2,9 +2,10 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import logging import math +import sys + import numpy as np import pandas as pd -import sys from easy_rec.python.utils import config_util diff --git a/easy_rec/python/tools/criteo/convert_data.py b/easy_rec/python/tools/criteo/convert_data.py index a85c7f582..7e1bc142e 100644 --- a/easy_rec/python/tools/criteo/convert_data.py +++ b/easy_rec/python/tools/criteo/convert_data.py @@ -4,11 +4,12 @@ import gzip import logging import multiprocessing -import numpy as np import os +import traceback + +import numpy as np import pandas as pd import six -import traceback from tensorflow.python.platform import gfile logging.basicConfig( diff --git a/easy_rec/python/tools/edit_lookup_graph.py b/easy_rec/python/tools/edit_lookup_graph.py index 1cf72144b..4ae0ad218 100644 --- a/easy_rec/python/tools/edit_lookup_graph.py +++ b/easy_rec/python/tools/edit_lookup_graph.py @@ -4,6 +4,7 @@ import logging import os import sys + import tensorflow as tf from tensorflow.core.protobuf import saved_model_pb2 from tensorflow.python.lib.io.file_io import file_exists, recursive_create_dir diff --git a/easy_rec/python/tools/faiss_index_pai.py b/easy_rec/python/tools/faiss_index_pai.py index 4adb1447b..d5415be6c 100644 --- a/easy_rec/python/tools/faiss_index_pai.py +++ b/easy_rec/python/tools/faiss_index_pai.py @@ -2,11 +2,12 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from __future__ import print_function -import faiss import logging -import numpy as np import os import sys + +import faiss +import numpy as np import tensorflow as tf from easy_rec.python.utils import io_util diff --git a/easy_rec/python/tools/feature_selection.py b/easy_rec/python/tools/feature_selection.py index a0478cd9d..ba7e72122 100644 --- a/easy_rec/python/tools/feature_selection.py +++ b/easy_rec/python/tools/feature_selection.py @@ -1,12 +1,13 @@ from __future__ import division, print_function import json -import numpy as np import os -import pandas as pd import sys -import tensorflow as tf from collections import OrderedDict + +import numpy as np +import pandas as pd +import tensorflow as tf from tensorflow.python.framework.meta_graph import read_meta_graph_file from easy_rec.python.utils import config_util, io_util diff --git a/easy_rec/python/tools/hit_rate_ds.py b/easy_rec/python/tools/hit_rate_ds.py index f8c4b9fa5..d04fba623 100644 --- a/easy_rec/python/tools/hit_rate_ds.py +++ b/easy_rec/python/tools/hit_rate_ds.py @@ -15,19 +15,21 @@ # """Evaluation of Top k hitrate.""" from __future__ import absolute_import, division, print_function -import graphlearn as gl import json import logging import os import sys + +import graphlearn as gl import tensorflow as tf from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils import config_util, io_util from easy_rec.python.utils.config_util import process_multi_file_input_path -from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch, load_graph, reduce_hitrate # NOQA from easy_rec.python.utils.hive_utils import HiveUtils +from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch, load_graph, reduce_hitrate # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/tools/hit_rate_pai.py b/easy_rec/python/tools/hit_rate_pai.py index 4a5e218a9..75e7697a5 100644 --- a/easy_rec/python/tools/hit_rate_pai.py +++ b/easy_rec/python/tools/hit_rate_pai.py @@ -16,9 +16,11 @@ from __future__ import absolute_import, division, print_function import sys + import tensorflow as tf from easy_rec.python.utils import io_util + from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch, load_graph, reduce_hitrate # NOQA flags = tf.app.flags diff --git a/easy_rec/python/tools/pre_check.py b/easy_rec/python/tools/pre_check.py index b3bb8fbd2..769b2769e 100644 --- a/easy_rec/python/tools/pre_check.py +++ b/easy_rec/python/tools/pre_check.py @@ -4,10 +4,12 @@ import logging import os import sys + import tensorflow as tf from easy_rec.python.input.input import Input from easy_rec.python.utils import config_util, fg_util, io_util + from easy_rec.python.utils.check_utils import check_env_and_input_path, check_sequence # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/tools/predict_and_chk.py b/easy_rec/python/tools/predict_and_chk.py index 407dff2d0..0247eddac 100644 --- a/easy_rec/python/tools/predict_and_chk.py +++ b/easy_rec/python/tools/predict_and_chk.py @@ -3,10 +3,11 @@ import argparse import json import logging -import numpy as np import os import sys +import numpy as np + import easy_rec from easy_rec.python.inference.predictor import Predictor diff --git a/easy_rec/python/tools/read_kafka.py b/easy_rec/python/tools/read_kafka.py index cedd0ed51..bc239d74b 100644 --- a/easy_rec/python/tools/read_kafka.py +++ b/easy_rec/python/tools/read_kafka.py @@ -4,6 +4,7 @@ import logging import os import sys + from kafka import KafkaConsumer from kafka.structs import TopicPartition diff --git a/easy_rec/python/tools/split_model_pai.py b/easy_rec/python/tools/split_model_pai.py index d5e325292..5bfe9c0a6 100644 --- a/easy_rec/python/tools/split_model_pai.py +++ b/easy_rec/python/tools/split_model_pai.py @@ -3,6 +3,7 @@ import logging import os import sys + import tensorflow as tf from tensorflow.core.framework import graph_pb2 from tensorflow.python.framework import importer, ops diff --git a/easy_rec/python/tools/split_pdn_model_pai.py b/easy_rec/python/tools/split_pdn_model_pai.py index 8d4cb312b..be61f637f 100644 --- a/easy_rec/python/tools/split_pdn_model_pai.py +++ b/easy_rec/python/tools/split_pdn_model_pai.py @@ -3,6 +3,7 @@ import logging import os import sys + import tensorflow as tf from tensorflow.core.framework import graph_pb2 from tensorflow.python.framework import importer, ops diff --git a/easy_rec/python/tools/test_saved_model.py b/easy_rec/python/tools/test_saved_model.py index bd3560ef7..6c0fa6024 100644 --- a/easy_rec/python/tools/test_saved_model.py +++ b/easy_rec/python/tools/test_saved_model.py @@ -3,8 +3,9 @@ import argparse import json import logging -import numpy as np import os + +import numpy as np import tensorflow as tf import easy_rec diff --git a/easy_rec/python/tools/view_saved_model.py b/easy_rec/python/tools/view_saved_model.py index 43cea4c6b..21435db66 100644 --- a/easy_rec/python/tools/view_saved_model.py +++ b/easy_rec/python/tools/view_saved_model.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import argparse import logging + from google.protobuf import text_format from tensorflow.core.protobuf import saved_model_pb2 from tensorflow.python.platform.gfile import GFile diff --git a/easy_rec/python/tools/write_kafka.py b/easy_rec/python/tools/write_kafka.py index 1c74d3b36..be94f2690 100644 --- a/easy_rec/python/tools/write_kafka.py +++ b/easy_rec/python/tools/write_kafka.py @@ -3,6 +3,7 @@ import argparse import logging import sys + # from kafka import KafkaConsumer from kafka import KafkaAdminClient, KafkaProducer from kafka.admin import NewTopic diff --git a/easy_rec/python/train_eval.py b/easy_rec/python/train_eval.py index fa24dce17..57f3c4ee9 100644 --- a/easy_rec/python/train_eval.py +++ b/easy_rec/python/train_eval.py @@ -4,10 +4,12 @@ import json import logging import os + import tensorflow as tf from easy_rec.python.main import _train_and_evaluate_impl from easy_rec.python.protos.train_pb2 import DistributionStrategy + from easy_rec.python.utils import config_util, ds_util, estimator_utils, fg_util, hpo_util # NOQA from easy_rec.python.utils.config_util import process_neg_sampler_data_path, set_eval_input_path, set_train_input_path # NOQA diff --git a/easy_rec/python/utils/config_util.py b/easy_rec/python/utils/config_util.py index 53171dbae..2e4536fa1 100644 --- a/easy_rec/python/utils/config_util.py +++ b/easy_rec/python/utils/config_util.py @@ -8,11 +8,12 @@ import datetime import json import logging -import numpy as np import os import re -import six import sys + +import numpy as np +import six import tensorflow as tf from google.protobuf import json_format, text_format from tensorflow.python.lib.io import file_io diff --git a/easy_rec/python/utils/convert_rtp_fg.py b/easy_rec/python/utils/convert_rtp_fg.py index cd96298d2..5792b1bbe 100644 --- a/easy_rec/python/utils/convert_rtp_fg.py +++ b/easy_rec/python/utils/convert_rtp_fg.py @@ -3,15 +3,17 @@ import json import logging import sys -import tensorflow as tf import traceback + +import tensorflow as tf from google.protobuf import text_format from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.protos.feature_config_pb2 import FeatureConfig, FeatureGroupConfig, WideOrDeep # NOQA from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig from easy_rec.python.utils import config_util +from easy_rec.python.protos.feature_config_pb2 import FeatureConfig, FeatureGroupConfig, WideOrDeep # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/utils/distribution_utils.py b/easy_rec/python/utils/distribution_utils.py index 8a8d35dac..f06e265a4 100644 --- a/easy_rec/python/utils/distribution_utils.py +++ b/easy_rec/python/utils/distribution_utils.py @@ -5,10 +5,12 @@ import json import logging import os + import tensorflow as tf from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import estimator_utils + from easy_rec.python.utils.estimator_utils import chief_to_master, master_to_chief # NOQA DistributionStrategyMap = { diff --git a/easy_rec/python/utils/ds_util.py b/easy_rec/python/utils/ds_util.py index b1b74f9c4..f2d051b21 100644 --- a/easy_rec/python/utils/ds_util.py +++ b/easy_rec/python/utils/ds_util.py @@ -4,6 +4,7 @@ import os import subprocess import traceback + from tensorflow.python.platform import gfile from easy_rec.python.utils import estimator_utils diff --git a/easy_rec/python/utils/embedding_utils.py b/easy_rec/python/utils/embedding_utils.py index 268737971..cb61dd177 100644 --- a/easy_rec/python/utils/embedding_utils.py +++ b/easy_rec/python/utils/embedding_utils.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import os + import tensorflow as tf from tensorflow.python.framework import ops diff --git a/easy_rec/python/utils/estimator_utils.py b/easy_rec/python/utils/estimator_utils.py index c894cb006..a89b9ad90 100644 --- a/easy_rec/python/utils/estimator_utils.py +++ b/easy_rec/python/utils/estimator_utils.py @@ -4,24 +4,27 @@ import json import logging -import numpy as np import os import re -import six import sys -import tensorflow as tf import time + +import numpy as np +import six +import tensorflow as tf from tensorflow.core.framework.summary_pb2 import Summary from tensorflow.python.client import device_lib from tensorflow.python.framework import errors_impl, meta_graph, ops from tensorflow.python.ops import array_ops from tensorflow.python.platform import gfile +from tensorflow.python.training.summary_io import SummaryWriterCache + +from easy_rec.python.utils import constant, embedding_utils, shape_utils + from tensorflow.python.training import basic_session_run_hooks, session_run_hook # NOQA from tensorflow.python.training.basic_session_run_hooks import SecondOrStepTimer # NOQA -from tensorflow.python.training.summary_io import SummaryWriterCache from easy_rec.python.ops.incr_record import get_sparse_indices, kv_resource_incr_gather # NOQA -from easy_rec.python.utils import constant, embedding_utils, shape_utils try: import horovod.tensorflow as hvd diff --git a/easy_rec/python/utils/export_big_model.py b/easy_rec/python/utils/export_big_model.py index fc4bc5a29..9575ee40f 100644 --- a/easy_rec/python/utils/export_big_model.py +++ b/easy_rec/python/utils/export_big_model.py @@ -3,21 +3,24 @@ import json import logging -import numpy as np import os -import tensorflow as tf import time + +import numpy as np +import tensorflow as tf from google.protobuf import json_format from tensorflow.core.protobuf import config_pb2 from tensorflow.python.framework import ops from tensorflow.python.ops.variables import global_variables -from tensorflow.python.platform.gfile import DeleteRecursively, Exists, GFile, Remove # NOQA from tensorflow.python.saved_model import signature_constants from tensorflow.python.training.device_setter import replica_device_setter -from tensorflow.python.training.monitored_session import ChiefSessionCreator, Scaffold # NOQA from tensorflow.python.training.saver import export_meta_graph import easy_rec + +from tensorflow.python.platform.gfile import DeleteRecursively, Exists, GFile, Remove # NOQA +from tensorflow.python.training.monitored_session import ChiefSessionCreator, Scaffold # NOQA + from easy_rec.python.utils import constant, estimator_utils, io_util, proto_util # NOQA from easy_rec.python.utils.meta_graph_editor import EMBEDDING_INITIALIZERS, MetaGraphEditor # NOQA diff --git a/easy_rec/python/utils/fg_util.py b/easy_rec/python/utils/fg_util.py index fc5596ec5..dadfb4124 100644 --- a/easy_rec/python/utils/fg_util.py +++ b/easy_rec/python/utils/fg_util.py @@ -1,10 +1,12 @@ import json import logging + import tensorflow as tf from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.protos.feature_config_pb2 import FeatureConfig from easy_rec.python.utils.config_util import get_compatible_feature_configs + from easy_rec.python.utils.convert_rtp_fg import load_input_field_and_feature_config # NOQA if tf.__version__ >= '2.0': diff --git a/easy_rec/python/utils/hit_rate_utils.py b/easy_rec/python/utils/hit_rate_utils.py index d70fb8fd5..32347e40b 100644 --- a/easy_rec/python/utils/hit_rate_utils.py +++ b/easy_rec/python/utils/hit_rate_utils.py @@ -1,5 +1,6 @@ -import graphlearn as gl import logging + +import graphlearn as gl import numpy as np import tensorflow as tf diff --git a/easy_rec/python/utils/hpo_util.py b/easy_rec/python/utils/hpo_util.py index 76f10e63c..1592ccc0e 100644 --- a/easy_rec/python/utils/hpo_util.py +++ b/easy_rec/python/utils/hpo_util.py @@ -3,6 +3,7 @@ import json import logging import os + import psutil import tensorflow as tf from tensorflow.python.summary import summary_iterator diff --git a/easy_rec/python/utils/hvd_utils.py b/easy_rec/python/utils/hvd_utils.py index 89f828bea..14929bd73 100644 --- a/easy_rec/python/utils/hvd_utils.py +++ b/easy_rec/python/utils/hvd_utils.py @@ -1,5 +1,6 @@ # -*- encoding: utf-8 -*- import logging + import tensorflow as tf from tensorflow.python.framework import ops from tensorflow.python.training import session_run_hook diff --git a/easy_rec/python/utils/load_class.py b/easy_rec/python/utils/load_class.py index 133bbfd52..dd66e2240 100644 --- a/easy_rec/python/utils/load_class.py +++ b/easy_rec/python/utils/load_class.py @@ -7,11 +7,12 @@ import os import pkgutil import pydoc -import six -import tensorflow as tf import traceback from abc import ABCMeta +import six +import tensorflow as tf + import easy_rec from easy_rec.python.utils import compat diff --git a/easy_rec/python/utils/meta_graph_editor.py b/easy_rec/python/utils/meta_graph_editor.py index 0921d8a1e..bd12e9280 100644 --- a/easy_rec/python/utils/meta_graph_editor.py +++ b/easy_rec/python/utils/meta_graph_editor.py @@ -1,7 +1,8 @@ # -*- encoding:utf-8 -*- import logging -import numpy as np import os + +import numpy as np import tensorflow as tf from google.protobuf import text_format from tensorflow.python.framework import ops diff --git a/easy_rec/python/utils/numpy_utils.py b/easy_rec/python/utils/numpy_utils.py index 84b9d8469..cfda857d2 100644 --- a/easy_rec/python/utils/numpy_utils.py +++ b/easy_rec/python/utils/numpy_utils.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import json + import numpy as np diff --git a/easy_rec/python/utils/pai_util.py b/easy_rec/python/utils/pai_util.py index d1f44aa90..e59acabe8 100644 --- a/easy_rec/python/utils/pai_util.py +++ b/easy_rec/python/utils/pai_util.py @@ -4,9 +4,10 @@ import logging import os import sys -import tensorflow as tf import traceback +import tensorflow as tf + if sys.version_info.major == 2: from urllib2 import HTTPError, Request, urlopen else: diff --git a/easy_rec/python/utils/tf_utils.py b/easy_rec/python/utils/tf_utils.py index 8602cb0ce..c82495e7f 100644 --- a/easy_rec/python/utils/tf_utils.py +++ b/easy_rec/python/utils/tf_utils.py @@ -3,6 +3,7 @@ """Common functions used for odps input.""" import json import os + import tensorflow as tf from easy_rec.python.protos.dataset_pb2 import DatasetConfig diff --git a/examples/data/amazon_books_data/process_amazon.py b/examples/data/amazon_books_data/process_amazon.py index b59a88d57..4f6467f67 100644 --- a/examples/data/amazon_books_data/process_amazon.py +++ b/examples/data/amazon_books_data/process_amazon.py @@ -1,6 +1,7 @@ -import pandas as pd import random +import pandas as pd + print('Start reading data...') title = ['UserID', 'BookID', 'Time'] print('Reading train data...') diff --git a/examples/data/movielens_1m/process_ml_1m.py b/examples/data/movielens_1m/process_ml_1m.py index 2da8141ba..f10e4161e 100644 --- a/examples/data/movielens_1m/process_ml_1m.py +++ b/examples/data/movielens_1m/process_ml_1m.py @@ -1,5 +1,6 @@ -import pandas as pd import re + +import pandas as pd from sklearn.utils import shuffle diff --git a/pai_jobs/run.py b/pai_jobs/run.py index 1f9b91419..db6e162a7 100644 --- a/pai_jobs/run.py +++ b/pai_jobs/run.py @@ -5,8 +5,9 @@ import logging # use few threads to avoid oss error import os -import tensorflow as tf import time + +import tensorflow as tf import yaml from tensorflow.python.platform import gfile @@ -14,6 +15,7 @@ from easy_rec.python.inference.odps_predictor import ODPSPredictor from easy_rec.python.inference.vector_retrieve import VectorRetrieve from easy_rec.python.tools.pre_check import run_check + from easy_rec.python.utils import config_util, constant, estimator_utils, fg_util, hpo_util, pai_util # NOQA from easy_rec.python.utils.distribution_utils import DistributionStrategyMap, set_distribution_config # NOQA diff --git a/setup.py b/setup.py index d6aea8ca1..b9f841c00 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,9 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from setuptools import find_packages, setup - import codecs import os +from setuptools import find_packages, setup def readme(): From 4697f88a6bebc6e174041f11d7bee7d1e75dd66a Mon Sep 17 00:00:00 2001 From: yangxudong Date: Tue, 11 Nov 2025 15:04:51 +0800 Subject: [PATCH 48/51] upgrade zero inflated lognormal loss, support export structure path, upgrade pre-commit hooks --- .github/workflows/code_style.yml | 6 ------ .pre-commit-config.yaml | 6 +----- easy_rec/python/model/deepfm.py | 16 ++++++++++++---- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 2a4617cc7..506d766d9 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -16,12 +16,6 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} submodules: recursive - - name: Clean pre-commit - run: | - pre-commit --version - pre-commit clean || true - pre-commit gc || true - - name: RunCiTest id: run_ci_test env: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d88700aed..50f0a52c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,11 +6,7 @@ repos: additional_dependencies: [ 'flake8-docstrings==1.5.0' ] - - repo: https://github.com/asottile/seed-isort-config - rev: v2.2.0 - hooks: - - id: seed-isort-config - - repo: https://github.com/timothycrosley/isort + - repo: https://github.com/pycqa/isort rev: 4.3.21 hooks: - id: isort diff --git a/easy_rec/python/model/deepfm.py b/easy_rec/python/model/deepfm.py index 8d0693475..ffbef4a87 100644 --- a/easy_rec/python/model/deepfm.py +++ b/easy_rec/python/model/deepfm.py @@ -49,7 +49,7 @@ def build_input_layer(self, model_config, feature_configs): if not has_final: assert model_config.deepfm.wide_output_dim == model_config.num_class self._wide_output_dim = model_config.deepfm.wide_output_dim - if self._wide_output_dim != 1: + if self._wide_output_dim != self._num_class: logging.warning( 'wide_output_dim not equal to 1, it is not a standard model' ) @@ -57,9 +57,17 @@ def build_input_layer(self, model_config, feature_configs): def build_predict_graph(self): # Wide - wide_fea = tf.reduce_sum( - self._wide_features, axis=1, keepdims=True, name='wide_feature' - ) + if self._num_class > 1 and self._wide_output_dim == self._num_class: + wide_shape = tf.shape(self._wide_features) + new_shape = tf.stack( + [-1, wide_shape[1] // self._num_class, self._num_class] + ) + wide_fea = tf.reshape(self._wide_features, new_shape) + wide_fea = tf.reduce_sum(wide_fea, axis=1, name='wide_feature') + else: + wide_fea = tf.reduce_sum( + self._wide_features, axis=1, keepdims=True, name='wide_feature' + ) # FM fm_fea = fm.FM(name='fm_feature')(self._fm_features) From 19ffbdc9ae031508a45eeb189be16a4fb8327a93 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Tue, 11 Nov 2025 21:02:51 +0800 Subject: [PATCH 49/51] upgrade zero inflated lognormal loss, support export structure path, upgrade pre-commit hooks --- .github/workflows/code_style.yml | 2 +- .pre-commit-config.yaml | 29 +- docs/post_fix.py | 4 +- docs/source/_ext/post_process.py | 4 +- docs/source/component/backbone.md | 84 +- docs/source/conf.py | 39 +- docs/source/feature/data.md | 4 +- docs/source/models/user_define.md | 7 +- docs/source/train.md | 4 +- easy_rec/__init__.py | 9 +- .../python/builders/hyperparams_builder.py | 27 +- easy_rec/python/builders/loss_builder.py | 116 +- easy_rec/python/builders/optimizer_builder.py | 80 +- easy_rec/python/builders/strategy_builder.py | 3 +- easy_rec/python/compat/adam_s.py | 67 +- easy_rec/python/compat/array_ops.py | 38 +- easy_rec/python/compat/dynamic_variable.py | 95 +- easy_rec/python/compat/early_stopping.py | 124 +- easy_rec/python/compat/embedding_ops.py | 70 +- .../python/compat/embedding_parallel_saver.py | 141 +- easy_rec/python/compat/estimator_train.py | 46 +- easy_rec/python/compat/exporter.py | 112 +- .../compat/feature_column/feature_column.py | 1089 ++++------- .../feature_column/feature_column_v2.py | 1697 ++++++----------- .../feature_column/sequence_feature_column.py | 130 +- .../python/compat/feature_column/utils.py | 30 +- easy_rec/python/compat/layers.py | 47 +- easy_rec/python/compat/optimizers.py | 206 +- easy_rec/python/compat/queues.py | 76 +- easy_rec/python/compat/regularizers.py | 67 +- easy_rec/python/compat/sok_optimizer.py | 65 +- .../python/compat/sync_replicas_optimizer.py | 136 +- .../python/compat/weight_decay_optimizers.py | 58 +- .../distribute_metrics_impl_pai.py | 1126 ++++------- .../distribute_metrics_impl_tf.py | 1195 ++++-------- easy_rec/python/core/learning_schedules.py | 80 +- easy_rec/python/core/metrics.py | 126 +- easy_rec/python/core/sampler.py | 354 ++-- easy_rec/python/eval.py | 50 +- easy_rec/python/export.py | 69 +- .../python/feature_column/feature_column.py | 248 +-- .../python/feature_column/feature_group.py | 6 +- easy_rec/python/hpo/emr_hpo.py | 129 +- easy_rec/python/hpo/generate_hpo_sql.py | 49 +- easy_rec/python/hpo/pai_hpo.py | 172 +- .../python/inference/client/client_demo.py | 32 +- easy_rec/python/inference/csv_predictor.py | 45 +- .../inference/hive_parquet_predictor.py | 80 +- easy_rec/python/inference/hive_predictor.py | 69 +- easy_rec/python/inference/odps_predictor.py | 23 +- .../python/inference/parquet_predictor.py | 46 +- .../python/inference/parquet_predictor_v2.py | 46 +- easy_rec/python/inference/predictor.py | 183 +- easy_rec/python/inference/processor/test.py | 81 +- easy_rec/python/inference/vector_retrieve.py | 37 +- easy_rec/python/input/batch_tfrecord_input.py | 75 +- easy_rec/python/input/criteo_binary_reader.py | 113 +- easy_rec/python/input/criteo_input.py | 53 +- easy_rec/python/input/csv_input.py | 93 +- easy_rec/python/input/csv_input_ex.py | 48 +- easy_rec/python/input/csv_input_v2.py | 31 +- easy_rec/python/input/datahub_input.py | 137 +- easy_rec/python/input/dummy_input.py | 15 +- easy_rec/python/input/hive_input.py | 65 +- easy_rec/python/input/hive_parquet_input.py | 64 +- easy_rec/python/input/hive_rtp_input.py | 112 +- easy_rec/python/input/input.py | 613 +++--- easy_rec/python/input/kafka_dataset.py | 53 +- easy_rec/python/input/kafka_input.py | 84 +- easy_rec/python/input/load_parquet.py | 117 +- easy_rec/python/input/odps_input.py | 36 +- easy_rec/python/input/odps_input_v2.py | 54 +- easy_rec/python/input/odps_input_v3.py | 49 +- easy_rec/python/input/odps_rtp_input.py | 148 +- easy_rec/python/input/odps_rtp_input_v2.py | 19 +- easy_rec/python/input/parquet_input.py | 142 +- easy_rec/python/input/parquet_input_v2.py | 43 +- easy_rec/python/input/parquet_input_v3.py | 67 +- easy_rec/python/input/rtp_input.py | 142 +- easy_rec/python/input/rtp_input_v2.py | 68 +- easy_rec/python/input/tfrecord_input.py | 51 +- easy_rec/python/layers/backbone.py | 68 +- easy_rec/python/layers/capsule_layer.py | 72 +- easy_rec/python/layers/cmbf.py | 215 +-- easy_rec/python/layers/common_layers.py | 46 +- easy_rec/python/layers/dnn.py | 26 +- easy_rec/python/layers/embed_input_layer.py | 11 +- easy_rec/python/layers/fm.py | 1 - easy_rec/python/layers/input_layer.py | 157 +- easy_rec/python/layers/keras/__init__.py | 15 +- easy_rec/python/layers/keras/activation.py | 18 +- easy_rec/python/layers/keras/attention.py | 34 +- .../python/layers/keras/auxiliary_loss.py | 9 +- easy_rec/python/layers/keras/blocks.py | 75 +- easy_rec/python/layers/keras/bst.py | 42 +- easy_rec/python/layers/keras/custom_ops.py | 70 +- easy_rec/python/layers/keras/data_augment.py | 36 +- easy_rec/python/layers/keras/din.py | 17 +- easy_rec/python/layers/keras/einsum_dense.py | 128 +- easy_rec/python/layers/keras/embedding.py | 10 +- easy_rec/python/layers/keras/fibinet.py | 62 +- easy_rec/python/layers/keras/interaction.py | 94 +- easy_rec/python/layers/keras/layer_norm.py | 31 +- easy_rec/python/layers/keras/mask_net.py | 28 +- .../layers/keras/multi_head_attention.py | 141 +- easy_rec/python/layers/keras/multi_task.py | 11 +- .../layers/keras/numerical_embedding.py | 78 +- easy_rec/python/layers/keras/ppnet.py | 69 +- easy_rec/python/layers/keras/transformer.py | 42 +- easy_rec/python/layers/layer_norm.py | 10 +- easy_rec/python/layers/mmoe.py | 16 +- easy_rec/python/layers/multihead_attention.py | 29 +- .../layers/multihead_cross_attention.py | 146 +- easy_rec/python/layers/senet.py | 17 +- easy_rec/python/layers/seq_input_layer.py | 58 +- .../python/layers/sequence_feature_layer.py | 112 +- easy_rec/python/layers/uniter.py | 134 +- easy_rec/python/layers/utils.py | 40 +- .../layers/variational_dropout_layer.py | 47 +- easy_rec/python/loss/circle_loss.py | 35 +- easy_rec/python/loss/contrastive_loss.py | 12 +- easy_rec/python/loss/f1_reweight_loss.py | 12 +- easy_rec/python/loss/focal_loss.py | 8 +- easy_rec/python/loss/jrc_loss.py | 42 +- easy_rec/python/loss/listwise_loss.py | 30 +- easy_rec/python/loss/multi_similarity.py | 18 +- easy_rec/python/loss/pairwise_loss.py | 109 +- .../loss/softmax_loss_with_negative_mining.py | 24 +- .../python/loss/zero_inflated_lognormal.py | 26 +- easy_rec/python/main.py | 323 ++-- easy_rec/python/model/autoint.py | 32 +- easy_rec/python/model/cmbf.py | 30 +- .../model/collaborative_metric_learning.py | 67 +- easy_rec/python/model/dat.py | 85 +- easy_rec/python/model/dbmtl.py | 49 +- easy_rec/python/model/dcn.py | 31 +- easy_rec/python/model/deepfm.py | 71 +- easy_rec/python/model/dlrm.py | 49 +- easy_rec/python/model/dropoutnet.py | 128 +- easy_rec/python/model/dssm.py | 99 +- easy_rec/python/model/dssm_senet.py | 75 +- easy_rec/python/model/dummy_model.py | 21 +- easy_rec/python/model/easy_rec_estimator.py | 301 +-- easy_rec/python/model/easy_rec_model.py | 129 +- easy_rec/python/model/esmm.py | 87 +- easy_rec/python/model/fm.py | 32 +- easy_rec/python/model/match_model.py | 198 +- easy_rec/python/model/mind.py | 257 +-- easy_rec/python/model/mmoe.py | 25 +- easy_rec/python/model/multi_task_model.py | 100 +- easy_rec/python/model/multi_tower.py | 26 +- easy_rec/python/model/multi_tower_bst.py | 119 +- easy_rec/python/model/multi_tower_din.py | 58 +- easy_rec/python/model/multi_tower_recall.py | 39 +- easy_rec/python/model/pdn.py | 79 +- easy_rec/python/model/ple.py | 56 +- easy_rec/python/model/rank_model.py | 281 ++- easy_rec/python/model/rocket_launching.py | 119 +- easy_rec/python/model/simple_multi_task.py | 22 +- easy_rec/python/model/uniter.py | 26 +- easy_rec/python/model/wide_and_deep.py | 65 +- easy_rec/python/ops/gen_kafka_ops.py | 71 +- easy_rec/python/ops/gen_str_avx_op.py | 9 +- easy_rec/python/ops/incr_record.py | 8 +- easy_rec/python/predict.py | 70 +- easy_rec/python/test/csv_input_test.py | 31 +- easy_rec/python/test/dh_local_run.py | 55 +- easy_rec/python/test/embed_test.py | 28 +- easy_rec/python/test/emr_run.py | 60 +- easy_rec/python/test/eval_metric_test.py | 41 +- easy_rec/python/test/excel_convert_test.py | 31 +- easy_rec/python/test/export_test.py | 219 +-- easy_rec/python/test/fg_test.py | 29 +- easy_rec/python/test/hive_input_test.py | 29 +- easy_rec/python/test/hpo_test.py | 99 +- easy_rec/python/test/kafka_test.py | 124 +- easy_rec/python/test/local_incr_test.py | 69 +- easy_rec/python/test/loss_test.py | 84 +- easy_rec/python/test/odps_command.py | 23 +- easy_rec/python/test/odps_local_run.py | 35 +- easy_rec/python/test/odps_run.py | 97 +- easy_rec/python/test/odps_test_prepare.py | 36 +- easy_rec/python/test/odps_test_util.py | 41 +- easy_rec/python/test/pre_check_test.py | 11 +- easy_rec/python/test/predictor_test.py | 201 +- easy_rec/python/test/rtp_convert_test.py | 69 +- easy_rec/python/test/run.py | 35 +- easy_rec/python/test/train_eval_test.py | 469 ++--- easy_rec/python/test/util_test.py | 41 +- .../test/zero_inflated_lognormal_test.py | 15 +- .../python/tools/add_boundaries_to_config.py | 20 +- .../tools/add_feature_info_to_config.py | 49 +- .../python/tools/convert_config_format.py | 14 +- easy_rec/python/tools/convert_rtp_data.py | 11 +- easy_rec/python/tools/convert_rtp_fg.py | 74 +- .../python/tools/create_config_from_excel.py | 110 +- easy_rec/python/tools/criteo/convert_data.py | 39 +- easy_rec/python/tools/edit_lookup_graph.py | 53 +- easy_rec/python/tools/faiss_index_pai.py | 49 +- easy_rec/python/tools/feature_selection.py | 105 +- easy_rec/python/tools/hit_rate_ds.py | 94 +- easy_rec/python/tools/hit_rate_pai.py | 77 +- easy_rec/python/tools/pre_check.py | 49 +- easy_rec/python/tools/predict_and_chk.py | 52 +- easy_rec/python/tools/read_kafka.py | 14 +- easy_rec/python/tools/split_model_pai.py | 90 +- easy_rec/python/tools/split_pdn_model_pai.py | 90 +- easy_rec/python/tools/test_saved_model.py | 29 +- easy_rec/python/tools/view_saved_model.py | 10 +- easy_rec/python/tools/write_kafka.py | 10 +- easy_rec/python/train_eval.py | 93 +- easy_rec/python/utils/activation.py | 8 +- easy_rec/python/utils/check_utils.py | 49 +- easy_rec/python/utils/config_util.py | 133 +- easy_rec/python/utils/constant.py | 3 +- easy_rec/python/utils/convert_rtp_fg.py | 115 +- easy_rec/python/utils/dag.py | 11 +- easy_rec/python/utils/distribution_utils.py | 118 +- easy_rec/python/utils/ds_util.py | 9 +- easy_rec/python/utils/embedding_utils.py | 1 - easy_rec/python/utils/estimator_utils.py | 231 +-- easy_rec/python/utils/export_big_model.py | 275 +-- easy_rec/python/utils/expr_util.py | 14 +- easy_rec/python/utils/fg_util.py | 9 +- easy_rec/python/utils/hit_rate_utils.py | 75 +- easy_rec/python/utils/hive_utils.py | 21 +- easy_rec/python/utils/hpo_util.py | 9 +- easy_rec/python/utils/hvd_utils.py | 5 +- easy_rec/python/utils/input_utils.py | 20 +- easy_rec/python/utils/io_util.py | 58 +- easy_rec/python/utils/load_class.py | 37 +- easy_rec/python/utils/meta_graph_editor.py | 317 ++- easy_rec/python/utils/multi_optimizer.py | 5 +- easy_rec/python/utils/odps_util.py | 16 +- easy_rec/python/utils/pai_util.py | 10 +- easy_rec/python/utils/proto_util.py | 16 +- easy_rec/python/utils/restore_filter.py | 2 - easy_rec/python/utils/shape_utils.py | 82 +- easy_rec/python/utils/test_utils.py | 220 +-- easy_rec/python/utils/tf_utils.py | 5 +- .../data/amazon_books_data/process_amazon.py | 20 +- examples/data/criteo/process_criteo_kaggle.py | 12 +- examples/data/movielens_1m/process_ml_1m.py | 24 +- git-lfs/git_lfs.py | 88 +- pai_jobs/run.py | 380 ++-- pyproject.toml | 52 + scripts/ci_test_change_files.py | 12 +- scripts/pre-commit | 4 +- setup.cfg | 55 +- setup.py | 5 +- 250 files changed, 8749 insertions(+), 14093 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 506d766d9..97f7291d8 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -23,7 +23,7 @@ jobs: PULL_REQUEST_NUM: ${{ github.event.pull_request.number }} run: | source ~/.bashrc - source activate tf25_py3 + conda activate py310 pre-commit run -a if [ $? -eq 0 ] then diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 50f0a52c2..b6aecaa0c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,28 +1,40 @@ repos: - repo: https://github.com/pycqa/flake8.git - rev: 5.0.0 + rev: 7.3.0 hooks: - id: flake8 additional_dependencies: [ - 'flake8-docstrings==1.5.0' + 'flake8-docstrings>=1.7.0' ] - repo: https://github.com/pycqa/isort - rev: 4.3.21 + rev: 7.0.0 hooks: - id: isort - - repo: https://github.com/pre-commit/mirrors-yapf - rev: v0.32.0 + args: ["--profile", "black", "--atomic"] + + - repo: https://github.com/google/yapf + rev: v0.43.0 hooks: - id: yapf + args: [--style=setup.cfg] + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.8 + hooks: + - id: ruff + args: [--fix, --select=E231,E501,W503, --preview] + types: [python] + - id: ruff-format + - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: trailing-whitespace args: ["--no-markdown-linebreak-ext"] - id: check-yaml - id: end-of-file-fixer - id: requirements-txt-fixer - - id: double-quote-string-fixer + #- id: double-quote-string-fixer - id: check-merge-conflict - id: mixed-line-ending args: ["--fix=lf"] @@ -36,5 +48,6 @@ repos: hooks: - id: mdformat additional_dependencies: [ - 'mdformat-tables' + 'mdformat-tables', + 'mdformat-black' ] diff --git a/docs/post_fix.py b/docs/post_fix.py index 920787310..726a32181 100644 --- a/docs/post_fix.py +++ b/docs/post_fix.py @@ -9,7 +9,5 @@ with open(sys.argv[1], 'w') as fout: for line_str in lines: if '_static/searchtools.js' in line_str: - fout.write( - ' \n' - ) + fout.write(' \n') fout.write(line_str) diff --git a/docs/source/_ext/post_process.py b/docs/source/_ext/post_process.py index 13bda9b34..85cb033c4 100644 --- a/docs/source/_ext/post_process.py +++ b/docs/source/_ext/post_process.py @@ -11,14 +11,12 @@ def __init__(self, document, startnode=None): super(PostFixLink, self).__init__(document, startnode) def apply(self, **kwargs): - def _visit(node): if not node.children: return for child in node.children: if isinstance(child, nodes.Element): - if 'refuri' in child.attributes and '.md#' in child.attributes[ - 'refuri']: + if 'refuri' in child.attributes and '.md#' in child.attributes['refuri']: src = child.attributes['refuri'] dst = src.replace('.md#', '.html#') logging.info('[PostFixLink] replace %s to %s' % (src, dst)) diff --git a/docs/source/component/backbone.md b/docs/source/component/backbone.md index 7e64f6350..74f006037 100644 --- a/docs/source/component/backbone.md +++ b/docs/source/component/backbone.md @@ -1102,10 +1102,12 @@ MovieLens-1M数据集效果： 定义一个继承[`tf.keras.layers.Layer`](https://keras.io/api/layers/base_layer/)的组件类，至少实现两个方法：`__init__`、`call`。 ```python -def __init__(self, params, name='xxx', reuse=None, **kwargs): - pass +def __init__(self, params, name="xxx", reuse=None, **kwargs): + pass + + def call(self, inputs, training=None, **kwargs): - pass + pass ``` `__init__`方法的第一个参数`params`接受框架传递给当前组件的参数。支持两种参数配置的方式：`google.protobuf.Struct`、自定义的protobuf message对象。params对象封装了对这两种格式的参数的统一读取接口，如下： @@ -1143,43 +1145,45 @@ FM layer的代码示例： ```python class FM(tf.keras.layers.Layer): - """Factorization Machine models pairwise (order-2) feature interactions without linear term and bias. - - References - - [Factorization Machines](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf) - Input shape. - - List of 2D tensor with shape: ``(batch_size,embedding_size)``. - - Or a 3D tensor with shape: ``(batch_size,field_size,embedding_size)`` - Output shape - - 2D tensor with shape: ``(batch_size, 1)``. - """ - - def __init__(self, params, name='fm', reuse=None, **kwargs): - super(FM, self).__init__(name, **kwargs) - self.reuse = reuse - self.use_variant = params.get_or_default('use_variant', False) - - def call(self, inputs, **kwargs): - if type(inputs) == list: - emb_dims = set(map(lambda x: int(x.shape[-1]), inputs)) - if len(emb_dims) != 1: - dims = ','.join([str(d) for d in emb_dims]) - raise ValueError('all embedding dim must be equal in FM layer:' + dims) - with tf.name_scope(self.name): - fea = tf.stack(inputs, axis=1) - else: - assert inputs.shape.ndims == 3, 'input of FM layer must be a 3D tensor or a list of 2D tensors' - fea = inputs - - with tf.name_scope(self.name): - square_of_sum = tf.square(tf.reduce_sum(fea, axis=1)) - sum_of_square = tf.reduce_sum(tf.square(fea), axis=1) - cross_term = tf.subtract(square_of_sum, sum_of_square) - if self.use_variant: - cross_term = 0.5 * cross_term - else: - cross_term = 0.5 * tf.reduce_sum(cross_term, axis=-1, keepdims=True) - return cross_term + """Factorization Machine models pairwise (order-2) feature interactions without linear term and bias. + + References + - [Factorization Machines](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf) + Input shape. + - List of 2D tensor with shape: ``(batch_size,embedding_size)``. + - Or a 3D tensor with shape: ``(batch_size,field_size,embedding_size)`` + Output shape + - 2D tensor with shape: ``(batch_size, 1)``. + """ + + def __init__(self, params, name="fm", reuse=None, **kwargs): + super(FM, self).__init__(name, **kwargs) + self.reuse = reuse + self.use_variant = params.get_or_default("use_variant", False) + + def call(self, inputs, **kwargs): + if type(inputs) == list: + emb_dims = set(map(lambda x: int(x.shape[-1]), inputs)) + if len(emb_dims) != 1: + dims = ",".join([str(d) for d in emb_dims]) + raise ValueError("all embedding dim must be equal in FM layer:" + dims) + with tf.name_scope(self.name): + fea = tf.stack(inputs, axis=1) + else: + assert ( + inputs.shape.ndims == 3 + ), "input of FM layer must be a 3D tensor or a list of 2D tensors" + fea = inputs + + with tf.name_scope(self.name): + square_of_sum = tf.square(tf.reduce_sum(fea, axis=1)) + sum_of_square = tf.reduce_sum(tf.square(fea), axis=1) + cross_term = tf.subtract(square_of_sum, sum_of_square) + if self.use_variant: + cross_term = 0.5 * cross_term + else: + cross_term = 0.5 * tf.reduce_sum(cross_term, axis=-1, keepdims=True) + return cross_term ``` # 如何搭建模型 diff --git a/docs/source/conf.py b/docs/source/conf.py index 8eac72396..d5d68dd8f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -41,11 +41,19 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', - 'sphinx.ext.napoleon', 'recommonmark', 'sphinx_markdown_tables', - 'post_process' + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon', + 'recommonmark', + 'sphinx_markdown_tables', + 'post_process', ] # Add any paths that contain templates here, relative to this directory. @@ -66,7 +74,7 @@ # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. # language = u'zh_CN' -language = u'en' +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -118,15 +126,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -136,17 +141,14 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ( - master_doc, 'easy_rec.tex', u'easy\\_rec Documentation', u'EasyRec Team', - 'manual' - ), + (master_doc, 'easy_rec.tex', 'easy\\_rec Documentation', 'EasyRec Team', 'manual'), ] # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, 'easy_rec', u'easy_rec Documentation', [author], 1)] +man_pages = [(master_doc, 'easy_rec', 'easy_rec Documentation', [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -155,8 +157,13 @@ # dir menu entry, description, category) texinfo_documents = [ ( - master_doc, 'easy_rec', u'easy_rec Documentation', author, 'easy_rec', - 'One line description of project.', 'Miscellaneous' + master_doc, + 'easy_rec', + 'easy_rec Documentation', + author, + 'easy_rec', + 'One line description of project.', + 'Miscellaneous', ), ] diff --git a/docs/source/feature/data.md b/docs/source/feature/data.md index 169902b78..7e424dc3e 100644 --- a/docs/source/feature/data.md +++ b/docs/source/feature/data.md @@ -79,8 +79,10 @@ process_lbl.py: ```python import numpy as np + + def remap_lbl(labels): - res = np.where(labels<5, 0, 1) + res = np.where(labels < 5, 0, 1) return res ``` diff --git a/docs/source/models/user_define.md b/docs/source/models/user_define.md index 4272fcf60..4e48214d0 100644 --- a/docs/source/models/user_define.md +++ b/docs/source/models/user_define.md @@ -298,12 +298,11 @@ python -m easy_rec.python.train_eval --pipeline_config_path samples/model_config 增加测试用例到easy_rec/python/test/train_eval_test.py ```python - def test_custom_model(self): +def test_custom_model(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/custom_model.config', - self._test_dir) + "samples/model_config/custom_model.config", self._test_dir + ) self.assertTrue(self._success) - ``` 运行CustomModel测试用例 diff --git a/docs/source/train.md b/docs/source/train.md index 564791957..1130c5ea0 100644 --- a/docs/source/train.md +++ b/docs/source/train.md @@ -101,10 +101,10 @@ import tensorflow as tf import os, sys - ckpt_reader = tf.train.NewCheckpointReader('experiments/model.ckpt-0') + ckpt_reader = tf.train.NewCheckpointReader("experiments/model.ckpt-0") ckpt_var2shape_map = ckpt_reader.get_variable_to_shape_map() for key in ckpt_var2shape_map: - print(key) + print(key) ``` - save_checkpoints_steps: 每隔多少步保存一次checkpoint, 默认是1000。当训练数据量很大的时候，这个值要设置大一些 diff --git a/easy_rec/__init__.py b/easy_rec/__init__.py index ecfa54f93..a35666f52 100644 --- a/easy_rec/__init__.py +++ b/easy_rec/__init__.py @@ -12,9 +12,7 @@ parent_dir = os.path.dirname(curr_dir) sys.path.insert(0, parent_dir) -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') # Avoid import tensorflow which conflicts with the version used in EasyRecProcessor if 'PROCESSOR_TEST' not in os.environ: @@ -25,6 +23,7 @@ def get_ops_dir(): import tensorflow as tf + if platform.system() == 'Linux': ops_dir = os.path.join(curr_dir, 'python/ops') if 'PAI' in tf.__version__: @@ -49,7 +48,9 @@ def get_ops_dir(): logging.warning('ops_dir[%s] does not exist' % ops_dir) ops_dir = None - from easy_rec.python.inference.predictor import Predictor # isort:skip # noqa: E402 + from easy_rec.python.inference.predictor import ( # isort:skip # noqa: E402 + Predictor, + ) from easy_rec.python.main import evaluate # isort:skip # noqa: E402 from easy_rec.python.main import distribute_evaluate # isort:skip # noqa: E402 from easy_rec.python.main import export # isort:skip # noqa: E402 diff --git a/easy_rec/python/builders/hyperparams_builder.py b/easy_rec/python/builders/hyperparams_builder.py index b9a1b9a65..73af4b54e 100644 --- a/easy_rec/python/builders/hyperparams_builder.py +++ b/easy_rec/python/builders/hyperparams_builder.py @@ -14,6 +14,7 @@ # limitations under the License. # ============================================================================== """Builder function to construct tf-slim arg_scope for convolution, fc ops.""" + import tensorflow as tf from easy_rec.python.compat import regularizers @@ -36,22 +37,16 @@ def build_regularizer(regularizer): """ regularizer_oneof = regularizer.WhichOneof('regularizer_oneof') if regularizer_oneof == 'l1_regularizer': - return regularizers.l1_regularizer( - scale=float(regularizer.l1_regularizer.scale) - ) + return regularizers.l1_regularizer(scale=float(regularizer.l1_regularizer.scale)) if regularizer_oneof == 'l2_regularizer': - return regularizers.l2_regularizer( - scale=float(regularizer.l2_regularizer.scale) - ) + return regularizers.l2_regularizer(scale=float(regularizer.l2_regularizer.scale)) if regularizer_oneof == 'l1_l2_regularizer': return regularizers.l1_l2_regularizer( scale_l1=float(regularizer.l1_l2_regularizer.scale_l1), - scale_l2=float(regularizer.l1_l2_regularizer.scale_l2) + scale_l2=float(regularizer.l1_l2_regularizer.scale_l2), ) - raise ValueError( - 'Unknown regularizer function: {}'.format(regularizer_oneof) - ) + raise ValueError('Unknown regularizer function: {}'.format(regularizer_oneof)) def build_initializer(initializer): @@ -70,19 +65,15 @@ def build_initializer(initializer): if initializer_oneof == 'truncated_normal_initializer': return tf.truncated_normal_initializer( mean=initializer.truncated_normal_initializer.mean, - stddev=initializer.truncated_normal_initializer.stddev + stddev=initializer.truncated_normal_initializer.stddev, ) if initializer_oneof == 'random_normal_initializer': return tf.random_normal_initializer( mean=initializer.random_normal_initializer.mean, - stddev=initializer.random_normal_initializer.stddev + stddev=initializer.random_normal_initializer.stddev, ) if initializer_oneof == 'glorot_normal_initializer': return tf.glorot_normal_initializer() if initializer_oneof == 'constant_initializer': - return tf.constant_initializer( - [x for x in initializer.constant_initializer.consts] - ) - raise ValueError( - 'Unknown initializer function: {}'.format(initializer_oneof) - ) + return tf.constant_initializer([x for x in initializer.constant_initializer.consts]) + raise ValueError('Unknown initializer function: {}'.format(initializer_oneof)) diff --git a/easy_rec/python/builders/loss_builder.py b/easy_rec/python/builders/loss_builder.py index f561be235..7a99a7de0 100644 --- a/easy_rec/python/builders/loss_builder.py +++ b/easy_rec/python/builders/loss_builder.py @@ -5,52 +5,49 @@ import numpy as np import tensorflow as tf +from easy_rec.python.loss.f1_reweight_loss import ( # NOQA + f1_reweight_sigmoid_cross_entropy, +) from easy_rec.python.loss.focal_loss import sigmoid_focal_loss_with_logits from easy_rec.python.loss.jrc_loss import jrc_loss +from easy_rec.python.loss.listwise_loss import ( # NOQA + listwise_distill_loss, + listwise_rank_loss, +) +from easy_rec.python.loss.pairwise_loss import ( # NOQA + pairwise_focal_loss, + pairwise_hinge_loss, + pairwise_logistic_loss, + pairwise_loss, +) +from easy_rec.python.loss.zero_inflated_lognormal import ( # NOQA + zero_inflated_lognormal_loss, +) from easy_rec.python.protos.loss_pb2 import LossType -from easy_rec.python.loss.f1_reweight_loss import f1_reweight_sigmoid_cross_entropy # NOQA -from easy_rec.python.loss.listwise_loss import listwise_distill_loss, listwise_rank_loss # NOQA -from easy_rec.python.loss.pairwise_loss import pairwise_focal_loss, pairwise_hinge_loss, pairwise_logistic_loss, pairwise_loss # NOQA -from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_loss # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 -def build( - loss_type, - label, - pred, - loss_weight=1.0, - num_class=1, - loss_param=None, - **kwargs -): +def build(loss_type, label, pred, loss_weight=1.0, num_class=1, loss_param=None, **kwargs): loss_name = kwargs.pop('loss_name') if 'loss_name' in kwargs else 'unknown' if loss_type == LossType.CLASSIFICATION: if num_class == 1: - return tf.losses.sigmoid_cross_entropy( - label, logits=pred, weights=loss_weight, **kwargs - ) + return tf.losses.sigmoid_cross_entropy(label, logits=pred, weights=loss_weight, **kwargs) else: - assert label.dtype in [tf.int32, tf.int64], \ - 'label.dtype must in [tf.int32, tf.int64] when use sparse_softmax_cross_entropy.' - return tf.losses.sparse_softmax_cross_entropy( - labels=label, logits=pred, weights=loss_weight, **kwargs - ) + assert label.dtype in [ + tf.int32, + tf.int64, + ], 'label.dtype must in [tf.int32, tf.int64] when use sparse_softmax_cross_entropy.' + return tf.losses.sparse_softmax_cross_entropy(labels=label, logits=pred, weights=loss_weight, **kwargs) elif loss_type == LossType.CROSS_ENTROPY_LOSS: return tf.losses.log_loss(label, pred, weights=loss_weight, **kwargs) elif loss_type == LossType.BINARY_CROSS_ENTROPY_LOSS: - losses = tf.keras.backend.binary_crossentropy( - label, pred, from_logits=True - ) + losses = tf.keras.backend.binary_crossentropy(label, pred, from_logits=True) return tf.reduce_mean(losses) elif loss_type in [LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS]: logging.info('%s is used' % LossType.Name(loss_type)) - return tf.losses.mean_squared_error( - labels=label, predictions=pred, weights=loss_weight, **kwargs - ) + return tf.losses.mean_squared_error(labels=label, predictions=pred, weights=loss_weight, **kwargs) elif loss_type == LossType.ZILN_LOSS: if loss_param is None: loss = zero_inflated_lognormal_loss(label, pred) @@ -67,7 +64,7 @@ def build( mu_reg=mu_reg, sigma_reg=sigma_reg, class_weight=class_weight, - reg_weight=reg_weight + reg_weight=reg_weight, ) if np.isscalar(loss_weight) and loss_weight != 1.0: return loss * loss_weight @@ -84,7 +81,7 @@ def build( loss_weight_strategy=loss_param.loss_weight_strategy, sample_weights=loss_weight, same_label_loss=loss_param.same_label_loss, - name=loss_name + name=loss_name, ) elif loss_type == LossType.PAIR_WISE_LOSS: session = kwargs.get('session_ids', None) @@ -97,7 +94,7 @@ def build( margin=margin, temperature=temp, weights=loss_weight, - name=loss_name + name=loss_name, ) elif loss_type == LossType.PAIRWISE_LOGISTIC_LOSS: session = kwargs.get('session_ids', None) @@ -116,7 +113,7 @@ def build( ohem_ratio=ohem_ratio, weights=loss_weight, use_label_margin=lbl_margin, - name=loss_name + name=loss_name, ) elif loss_type == LossType.PAIRWISE_HINGE_LOSS: session = kwargs.get('session_ids', None) @@ -140,14 +137,12 @@ def build( label_is_logits=label_is_logits, use_label_margin=use_label_margin, use_exponent=use_exponent, - name=loss_name + name=loss_name, ) elif loss_type == LossType.PAIRWISE_FOCAL_LOSS: session = kwargs.get('session_ids', None) if loss_param is None: - return pairwise_focal_loss( - label, pred, session_ids=session, weights=loss_weight, name=loss_name - ) + return pairwise_focal_loss(label, pred, session_ids=session, weights=loss_weight, name=loss_name) hinge_margin = None if loss_param.HasField('hinge_margin'): hinge_margin = loss_param.hinge_margin @@ -161,7 +156,7 @@ def build( ohem_ratio=loss_param.ohem_ratio, temperature=loss_param.temperature, weights=loss_weight, - name=loss_name + name=loss_name, ) elif loss_type == LossType.LISTWISE_RANK_LOSS: session = kwargs.get('session_ids', None) @@ -180,7 +175,7 @@ def build( label_is_logits=label_is_logits, transform_fn=trans_fn, scale_logits=scale, - weights=loss_weight + weights=loss_weight, ) elif loss_type == LossType.LISTWISE_DISTILL_LOSS: session = kwargs.get('session_ids', None) @@ -199,7 +194,7 @@ def build( label_clip_max_value=label_clip_max_value, transform_fn=trans_fn, scale_logits=scale, - weights=loss_weight + weights=loss_weight, ) elif loss_type == LossType.F1_REWEIGHTED_LOSS: f1_beta_square = 1.0 if loss_param is None else loss_param.f1_beta_square @@ -209,13 +204,11 @@ def build( pred, f1_beta_square, weights=loss_weight, - label_smoothing=label_smoothing + label_smoothing=label_smoothing, ) elif loss_type == LossType.BINARY_FOCAL_LOSS: if loss_param is None: - return sigmoid_focal_loss_with_logits( - label, pred, sample_weights=loss_weight, name=loss_name - ) + return sigmoid_focal_loss_with_logits(label, pred, sample_weights=loss_weight, name=loss_name) gamma = loss_param.gamma alpha = None if loss_param.HasField('alpha'): @@ -228,7 +221,7 @@ def build( ohem_ratio=loss_param.ohem_ratio, sample_weights=loss_weight, label_smoothing=loss_param.label_smoothing, - name=loss_name + name=loss_name, ) else: raise ValueError('unsupported loss type: %s' % LossType.Name(loss_type)) @@ -248,9 +241,10 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): """ loss_dict = {} for kd in kds: - assert kd.pred_name in prediction_dict, \ - 'invalid predict_name: %s available ones: %s' % ( - kd.pred_name, ','.join(prediction_dict.keys())) + assert kd.pred_name in prediction_dict, 'invalid predict_name: %s available ones: %s' % ( + kd.pred_name, + ','.join(prediction_dict.keys()), + ) loss_name = kd.loss_name if not loss_name: @@ -258,17 +252,15 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): loss_name += '_' + kd.soft_label_name.replace('/', '_') loss_weight = kd.loss_weight - if kd.HasField('task_space_indicator_name' - ) and kd.HasField('task_space_indicator_value'): + if kd.HasField('task_space_indicator_name') and kd.HasField('task_space_indicator_value'): in_task_space = tf.to_float( tf.equal( feature_dict[kd.task_space_indicator_name], - kd.task_space_indicator_value + kd.task_space_indicator_value, ) ) loss_weight = loss_weight * ( - kd.in_task_space_weight * in_task_space + kd.out_task_space_weight * - (1 - in_task_space) + kd.in_task_space_weight * in_task_space + kd.out_task_space_weight * (1 - in_task_space) ) label = label_dict[kd.soft_label_name] @@ -336,22 +328,14 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): labels = label preds = pred losses = tf.keras.losses.KLD(labels, preds) - loss_dict[loss_name - ] = tf.reduce_mean(losses, name=loss_name) * loss_weight + loss_dict[loss_name] = tf.reduce_mean(losses, name=loss_name) * loss_weight elif kd.loss_type == LossType.BINARY_CROSS_ENTROPY_LOSS: - losses = tf.keras.backend.binary_crossentropy( - label, pred, from_logits=True - ) - loss_dict[loss_name - ] = tf.reduce_mean(losses, name=loss_name) * loss_weight + losses = tf.keras.backend.binary_crossentropy(label, pred, from_logits=True) + loss_dict[loss_name] = tf.reduce_mean(losses, name=loss_name) * loss_weight elif kd.loss_type == LossType.CROSS_ENTROPY_LOSS: - loss_dict[loss_name] = tf.losses.log_loss( - label, pred, weights=loss_weight - ) + loss_dict[loss_name] = tf.losses.log_loss(label, pred, weights=loss_weight) elif kd.loss_type == LossType.L2_LOSS: - loss_dict[loss_name] = tf.losses.mean_squared_error( - labels=label, predictions=pred, weights=loss_weight - ) + loss_dict[loss_name] = tf.losses.mean_squared_error(labels=label, predictions=pred, weights=loss_weight) else: loss_param = kd.WhichOneof('loss_param') kwargs = {} @@ -365,6 +349,6 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): pred, loss_weight=loss_weight, loss_param=loss_param, - **kwargs + **kwargs, ) return loss_dict diff --git a/easy_rec/python/builders/optimizer_builder.py b/easy_rec/python/builders/optimizer_builder.py index 990647e65..6b1cd57b4 100644 --- a/easy_rec/python/builders/optimizer_builder.py +++ b/easy_rec/python/builders/optimizer_builder.py @@ -14,6 +14,7 @@ # limitations under the License. # ============================================================================== """Functions to build training optimizers.""" + import logging import tensorflow as tf @@ -49,24 +50,20 @@ def build(optimizer_config): learning_rate, decay=config.decay, momentum=config.momentum_optimizer_value, - epsilon=config.epsilon + epsilon=config.epsilon, ) if optimizer_type == 'momentum_optimizer': config = optimizer_config.momentum_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - optimizer = tf.train.MomentumOptimizer( - learning_rate, momentum=config.momentum_optimizer_value - ) + optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=config.momentum_optimizer_value) if optimizer_type == 'adam_optimizer': config = optimizer_config.adam_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - optimizer = tf.train.AdamOptimizer( - learning_rate, beta1=config.beta1, beta2=config.beta2 - ) + optimizer = tf.train.AdamOptimizer(learning_rate, beta1=config.beta1, beta2=config.beta2) if optimizer_type == 'adamw_optimizer': config = optimizer_config.adamw_optimizer @@ -77,21 +74,19 @@ def build(optimizer_config): weight_decay=config.weight_decay, learning_rate=learning_rate, beta1=config.beta1, - beta2=config.beta2 + beta2=config.beta2, ) if optimizer_type == 'adam_asyncw_optimizer': config = optimizer_config.adam_asyncw_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - logging.info( - 'adam_asyncw_optimizer weight_decay = %.8f' % config.weight_decay - ) + logging.info('adam_asyncw_optimizer weight_decay = %.8f' % config.weight_decay) optimizer = weight_decay_optimizers.AdamAsyncWOptimizer( weight_decay=config.weight_decay, learning_rate=learning_rate, beta1=config.beta1, - beta2=config.beta2 + beta2=config.beta2, ) if optimizer_type == 'lazy_adam_optimizer': @@ -99,39 +94,31 @@ def build(optimizer_config): learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) from easy_rec.python.compat.adam_s import AdamOptimizerS - optimizer = AdamOptimizerS( - learning_rate=learning_rate, beta1=config.beta1, beta2=config.beta2 - ) + + optimizer = AdamOptimizerS(learning_rate=learning_rate, beta1=config.beta1, beta2=config.beta2) if optimizer_type == 'momentumw_optimizer': config = optimizer_config.momentumw_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - logging.info( - 'momentumw_optimizer weight_decay = %.8f' % config.weight_decay - ) + logging.info('momentumw_optimizer weight_decay = %.8f' % config.weight_decay) optimizer = weight_decay_optimizers.MomentumWOptimizer( weight_decay=config.weight_decay, learning_rate=learning_rate, - momentum=config.momentum_optimizer_value + momentum=config.momentum_optimizer_value, ) if optimizer_type == 'adagrad_optimizer': config = optimizer_config.adagrad_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - optimizer = tf.train.AdagradOptimizer( - learning_rate, - initial_accumulator_value=config.initial_accumulator_value - ) + optimizer = tf.train.AdagradOptimizer(learning_rate, initial_accumulator_value=config.initial_accumulator_value) if optimizer_type == 'adam_async_optimizer': config = optimizer_config.adam_async_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - optimizer = tf.train.AdamAsyncOptimizer( - learning_rate, beta1=config.beta1, beta2=config.beta2 - ) + optimizer = tf.train.AdamAsyncOptimizer(learning_rate, beta1=config.beta1, beta2=config.beta2) if optimizer_type == 'ftrl_optimizer': config = optimizer_config.ftrl_optimizer @@ -143,16 +130,14 @@ def build(optimizer_config): initial_accumulator_value=config.initial_accumulator_value, l1_regularization_strength=config.l1_reg, l2_regularization_strength=config.l2_reg, - l2_shrinkage_regularization_strength=config.l2_shrinkage_reg + l2_shrinkage_regularization_strength=config.l2_shrinkage_reg, ) if optimizer is None: raise ValueError('Optimizer %s not supported.' % optimizer_type) if optimizer_config.use_moving_average: - optimizer = tf.contrib.opt.MovingAverageOptimizer( - optimizer, average_decay=optimizer_config.moving_average_decay - ) + optimizer = tf.contrib.opt.MovingAverageOptimizer(optimizer, average_decay=optimizer_config.moving_average_decay) return optimizer, summary_vars @@ -173,9 +158,7 @@ def _create_learning_rate(learning_rate_config): learning_rate_type = learning_rate_config.WhichOneof('learning_rate') if learning_rate_type == 'constant_learning_rate': config = learning_rate_config.constant_learning_rate - learning_rate = tf.constant( - config.learning_rate, dtype=tf.float32, name='learning_rate' - ) + learning_rate = tf.constant(config.learning_rate, dtype=tf.float32, name='learning_rate') if learning_rate_type == 'exponential_decay_learning_rate': config = learning_rate_config.exponential_decay_learning_rate @@ -187,7 +170,7 @@ def _create_learning_rate(learning_rate_config): burnin_learning_rate=config.burnin_learning_rate, burnin_steps=config.burnin_steps, min_learning_rate=config.min_learning_rate, - staircase=config.staircase + staircase=config.staircase, ) if learning_rate_type == 'manual_step_learning_rate': @@ -198,30 +181,41 @@ def _create_learning_rate(learning_rate_config): learning_rate_sequence = [config.initial_learning_rate] learning_rate_sequence += [x.learning_rate for x in config.schedule] learning_rate = learning_schedules.manual_stepping( - tf.train.get_or_create_global_step(), learning_rate_step_boundaries, - learning_rate_sequence, config.warmup + tf.train.get_or_create_global_step(), + learning_rate_step_boundaries, + learning_rate_sequence, + config.warmup, ) if learning_rate_type == 'cosine_decay_learning_rate': config = learning_rate_config.cosine_decay_learning_rate learning_rate = learning_schedules.cosine_decay_with_warmup( - tf.train.get_or_create_global_step(), config.learning_rate_base, - config.total_steps, config.warmup_learning_rate, config.warmup_steps, - config.hold_base_rate_steps + tf.train.get_or_create_global_step(), + config.learning_rate_base, + config.total_steps, + config.warmup_learning_rate, + config.warmup_steps, + config.hold_base_rate_steps, ) if learning_rate_type == 'poly_decay_learning_rate': config = learning_rate_config.poly_decay_learning_rate learning_rate = tf.train.polynomial_decay( - config.learning_rate_base, tf.train.get_or_create_global_step(), - config.total_steps, config.end_learning_rate, config.power + config.learning_rate_base, + tf.train.get_or_create_global_step(), + config.total_steps, + config.end_learning_rate, + config.power, ) if learning_rate_type == 'transformer_learning_rate': config = learning_rate_config.transformer_learning_rate learning_rate = learning_schedules.transformer_policy( - tf.train.get_or_create_global_step(), config.learning_rate_base, - config.hidden_size, config.warmup_steps, config.step_scaling_rate + tf.train.get_or_create_global_step(), + config.learning_rate_base, + config.hidden_size, + config.warmup_steps, + config.step_scaling_rate, ) if learning_rate is None: diff --git a/easy_rec/python/builders/strategy_builder.py b/easy_rec/python/builders/strategy_builder.py index be26c2e9f..2524aa46a 100644 --- a/easy_rec/python/builders/strategy_builder.py +++ b/easy_rec/python/builders/strategy_builder.py @@ -23,12 +23,13 @@ def build(train_config): # only works using pai-tf elif train_config.train_distribute == DistributionStrategy.ExascaleStrategy: import pai + distribution = pai.distribute.ExascaleStrategy( max_splits=10, issorted=True, optimize_clip_by_global_norm=False, enable_sparse_allreduce=False, - enable_hierarchical_allreduce=True + enable_hierarchical_allreduce=True, ) # the older version of MultiWorkerMirroredStrategy # works under tf1.12 to tf1.15 diff --git a/easy_rec/python/compat/adam_s.py b/easy_rec/python/compat/adam_s.py index d65b0c108..6afeb59bd 100644 --- a/easy_rec/python/compat/adam_s.py +++ b/easy_rec/python/compat/adam_s.py @@ -13,14 +13,20 @@ # limitations under the License. # ============================================================================== """Adam for TensorFlow.""" + from __future__ import absolute_import, division, print_function from tensorflow.python.eager import context from tensorflow.python.framework import ops +from tensorflow.python.ops import ( # NOQA + array_ops, + control_flow_ops, + math_ops, + resource_variable_ops, + state_ops, +) from tensorflow.python.training import optimizer, training_ops -from tensorflow.python.ops import array_ops, control_flow_ops, math_ops, resource_variable_ops, state_ops # NOQA - class AdamOptimizerS(optimizer.Optimizer): """Optimizer that implements the Adam algorithm. @@ -38,7 +44,7 @@ def __init__( beta2=0.999, epsilon=1e-8, use_locking=False, - name='Adam' + name='Adam', ): r"""Construct a new Adam optimizer. @@ -116,7 +122,7 @@ def _get_beta_accumulators(self): graph = ops.get_default_graph() return ( self._get_non_slot_variable('beta1_power', graph=graph), - self._get_non_slot_variable('beta2_power', graph=graph) + self._get_non_slot_variable('beta2_power', graph=graph), ) def _create_slots(self, var_list): @@ -125,12 +131,8 @@ def _create_slots(self, var_list): # workers (these need to go on the same PS, otherwise some updates are # silently ignored). first_var = min(var_list, key=lambda x: x.name) - self._create_non_slot_variable( - initial_value=self._beta1, name='beta1_power', colocate_with=first_var - ) - self._create_non_slot_variable( - initial_value=self._beta2, name='beta2_power', colocate_with=first_var - ) + self._create_non_slot_variable(initial_value=self._beta1, name='beta1_power', colocate_with=first_var) + self._create_non_slot_variable(initial_value=self._beta2, name='beta2_power', colocate_with=first_var) # Create slots for the first and second moments. for v in var_list: @@ -163,7 +165,7 @@ def _apply_dense(self, grad, var): math_ops.cast(self._beta2_t, var.dtype.base_dtype), math_ops.cast(self._epsilon_t, var.dtype.base_dtype), grad, - use_locking=self._use_locking + use_locking=self._use_locking, ).op def _resource_apply_dense(self, grad, var): @@ -181,7 +183,7 @@ def _resource_apply_dense(self, grad, var): math_ops.cast(self._beta2_t, grad.dtype.base_dtype), math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), grad, - use_locking=self._use_locking + use_locking=self._use_locking, ) def _apply_sparse_shared(self, grad, var, indices, scatter_add): @@ -192,7 +194,7 @@ def _apply_sparse_shared(self, grad, var, indices, scatter_add): beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) - lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) + lr = lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power) # m_t = beta1 * m + (1 - beta1) * g_t m = self.get_slot(var, 'm') m_scaled_g_values = grad * (1 - beta1_t) @@ -210,44 +212,31 @@ def _apply_sparse_shared(self, grad, var, indices, scatter_add): # var_update = state_ops.assign_sub( # var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking) v_part_sqrt = math_ops.sqrt(v_part_n) - var_update = scatter_add( - var, indices, -lr * m_part_n / (v_part_sqrt + epsilon_t) - ) + var_update = scatter_add(var, indices, -lr * m_part_n / (v_part_sqrt + epsilon_t)) return control_flow_ops.group(*[var_update, m_t, v_t]) def _apply_sparse(self, grad, var): return self._apply_sparse_shared( - grad.values, - var, - grad.indices, - lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda - x, - i, - v, - use_locking=self._use_locking)) + grad.values, + var, + grad.indices, + lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda + x, i, v, use_locking=self._use_locking + ), + ) def _resource_scatter_add(self, x, i, v): - with ops.control_dependencies( - [resource_variable_ops.resource_scatter_add(x.handle, i, v)] - ): + with ops.control_dependencies([resource_variable_ops.resource_scatter_add(x.handle, i, v)]): return x.value() def _resource_apply_sparse(self, grad, var, indices): - return self._apply_sparse_shared( - grad, var, indices, self._resource_scatter_add - ) + return self._apply_sparse_shared(grad, var, indices, self._resource_scatter_add) def _finish(self, update_ops, name_scope): # Update the power accumulators. with ops.control_dependencies(update_ops): beta1_power, beta2_power = self._get_beta_accumulators() with ops.colocate_with(beta1_power): - update_beta1 = beta1_power.assign( - beta1_power * self._beta1_t, use_locking=self._use_locking - ) - update_beta2 = beta2_power.assign( - beta2_power * self._beta2_t, use_locking=self._use_locking - ) - return control_flow_ops.group( - *update_ops + [update_beta1, update_beta2], name=name_scope - ) + update_beta1 = beta1_power.assign(beta1_power * self._beta1_t, use_locking=self._use_locking) + update_beta2 = beta2_power.assign(beta2_power * self._beta2_t, use_locking=self._use_locking) + return control_flow_ops.group(*update_ops + [update_beta1, update_beta2], name=name_scope) diff --git a/easy_rec/python/compat/array_ops.py b/easy_rec/python/compat/array_ops.py index 44ca091ba..4b847ba05 100644 --- a/easy_rec/python/compat/array_ops.py +++ b/easy_rec/python/compat/array_ops.py @@ -10,9 +10,7 @@ def convert_to_int_tensor(tensor, name, dtype=tf.int32): if tensor.dtype.is_integer: tensor = gen_math_ops.cast(tensor, dtype) else: - raise TypeError( - '%s must be an integer tensor; dtype=%s' % (name, tensor.dtype) - ) + raise TypeError('%s must be an integer tensor; dtype=%s' % (name, tensor.dtype)) return tensor @@ -57,9 +55,7 @@ def get_positive_axis(axis, ndims): elif -ndims <= axis < 0: return axis + ndims else: - raise ValueError( - 'axis=%s out of bounds: expected %s<=axis<%s' % (axis, -ndims, ndims) - ) + raise ValueError('axis=%s out of bounds: expected %s<=axis<%s' % (axis, -ndims, ndims)) elif axis < 0: raise ValueError('axis may only be negative if ndims is statically known.') return axis @@ -73,9 +69,7 @@ def tile_one_dimension(data, axis, multiple): multiples[axis] = multiple else: ones_value = tf.ones(tf.rank(data), tf.int32) - multiples = tf.concat( - [ones_value[:axis], [multiple], ones_value[axis + 1:]], axis=0 - ) + multiples = tf.concat([ones_value[:axis], [multiple], ones_value[axis + 1 :]], axis=0) return tf.tile(data, multiples) @@ -84,10 +78,7 @@ def _all_dimensions(x): # Fast path: avoid creating Rank and Range ops if ndims is known. if isinstance(x, ops.Tensor) and x.get_shape().ndims is not None: return constant_op.constant(np.arange(x.get_shape().ndims), dtype=tf.int32) - if ( - isinstance(x, sparse_tensor.SparseTensor) - and x.dense_shape.get_shape().is_fully_defined() - ): + if isinstance(x, sparse_tensor.SparseTensor) and x.dense_shape.get_shape().is_fully_defined(): r = x.dense_shape.get_shape().dims[0].value # sparse.dense_shape is 1-D. return constant_op.constant(np.arange(r), dtype=tf.int32) @@ -150,27 +141,21 @@ def repeat_with_axis(data, repeats, axis, name=None): if repeats.shape.ndims == 0: expanded = tf.expand_dims(data, axis + 1) tiled = tile_one_dimension(expanded, axis + 1, repeats) - result_shape = tf.concat( - [data_shape[:axis], [-1], data_shape[axis + 1:]], axis=0 - ) + result_shape = tf.concat([data_shape[:axis], [-1], data_shape[axis + 1 :]], axis=0) return tf.reshape(tiled, result_shape) # Broadcast the `repeats` tensor so rank(repeats) == axis + 1. if repeats.shape.ndims != axis + 1: repeats_shape = tf.shape(repeats) repeats_ndims = tf.rank(repeats) - broadcast_shape = tf.concat( - [data_shape[:axis + 1 - repeats_ndims], repeats_shape], axis=0 - ) + broadcast_shape = tf.concat([data_shape[: axis + 1 - repeats_ndims], repeats_shape], axis=0) repeats = tf.broadcast_to(repeats, broadcast_shape) repeats.set_shape([None] * (axis + 1)) # Create a "sequence mask" based on `repeats`, where slices across `axis` # contain one `True` value for each repetition. E.g., if # `repeats = [3, 1, 2]`, then `mask = [[1, 1, 1], [1, 0, 0], [1, 1, 0]]`. - max_repeat = gen_math_ops.maximum( - 0, gen_math_ops._max(repeats, _all_dimensions(repeats)) - ) + max_repeat = gen_math_ops.maximum(0, gen_math_ops._max(repeats, _all_dimensions(repeats))) mask = tf.sequence_mask(repeats, max_repeat) # Add a new dimension around each value that needs to be repeated, and @@ -186,18 +171,13 @@ def repeat_with_axis(data, repeats, axis, name=None): if axis == 0: result = masked else: - result_shape = tf.concat( - [data_shape[:axis], [-1], data_shape[axis + 1:]], axis=0 - ) + result_shape = tf.concat([data_shape[:axis], [-1], data_shape[axis + 1 :]], axis=0) result = tf.reshape(masked, result_shape) # Preserve shape information. if data.shape.ndims is not None: new_axis_size = 0 if repeats.shape[0] == 0 else None - result.set_shape( - data.shape[:axis].concatenate([new_axis_size] - ).concatenate(data.shape[axis + 1:]) - ) + result.set_shape(data.shape[:axis].concatenate([new_axis_size]).concatenate(data.shape[axis + 1 :])) return result diff --git a/easy_rec/python/compat/dynamic_variable.py b/easy_rec/python/compat/dynamic_variable.py index d41e7a716..04fea08bb 100644 --- a/easy_rec/python/compat/dynamic_variable.py +++ b/easy_rec/python/compat/dynamic_variable.py @@ -21,10 +21,13 @@ from sparse_operation_kit.experiment.communication import num_gpus from tensorflow.python.eager import context from tensorflow.python.framework import ops + # from tensorflow.python.ops import array_ops from tensorflow.python.ops import resource_variable_ops - -from tensorflow.python.ops.resource_variable_ops import ResourceVariable, variable_accessed # NOQA +from tensorflow.python.ops.resource_variable_ops import ( # NOQA + ResourceVariable, + variable_accessed, +) # from tensorflow.python.util import object_identity @@ -97,19 +100,15 @@ def __init__( mode=None, variable_def=None, import_scope=None, - **kwargs + **kwargs, ): self._indices = None if variable_def is not None: - super(DynamicVariable, self)._init_from_proto( - variable_def, import_scope=import_scope, validate_shape=False - ) + super(DynamicVariable, self)._init_from_proto(variable_def, import_scope=import_scope, validate_shape=False) g = ops.get_default_graph() handle = g.as_graph_element( - ops.prepend_name_scope( - variable_def.variable_name, import_scope=import_scope - ), - allow_operation=False + ops.prepend_name_scope(variable_def.variable_name, import_scope=import_scope), + allow_operation=False, ) self._dimension = handle.op.get_attr('shape').dim[-1].size self._key_type = handle.op.get_attr('key_type') @@ -138,9 +137,7 @@ def __init__( self._config = json.dumps(kwargs) self._config_dict = kwargs if var_type == 'hybrid' and self._key_type != tf.int64: - raise NotImplementedError( - 'only key_type tf.int64 is supported in HKV backend' - ) + raise NotImplementedError('only key_type tf.int64 is supported in HKV backend') if name is None: global dynamic_variable_count name = 'sok_dynamic_Variable_' + str(dynamic_variable_count) @@ -209,21 +206,17 @@ def __init__( self._initializer_op = tf.group([self._initializer_op, init_op]) # self._is_initialized_op = tf.group([self._is_initialized_op, is_initialized_op]) - handle_data = ( - resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult. - HandleData() - ) + handle_data = resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData() handle_data.is_set = True handle_data.shape_and_type.append( - resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult. - HandleShapeAndType( + resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult.HandleShapeAndType( shape=self.shape.as_proto(), dtype=self.dtype.as_datatype_enum ) ) resource_variable_ops._set_handle_shapes_and_types( self._handle, handle_data, - graph_mode=False if context.executing_eagerly() else True + graph_mode=False if context.executing_eagerly() else True, ) def is_static(self): @@ -263,9 +256,7 @@ def __repr__(self): @property def size(self): - return dynamic_variable_ops.dummy_var_shape( - self._dummy_handle, key_type=self._key_type, dtype=self._handle_dtype - ) + return dynamic_variable_ops.dummy_var_shape(self._dummy_handle, key_type=self._key_type, dtype=self._handle_dtype) @property def indices(self): @@ -324,7 +315,7 @@ def sparse_read(self, indices, name=None, lookup_only=False): self._dummy_handle, indices, dtype=self._handle_dtype, - lookup_only=lookup_only + lookup_only=lookup_only, ) def scatter_sub(self, sparse_delta, use_locking=False, name=None): @@ -377,9 +368,7 @@ def to_proto(self, *args, **kwargs): @staticmethod def from_proto(variable_def, import_scope=None): if '/DummyVarHandle' in variable_def.variable_name: - return DynamicVariable( - dimension=0, variable_def=variable_def, import_scope=import_scope - ) + return DynamicVariable(dimension=0, variable_def=variable_def, import_scope=import_scope) else: return _resource_var_from_proto(variable_def, import_scope) # raise NotImplementedError("from_proto() is not supported.") @@ -395,16 +384,12 @@ def is_initialized(self, name): return True if self.is_static(): return self._base.is_initialized(name) - raise NotImplementedError( - 'is_initialized() is not supported in dynamic mode.' - ) + raise NotImplementedError('is_initialized() is not supported in dynamic mode.') def _read_variable_op(self): if self.is_static(): return self._base._read_variable_op() - raise NotImplementedError( - '_read_variable_op() is not supported in dynamic mode.' - ) + raise NotImplementedError('_read_variable_op() is not supported in dynamic mode.') def value(self): if self.is_static(): @@ -414,16 +399,12 @@ def value(self): def _dense_var_to_tensor(self, *args, **kwargs): if self.is_static(): return self._base._dense_var_to_tensor(*args, **kwargs) - raise NotImplementedError( - '_dense_var_to_tensor() is not supported in dynamic mode.' - ) + raise NotImplementedError('_dense_var_to_tensor() is not supported in dynamic mode.') def _gather_saveables_for_checkpoint(self): if self.is_static(): return self._base._gather_saveables_for_checkpoint() - raise NotImplementedError( - '_gather_saveables_for_checkpoint() is not supported in dynamic mode.' - ) + raise NotImplementedError('_gather_saveables_for_checkpoint() is not supported in dynamic mode.') def gather_nd(self, *args, **kwargs): if self.is_static(): @@ -443,58 +424,42 @@ def assign(self, *args, **kwargs): def scatter_max(self, *args, **kwargs): if self.is_static(): return self._base.scatter_max(*args, **kwargs) - raise NotImplementedError( - 'scatter_max() is not supported in dynamic mode.' - ) + raise NotImplementedError('scatter_max() is not supported in dynamic mode.') def scatter_min(self, *args, **kwargs): if self.is_static(): return self._base.scatter_min(*args, **kwargs) - raise NotImplementedError( - 'scatter_min() is not supported in dynamic mode.' - ) + raise NotImplementedError('scatter_min() is not supported in dynamic mode.') def scatter_mul(self, *args, **kwargs): if self.is_static(): return self._base.scatter_mul(*args, **kwargs) - raise NotImplementedError( - 'scatter_mul() is not supported in dynamic mode.' - ) + raise NotImplementedError('scatter_mul() is not supported in dynamic mode.') def scatter_dim(self, *args, **kwargs): if self.is_static(): return self._base.scatter_dim(*args, **kwargs) - raise NotImplementedError( - 'scatter_dim() is not supported in dynamic mode.' - ) + raise NotImplementedError('scatter_dim() is not supported in dynamic mode.') def batch_scatter_update(self, *args, **kwargs): if self.is_static(): return self._base.batch_scatter_update(*args, **kwargs) - raise NotImplementedError( - 'batch_scatter_update() is not supported in dynamic mode.' - ) + raise NotImplementedError('batch_scatter_update() is not supported in dynamic mode.') def scatter_nd_sub(self, *args, **kwargs): if self.is_static(): return self._base.scatter_nd_sub(*args, **kwargs) - raise NotImplementedError( - 'scatter_nd_sub() is not supported in dynamic mode.' - ) + raise NotImplementedError('scatter_nd_sub() is not supported in dynamic mode.') def scatter_nd_update(self, *args, **kwargs): if self.is_static(): return self._base.scatter_nd_update(*args, **kwargs) - raise NotImplementedError( - 'scatter_nd_update() is not supported in dynamic mode.' - ) + raise NotImplementedError('scatter_nd_update() is not supported in dynamic mode.') def _strided_slice_assign(self, *args, **kwargs): if self.is_static(): return self._base._strided_slice_assign(*args, **kwargs) - raise NotImplementedError( - '_strided_slice_assign() is not supported in dynamic mode.' - ) + raise NotImplementedError('_strided_slice_assign() is not supported in dynamic mode.') def __int__(self, *args, **kwargs): if self.is_static(): @@ -538,9 +503,7 @@ def export(var): the values of the given variable. """ if isinstance(var, DynamicVariable): - indices, values = dynamic_variable_ops.dummy_var_export( - var.handle, key_type=var.key_type, dtype=var.handle_dtype - ) + indices, values = dynamic_variable_ops.dummy_var_export(var.handle, key_type=var.key_type, dtype=var.handle_dtype) with tf.device('CPU'): indices = tf.identity(indices) values = tf.identity(values) diff --git a/easy_rec/python/compat/early_stopping.py b/easy_rec/python/compat/early_stopping.py index 2a69f6d88..a977e1843 100644 --- a/easy_rec/python/compat/early_stopping.py +++ b/easy_rec/python/compat/early_stopping.py @@ -21,21 +21,26 @@ import os import threading import time +from distutils.version import LooseVersion import tensorflow as tf -from distutils.version import LooseVersion from tensorflow.python.framework import dtypes, ops from tensorflow.python.ops import init_ops, state_ops, variable_scope from tensorflow.python.platform import gfile, tf_logging from tensorflow.python.summary import summary_iterator +from tensorflow.python.training import ( # NOQA + basic_session_run_hooks, + session_run_hook, + training_util, +) from easy_rec.python.utils.config_util import parse_time from easy_rec.python.utils.load_class import load_by_path -from tensorflow.python.training import basic_session_run_hooks, session_run_hook, training_util # NOQA - if LooseVersion(tf.__version__) >= LooseVersion('2.12.0'): - from tensorflow_estimator.python.estimator.estimator_export import estimator_export # NOQA + from tensorflow_estimator.python.estimator.estimator_export import ( # NOQA + estimator_export, + ) else: from tensorflow.python.util.tf_export import estimator_export @@ -54,9 +59,7 @@ def find_early_stop_var(var_list): @estimator_export('estimator.experimental.make_early_stopping_hook') -def make_early_stopping_hook( - estimator, should_stop_fn, run_every_secs=60, run_every_steps=None -): +def make_early_stopping_hook(estimator, should_stop_fn, run_every_secs=60, run_every_steps=None): """Creates early-stopping hook. Returns a `SessionRunHook` that stops training when `should_stop_fn` returns `True`. Usage example: @@ -95,15 +98,10 @@ def make_early_stopping_hook( ValueError: If both `run_every_secs` and `run_every_steps` are set. """ if run_every_secs is not None and run_every_steps is not None: - raise ValueError( - 'Only one of `run_every_secs` and `run_every_steps` must ' - 'be set.' - ) + raise ValueError('Only one of `run_every_secs` and `run_every_steps` must ' 'be set.') if estimator.config.is_chief: - return _StopOnPredicateHook( - should_stop_fn, run_every_secs, run_every_steps - ) + return _StopOnPredicateHook(should_stop_fn, run_every_secs, run_every_steps) else: return _CheckForStoppingHook() @@ -116,7 +114,7 @@ def stop_if_higher_hook( eval_dir=None, min_steps=0, run_every_secs=60, - run_every_steps=None + run_every_steps=None, ): """Creates hook to stop if the given metric is higher than the threshold. @@ -164,7 +162,7 @@ def stop_if_higher_hook( eval_dir=eval_dir, min_steps=min_steps, run_every_secs=run_every_secs, - run_every_steps=run_every_steps + run_every_steps=run_every_steps, ) @@ -176,7 +174,7 @@ def stop_if_lower_hook( eval_dir=None, min_steps=0, run_every_secs=60, - run_every_steps=None + run_every_steps=None, ): """Creates hook to stop if the given metric is lower than the threshold. @@ -224,7 +222,7 @@ def stop_if_lower_hook( eval_dir=eval_dir, min_steps=min_steps, run_every_secs=run_every_secs, - run_every_steps=run_every_steps + run_every_steps=run_every_steps, ) @@ -236,7 +234,7 @@ def stop_if_no_increase_hook( eval_dir=None, min_steps=0, run_every_secs=60, - run_every_steps=None + run_every_steps=None, ): """Creates hook to stop if metric does not increase within given max steps. @@ -285,7 +283,7 @@ def stop_if_no_increase_hook( eval_dir=eval_dir, min_steps=min_steps, run_every_secs=run_every_secs, - run_every_steps=run_every_steps + run_every_steps=run_every_steps, ) @@ -295,7 +293,7 @@ def custom_early_stop_hook( custom_stop_func, custom_stop_func_params, run_every_secs=60, - run_every_steps=None + run_every_steps=None, ): """Custom early stop hook for complex early stop conditions. @@ -319,8 +317,7 @@ def custom_early_stop_hook( if eval_dir is None: eval_dir = estimator.eval_dir() - if isinstance(custom_stop_func, - str) or isinstance(custom_stop_func, type(u'')): + if isinstance(custom_stop_func, str) or isinstance(custom_stop_func, type('')): custom_stop_func = load_by_path(custom_stop_func) def _custom_stop_fn(): @@ -331,7 +328,7 @@ def _custom_stop_fn(): estimator=estimator, should_stop_fn=_custom_stop_fn, run_every_secs=run_every_secs, - run_every_steps=run_every_steps + run_every_steps=run_every_steps, ) @@ -343,7 +340,7 @@ def stop_if_no_decrease_hook( eval_dir=None, min_steps=0, run_every_secs=60, - run_every_steps=None + run_every_steps=None, ): """Creates hook to stop if metric does not decrease within given max steps. @@ -392,7 +389,7 @@ def stop_if_no_decrease_hook( eval_dir=eval_dir, min_steps=min_steps, run_every_secs=run_every_secs, - run_every_steps=run_every_steps + run_every_steps=run_every_steps, ) @@ -415,14 +412,18 @@ def read_eval_metrics(eval_dir): metrics[value.tag] = value.simple_value if metrics: eval_metrics_dict[event.step].update(metrics) - return collections.OrderedDict( - sorted(eval_metrics_dict.items(), key=lambda t: t[0]) - ) + return collections.OrderedDict(sorted(eval_metrics_dict.items(), key=lambda t: t[0])) def _stop_if_threshold_crossed_hook( - estimator, metric_name, threshold, higher_is_better, eval_dir, min_steps, - run_every_secs, run_every_steps + estimator, + metric_name, + threshold, + higher_is_better, + eval_dir, + min_steps, + run_every_secs, + run_every_steps, ): """Creates early-stopping hook to stop training if threshold is crossed.""" if eval_dir is None: @@ -441,9 +442,12 @@ def stop_if_threshold_crossed_fn(): val = metrics[metric_name] if is_lhs_better(val, threshold): tf_logging.info( - 'At step %s, metric "%s" has value %s which is %s the configured ' - 'threshold (%s) for early stopping.', step, metric_name, val, - greater_or_lesser, threshold + 'At step %s, metric "%s" has value %s which is %s the configured ' 'threshold (%s) for early stopping.', + step, + metric_name, + val, + greater_or_lesser, + threshold, ) return True return False @@ -452,13 +456,19 @@ def stop_if_threshold_crossed_fn(): estimator=estimator, should_stop_fn=stop_if_threshold_crossed_fn, run_every_secs=run_every_secs, - run_every_steps=run_every_steps + run_every_steps=run_every_steps, ) def _stop_if_no_metric_improvement_hook( - estimator, metric_name, max_steps_without_improvement, higher_is_better, - eval_dir, min_steps, run_every_secs, run_every_steps + estimator, + metric_name, + max_steps_without_improvement, + higher_is_better, + eval_dir, + min_steps, + run_every_secs, + run_every_steps, ): """Returns hook to stop training if given metric shows no improvement.""" if eval_dir is None: @@ -484,8 +494,10 @@ def stop_if_no_metric_improvement_fn(): tf_logging.info( 'No %s in metric "%s" for %s steps, which is greater than or equal ' 'to max steps (%s) configured for early stopping.', - increase_or_decrease, metric_name, step - best_val_step, - max_steps_without_improvement + increase_or_decrease, + metric_name, + step - best_val_step, + max_steps_without_improvement, ) return True return False @@ -494,7 +506,7 @@ def stop_if_no_metric_improvement_fn(): estimator=estimator, should_stop_fn=stop_if_no_metric_improvement_fn, run_every_secs=run_every_secs, - run_every_steps=run_every_steps + run_every_steps=run_every_steps, ) @@ -508,26 +520,20 @@ def _summaries(eval_dir): `tensorflow.Event` object read from the event files. """ if gfile.Exists(eval_dir): - for event_file in gfile.Glob( - os.path.join(eval_dir, _EVENT_FILE_GLOB_PATTERN) - ): + for event_file in gfile.Glob(os.path.join(eval_dir, _EVENT_FILE_GLOB_PATTERN)): for event in summary_iterator.summary_iterator(event_file): yield event def _get_or_create_stop_var(): - with variable_scope.variable_scope( - name_or_scope=EARLY_STOP_SIG_SCOPE, - values=[], - reuse=variable_scope.AUTO_REUSE - ): + with variable_scope.variable_scope(name_or_scope=EARLY_STOP_SIG_SCOPE, values=[], reuse=variable_scope.AUTO_REUSE): return variable_scope.get_variable( name=EARLY_STOP_SIG_NAME, shape=[], dtype=dtypes.bool, initializer=init_ops.constant_initializer(False), collections=[ops.GraphKeys.GLOBAL_VARIABLES], - trainable=False + trainable=False, ) @@ -539,9 +545,7 @@ def __init__(self, should_stop_fn, run_every_secs=60, run_every_steps=None): raise TypeError('`should_stop_fn` must be callable.') self._should_stop_fn = should_stop_fn - self._timer = basic_session_run_hooks.SecondOrStepTimer( - every_secs=run_every_secs, every_steps=run_every_steps - ) + self._timer = basic_session_run_hooks.SecondOrStepTimer(every_secs=run_every_secs, every_steps=run_every_steps) self._global_step_tensor = None self._stop_var = _get_or_create_stop_var() self._stop_op = None @@ -559,9 +563,7 @@ def after_run(self, run_context, run_values): if self._timer.should_trigger_for_step(global_step): self._timer.update_last_triggered_step(global_step) if self._should_stop_fn(): - tf_logging.info( - 'Requesting early stopping at global step %d', global_step - ) + tf_logging.info('Requesting early stopping at global step %d', global_step) run_context.session.run(self._stop_op) run_context.request_stop() @@ -587,14 +589,11 @@ def after_run(self, run_context, run_values): class OssStopSignalHook(session_run_hook.SessionRunHook): - def __init__(self, model_dir, run_every_secs=10, run_every_steps=None): self._stop_sig_file = os.path.join(model_dir, 'OSS_STOP_SIGNAL') self._stop = False self._check_run = True - self._timer = basic_session_run_hooks.SecondOrStepTimer( - every_secs=run_every_secs, every_steps=run_every_steps - ) + self._timer = basic_session_run_hooks.SecondOrStepTimer(every_secs=run_every_secs, every_steps=run_every_steps) sleep_time = run_every_secs if run_every_secs is not None else 1 self._curr_step = 0 @@ -604,9 +603,7 @@ def _check_stop(): self._timer.update_last_triggered_step(self._curr_step) if gfile.Exists(self._stop_sig_file): self._stop = True - logging.info( - 'OssStopSignalHook: stop on signal %s' % self._stop_sig_file - ) + logging.info('OssStopSignalHook: stop on signal %s' % self._stop_sig_file) break else: time.sleep(sleep_time) @@ -645,14 +642,13 @@ def oss_stop_hook(estimator, run_every_secs=10, run_every_steps=None): return OssStopSignalHook( estimator.model_dir, run_every_secs=run_every_secs, - run_every_steps=run_every_steps + run_every_steps=run_every_steps, ) else: return _CheckForStoppingHook() class DeadlineStopHook(session_run_hook.SessionRunHook): - def __init__(self, deadline_ts): self._deadline_ts = deadline_ts self._stop_var = _get_or_create_stop_var() diff --git a/easy_rec/python/compat/embedding_ops.py b/easy_rec/python/compat/embedding_ops.py index 09215f3b8..ef66595f2 100644 --- a/easy_rec/python/compat/embedding_ops.py +++ b/easy_rec/python/compat/embedding_ops.py @@ -10,10 +10,7 @@ def _prune_invalid_ids(sparse_ids, sparse_weights): """Prune invalid IDs (< 0) from the input ids and weights.""" is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) if sparse_weights is not None: - is_id_valid = math_ops.logical_and( - is_id_valid, - array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool) - ) + is_id_valid = math_ops.logical_and(is_id_valid, array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) if sparse_weights is not None: sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) @@ -37,7 +34,7 @@ def safe_embedding_lookup_sparse( default_id=None, name=None, partition_strategy='div', - max_norm=None + max_norm=None, ): """Lookup embedding results, accounting for invalid IDs and empty features. @@ -88,52 +85,36 @@ def safe_embedding_lookup_sparse( raise ValueError('Missing embedding_weights %s.' % embedding_weights) embed_tensors = [ops.convert_to_tensor(embedding_weights)] - with ops.name_scope( - name, 'embedding_lookup', embed_tensors + [sparse_ids, sparse_weights] - ) as scope: + with ops.name_scope(name, 'embedding_lookup', embed_tensors + [sparse_ids, sparse_weights]) as scope: # Reshape higher-rank sparse ids and weights to linear segment ids. original_shape = sparse_ids.dense_shape original_rank_dim = sparse_ids.dense_shape.get_shape()[0] - original_rank = ( - array_ops.size(original_shape) - if original_rank_dim.value is None else original_rank_dim.value - ) + original_rank = array_ops.size(original_shape) if original_rank_dim.value is None else original_rank_dim.value sparse_ids = sparse_ops.sparse_reshape( - sparse_ids, [ - math_ops.reduce_prod( - array_ops.slice(original_shape, [0], [original_rank - 1]) - ), - array_ops.gather(original_shape, original_rank - 1) - ] + sparse_ids, + [ + math_ops.reduce_prod(array_ops.slice(original_shape, [0], [original_rank - 1])), + array_ops.gather(original_shape, original_rank - 1), + ], ) if sparse_weights is not None: - sparse_weights = sparse_tensor.SparseTensor( - sparse_ids.indices, sparse_weights.values, sparse_ids.dense_shape - ) + sparse_weights = sparse_tensor.SparseTensor(sparse_ids.indices, sparse_weights.values, sparse_ids.dense_shape) # Prune invalid ids and weights. sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) if combiner != 'sum': - sparse_ids, sparse_weights = _prune_invalid_weights( - sparse_ids, sparse_weights - ) + sparse_ids, sparse_weights = _prune_invalid_weights(sparse_ids, sparse_weights) # Fill in dummy values for empty features, if necessary. - sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows( - sparse_ids, default_id or 0 - ) + sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, default_id or 0) if sparse_weights is not None: - sparse_weights, _ = sparse_ops.sparse_fill_empty_rows( - sparse_weights, 1.0 - ) + sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) indices = sparse_ids.indices values = sparse_ids.values if values.dtype != dtypes.int64: values = math_ops.to_int64(values) - sparse_ids = sparse_tensor.SparseTensor( - indices=indices, values=values, dense_shape=sparse_ids.dense_shape - ) + sparse_ids = sparse_tensor.SparseTensor(indices=indices, values=values, dense_shape=sparse_ids.dense_shape) result = embedding_ops.embedding_lookup_sparse( embedding_weights, @@ -142,7 +123,7 @@ def safe_embedding_lookup_sparse( combiner=combiner, partition_strategy=partition_strategy, name=None if default_id is None else scope, - max_norm=max_norm + max_norm=max_norm, ) if default_id is None: @@ -150,12 +131,10 @@ def safe_embedding_lookup_sparse( # for use in Select. is_row_empty = array_ops.tile( array_ops.reshape(is_row_empty, [-1, 1]), - array_ops.stack([1, array_ops.shape(result)[1]]) + array_ops.stack([1, array_ops.shape(result)[1]]), ) - result = array_ops.where( - is_row_empty, array_ops.zeros_like(result), result, name=scope - ) + result = array_ops.where(is_row_empty, array_ops.zeros_like(result), result, name=scope) # Reshape back from linear ids back into higher-dimensional dense result. final_result = array_ops.reshape( @@ -163,15 +142,16 @@ def safe_embedding_lookup_sparse( array_ops.concat( [ array_ops.slice( - math_ops.cast(original_shape, dtypes.int32), [0], - [original_rank - 1] + math_ops.cast(original_shape, dtypes.int32), + [0], + [original_rank - 1], ), - array_ops.slice(array_ops.shape(result), [1], [-1]) - ], 0 - ) + array_ops.slice(array_ops.shape(result), [1], [-1]), + ], + 0, + ), ) final_result.set_shape( - tensor_shape.unknown_shape((original_rank_dim - 1).value - ).concatenate(result.get_shape()[1:]) + tensor_shape.unknown_shape((original_rank_dim - 1).value).concatenate(result.get_shape()[1:]) ) return final_result diff --git a/easy_rec/python/compat/embedding_parallel_saver.py b/easy_rec/python/compat/embedding_parallel_saver.py index 2b5646185..f540bb42d 100644 --- a/easy_rec/python/compat/embedding_parallel_saver.py +++ b/easy_rec/python/compat/embedding_parallel_saver.py @@ -6,15 +6,20 @@ import numpy as np from tensorflow.core.protobuf import saver_pb2 from tensorflow.python.framework import dtypes, ops + +# from tensorflow.python.ops import math_ops +# from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import ( # NOQA + array_ops, + control_flow_ops, + script_ops, + state_ops, +) from tensorflow.python.platform import gfile from tensorflow.python.training import saver from easy_rec.python.utils import constant -# from tensorflow.python.ops import math_ops -# from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops, control_flow_ops, script_ops, state_ops # NOQA - try: import horovod.tensorflow as hvd from sparse_operation_kit.experiment import raw_ops as dynamic_variable_ops @@ -28,6 +33,7 @@ from tensorflow.python.framework.load_library import load_op_library import easy_rec + load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) except Exception as ex: @@ -43,7 +49,6 @@ def _get_embed_part_id(embed_file): class EmbeddingParallelSaver(saver.Saver): - def __init__( self, var_list=None, @@ -60,22 +65,17 @@ def __init__( write_version=saver_pb2.SaverDef.V2, pad_step_number=False, save_relative_paths=False, - filename=None + filename=None, ): self._kv_vars = [] self._embed_vars = [] tf_vars = [] embed_para_vars = ops.get_collection(constant.EmbeddingParallel) for var in var_list: - if dynamic_variable is not None and isinstance( - var, dynamic_variable.DynamicVariable - ): + if dynamic_variable is not None and isinstance(var, dynamic_variable.DynamicVariable): self._kv_vars.append(var) elif var.name in embed_para_vars: - logging.info( - 'save shard embedding %s part_id=%d part_shape=%s' % - (var.name, hvd.rank(), var.get_shape()) - ) + logging.info('save shard embedding %s part_id=%d part_shape=%s' % (var.name, hvd.rank(), var.get_shape())) self._embed_vars.append(var) else: tf_vars.append(var) @@ -94,7 +94,7 @@ def __init__( write_version=write_version, pad_step_number=pad_step_number, save_relative_paths=save_relative_paths, - filename=filename + filename=filename, ) self._is_build = False @@ -102,18 +102,14 @@ def _has_embed_vars(self): return (len(self._kv_vars) + len(self._embed_vars)) > 0 def _save_dense_embedding(self, embed_var): - logging.info( - 'task[%d] save_dense_embed: %s' % (hvd.rank(), embed_var.name) - ) + logging.info('task[%d] save_dense_embed: %s' % (hvd.rank(), embed_var.name)) def _save_embed(embed, filename, var_name): task_id = hvd.rank() filename = filename.decode('utf-8') var_name = var_name.decode('utf-8').replace('/', '__') embed_dir = filename + '-embedding/' - logging.info( - 'task[%d] save_dense_embed: %s to %s' % (task_id, var_name, embed_dir) - ) + logging.info('task[%d] save_dense_embed: %s to %s' % (task_id, var_name, embed_dir)) if not gfile.Exists(embed_dir): gfile.MakeDirs(embed_dir) embed_file = filename + '-embedding/embed-' + var_name + '-part-%d.bin' % task_id @@ -130,24 +126,16 @@ def _save_embed(embed, filename, var_name): gfile.DeleteRecursively(embed_file) return np.asarray([embed_file], order='C', dtype=np.object) - file_name = ops.get_default_graph().get_tensor_by_name( - self.saver_def.filename_tensor_name - ) - save_paths = script_ops.py_func( - _save_embed, [embed_var, file_name, embed_var.name], dtypes.string - ) + file_name = ops.get_default_graph().get_tensor_by_name(self.saver_def.filename_tensor_name) + save_paths = script_ops.py_func(_save_embed, [embed_var, file_name, embed_var.name], dtypes.string) return save_paths def _load_dense_embedding(self, embed_var): - file_name = ops.get_default_graph().get_tensor_by_name( - self.saver_def.filename_tensor_name - ) + file_name = ops.get_default_graph().get_tensor_by_name(self.saver_def.filename_tensor_name) embed_dim = embed_var.get_shape()[-1] embed_part_size = embed_var.get_shape()[0] - def _load_embed( - embed, embed_dim, embed_part_size, part_id, part_num, filename, var_name - ): + def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, filename, var_name): filename = filename.decode('utf-8') var_name = var_name.decode('utf-8').replace('/', '__') embed_pattern = filename + '-embedding/embed-' + var_name + '-part-*.bin' @@ -156,13 +144,11 @@ def _load_embed( embed_files.sort(key=_get_embed_part_id) logging.info( - 'task[%d] embed_files=%s embed_dim=%d embed_part_size=%d' % - (part_id, ','.join(embed_files), embed_dim, embed_part_size) + 'task[%d] embed_files=%s embed_dim=%d embed_part_size=%d' + % (part_id, ','.join(embed_files), embed_dim, embed_part_size) ) - part_embed_vals = np.zeros( - [embed_part_size, embed_dim], dtype=np.float32 - ) + part_embed_vals = np.zeros([embed_part_size, embed_dim], dtype=np.float32) part_update_cnt = 0 for embed_file in embed_files: part_id_o = _get_embed_part_id(embed_file) @@ -174,7 +160,7 @@ def _load_embed( sel_ids = np.where( np.logical_and( (embed_ids_o % part_num) == part_id, - embed_ids_o < embed_part_size * part_num + embed_ids_o < embed_part_size * part_num, ) )[0] part_update_cnt += len(sel_ids) @@ -192,15 +178,21 @@ def _load_embed( embed_dim=embed_dim, embed_part_size=embed_part_size, var_name='embed-' + embed_var.name.replace('/', '__'), - ckpt_path=file_name + ckpt_path=file_name, ) else: embed_val = script_ops.py_func( - _load_embed, [ - embed_var, embed_dim, embed_part_size, + _load_embed, + [ + embed_var, + embed_dim, + embed_part_size, hvd.rank(), - hvd.size(), file_name, embed_var.name - ], dtypes.float32 + hvd.size(), + file_name, + embed_var.name, + ], + dtypes.float32, ) embed_val.set_shape(embed_var.get_shape()) return state_ops.assign(embed_var, embed_val) @@ -209,9 +201,7 @@ def _save_kv_embedding(self, sok_var): indices, values = dynamic_variable_ops.dummy_var_export( sok_var.handle, key_type=sok_var.key_type, dtype=sok_var.handle_dtype ) - file_name = ops.get_default_graph().get_tensor_by_name( - self.saver_def.filename_tensor_name - ) + file_name = ops.get_default_graph().get_tensor_by_name(self.saver_def.filename_tensor_name) def _save_key_vals(indices, values, filename, var_name): var_name = var_name.decode('utf-8').replace('/', '__') @@ -240,25 +230,19 @@ def _save_key_vals(indices, values, filename, var_name): return np.asarray([key_file, val_file], order='C', dtype=np.object) - save_paths = script_ops.py_func( - _save_key_vals, [indices, values, file_name, sok_var.name], dtypes.string - ) + save_paths = script_ops.py_func(_save_key_vals, [indices, values, file_name, sok_var.name], dtypes.string) return save_paths def _load_kv_embedding(self, sok_var): - def _load_key_vals(filename, var_name): var_name = var_name.decode('utf-8').replace('/', '__') filename = filename.decode('utf-8') key_file_pattern = filename + '-embedding/embed-' + var_name + '-part-*.key' logging.info( - 'key_file_pattern=%s filename=%s var_name=%s var=%s' % - (key_file_pattern, filename, var_name, str(sok_var)) + 'key_file_pattern=%s filename=%s var_name=%s var=%s' % (key_file_pattern, filename, var_name, str(sok_var)) ) key_files = gfile.Glob(key_file_pattern) - logging.info( - 'key_file_pattern=%s file_num=%d' % (key_file_pattern, len(key_files)) - ) + logging.info('key_file_pattern=%s file_num=%d' % (key_file_pattern, len(key_files))) all_keys = [] all_vals = [] for key_file in key_files: @@ -270,19 +254,25 @@ def _load_key_vals(filename, var_name): break all_keys.append(tmp_keys.take(tmp_ids, axis=0)) logging.info( - 'part_keys.shape=%s %s %s' % - (str(tmp_keys.shape), str(tmp_ids.shape), str(all_keys[-1].shape)) + 'part_keys.shape=%s %s %s' + % ( + str(tmp_keys.shape), + str(tmp_ids.shape), + str(all_keys[-1].shape), + ) ) val_file = key_file[:-4] + 'vals' with gfile.GFile(val_file, 'rb') as fin: - tmp_vals = np.frombuffer(fin.read(), dtype=np.float32).reshape( - [-1, sok_var._dimension] - ) + tmp_vals = np.frombuffer(fin.read(), dtype=np.float32).reshape([-1, sok_var._dimension]) all_vals.append(tmp_vals.take(tmp_ids, axis=0)) logging.info( - 'part_vals.shape=%s %s %s' % - (str(tmp_vals.shape), str(tmp_ids.shape), str(all_vals[-1].shape)) + 'part_vals.shape=%s %s %s' + % ( + str(tmp_vals.shape), + str(tmp_ids.shape), + str(all_vals[-1].shape), + ) ) all_keys = np.concatenate(all_keys, axis=0) @@ -294,24 +284,21 @@ def _load_key_vals(filename, var_name): all_vals = all_vals.take(shuffle_ids, axis=0) return all_keys, all_vals - file_name = ops.get_default_graph().get_tensor_by_name( - self.saver_def.filename_tensor_name - ) + file_name = ops.get_default_graph().get_tensor_by_name(self.saver_def.filename_tensor_name) if load_embed_lib is not None: keys, vals = load_embed_lib.load_kv_embed( task_index=hvd.rank(), task_num=hvd.size(), embed_dim=sok_var._dimension, var_name='embed-' + sok_var.name.replace('/', '__'), - ckpt_path=file_name + ckpt_path=file_name, ) else: - logging.warning( - 'libload_embed.so not loaded, will use python script_ops' - ) + logging.warning('libload_embed.so not loaded, will use python script_ops') keys, vals = script_ops.py_func( - _load_key_vals, [file_name, sok_var.name], - (dtypes.int64, dtypes.float32) + _load_key_vals, + [file_name, sok_var.name], + (dtypes.int64, dtypes.float32), ) with ops.control_dependencies([sok_var._initializer_op]): return dynamic_variable_ops.dummy_var_assign(sok_var.handle, keys, vals) @@ -327,17 +314,13 @@ def build(self): restore_ops.append(self._load_kv_embedding(sok_var)) for embed_var in self._embed_vars: restore_ops.append(self._load_dense_embedding(embed_var)) - old_restore_op = ops.get_default_graph().get_operation_by_name( - self.saver_def.restore_op_name - ) + old_restore_op = ops.get_default_graph().get_operation_by_name(self.saver_def.restore_op_name) restore_ops.append(old_restore_op) restore_op_n = control_flow_ops.group(restore_ops) self.saver_def.restore_op_name = restore_op_n.name if self.saver_def.save_tensor_name and self._has_embed_vars(): - file_name = ops.get_default_graph().get_tensor_by_name( - self.saver_def.filename_tensor_name - ) + file_name = ops.get_default_graph().get_tensor_by_name(self.saver_def.filename_tensor_name) save_part_ops = [] for sok_var in self._kv_vars: save_part_op = self._save_kv_embedding(sok_var) @@ -345,9 +328,7 @@ def build(self): for embed_var in self._embed_vars: save_part_op = self._save_dense_embedding(embed_var) save_part_ops.append(save_part_op) - old_save_op = ops.get_default_graph().get_tensor_by_name( - self.saver_def.save_tensor_name - ) + old_save_op = ops.get_default_graph().get_tensor_by_name(self.saver_def.save_tensor_name) # only the first worker needs to save non embedding variables if hvd.rank() == 0: save_part_ops.append(old_save_op) diff --git a/easy_rec/python/compat/estimator_train.py b/easy_rec/python/compat/estimator_train.py index 917c451da..543f75d16 100644 --- a/easy_rec/python/compat/estimator_train.py +++ b/easy_rec/python/compat/estimator_train.py @@ -4,15 +4,18 @@ import os import tensorflow as tf +from tensorflow.python.distribute import estimator_training as distribute_coordinator_training # NOQA from tensorflow.python.estimator import run_config as run_config_lib +from tensorflow.python.estimator.training import ( # NOQA + _assert_eval_spec, + _ContinuousEvalListener, + _TrainingExecutor, +) from tensorflow.python.util import compat from easy_rec.python.compat.exporter import FinalExporter from easy_rec.python.utils import estimator_utils -from tensorflow.python.distribute import estimator_training as distribute_coordinator_training # NOQA -from tensorflow.python.estimator.training import _assert_eval_spec, _ContinuousEvalListener, _TrainingExecutor # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 gfile = tf.gfile @@ -23,9 +26,7 @@ class TrainDoneListener(_ContinuousEvalListener): def __init__(self, estimator): self._model_dir = estimator.model_dir - self._train_done_file = os.path.join( - self._model_dir, 'ESTIMATOR_TRAIN_DONE' - ) + self._train_done_file = os.path.join(self._model_dir, 'ESTIMATOR_TRAIN_DONE') @property def train_done_file(self): @@ -46,8 +47,7 @@ def after_eval(self, eval_result): latest_ckpt_path = estimator_utils.latest_checkpoint(model_dir) if latest_ckpt_path != last_ckpt_path: logging.info( - 'TrainDoneListener: latest_ckpt_path[%s] != last_ckpt_path[%s]' % - (latest_ckpt_path, last_ckpt_path) + 'TrainDoneListener: latest_ckpt_path[%s] != last_ckpt_path[%s]' % (latest_ckpt_path, last_ckpt_path) ) # there are more checkpoints wait to be evaluated return True @@ -63,7 +63,7 @@ def train_and_evaluate(estimator, train_spec, eval_spec): estimator=estimator, train_spec=train_spec, eval_spec=eval_spec, - continuous_eval_listener=train_done_listener + continuous_eval_listener=train_done_listener, ) config = estimator.config @@ -71,18 +71,14 @@ def train_and_evaluate(estimator, train_spec, eval_spec): # environment, we run `train_and_evaluate` via distribute coordinator. if distribute_coordinator_training.should_run_distribute_coordinator(config): logging.info('Running `train_and_evaluate` with Distribute Coordinator.') - distribute_coordinator_training.train_and_evaluate( - estimator, train_spec, eval_spec, _TrainingExecutor - ) + distribute_coordinator_training.train_and_evaluate(estimator, train_spec, eval_spec, _TrainingExecutor) return - if ( - config.task_type == run_config_lib.TaskType.EVALUATOR - and config.task_id > 0 - ): + if config.task_type == run_config_lib.TaskType.EVALUATOR and config.task_id > 0: raise ValueError( - 'For distributed training, there can only be one `evaluator` task ' - '(with task id 0). Given task id {}'.format(config.task_id) + 'For distributed training, there can only be one `evaluator` task ' '(with task id 0). Given task id {}'.format( + config.task_id + ) ) result = executor.run() @@ -90,25 +86,19 @@ def train_and_evaluate(estimator, train_spec, eval_spec): # fix for the bug evaluator fails to export in case num_epoch is reached # before num_steps is reached or num_steps is set to infinite if estimator_utils.is_evaluator(): - export_dir_base = os.path.join( - compat.as_str_any(estimator.model_dir), compat.as_str_any('export') - ) + export_dir_base = os.path.join(compat.as_str_any(estimator.model_dir), compat.as_str_any('export')) for exporter in eval_spec.exporters: if isinstance(exporter, FinalExporter): - export_path = os.path.join( - compat.as_str_any(export_dir_base), compat.as_str_any(exporter.name) - ) + export_path = os.path.join(compat.as_str_any(export_dir_base), compat.as_str_any(exporter.name)) # avoid duplicate export if gfile.IsDirectory(export_path + '/'): continue exporter.export( estimator=estimator, export_path=export_path, - checkpoint_path=estimator_utils.latest_checkpoint( - estimator.model_dir - ), + checkpoint_path=estimator_utils.latest_checkpoint(estimator.model_dir), eval_result=None, - is_the_final_export=True + is_the_final_export=True, ) if estimator_utils.is_chief(): diff --git a/easy_rec/python/compat/exporter.py b/easy_rec/python/compat/exporter.py index f25c525e0..2dd24694f 100644 --- a/easy_rec/python/compat/exporter.py +++ b/easy_rec/python/compat/exporter.py @@ -47,14 +47,10 @@ def _loss_smaller(best_eval_result, current_eval_result): """ default_key = metric_keys.MetricKeys.LOSS if not best_eval_result or default_key not in best_eval_result: - raise ValueError( - 'best_eval_result cannot be empty or no loss is found in it.' - ) + raise ValueError('best_eval_result cannot be empty or no loss is found in it.') if not current_eval_result or default_key not in current_eval_result: - raise ValueError( - 'current_eval_result cannot be empty or no loss is found in it.' - ) + raise ValueError('current_eval_result cannot be empty or no loss is found in it.') return best_eval_result[default_key] > current_eval_result[default_key] @@ -63,21 +59,12 @@ def _verify_compare_fn_args(compare_fn): """Verifies compare_fn arguments.""" args = set(util.fn_args(compare_fn)) if 'best_eval_result' not in args: - raise ValueError( - 'compare_fn (%s) must include best_eval_result argument.' % compare_fn - ) + raise ValueError('compare_fn (%s) must include best_eval_result argument.' % compare_fn) if 'current_eval_result' not in args: - raise ValueError( - 'compare_fn (%s) must include current_eval_result argument.' % compare_fn - ) - non_valid_args = list( - args - set(['best_eval_result', 'current_eval_result']) - ) + raise ValueError('compare_fn (%s) must include current_eval_result argument.' % compare_fn) + non_valid_args = list(args - set(['best_eval_result', 'current_eval_result'])) if non_valid_args: - raise ValueError( - 'compare_fn (%s) has following not expected args: %s' % - (compare_fn, non_valid_args) - ) + raise ValueError('compare_fn (%s) has following not expected args: %s' % (compare_fn, non_valid_args)) def _get_ckpt_version(path): @@ -102,7 +89,7 @@ def __init__( compare_fn=_loss_smaller, assets_extra=None, as_text=False, - exports_to_keep=5 + exports_to_keep=5, ): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. @@ -184,9 +171,7 @@ def make_train_and_eval_fn(): raise ValueError('`compare_fn` must not be None.') _verify_compare_fn_args(self._compare_fn) - self._saved_model_exporter = _SavedModelExporter( - name, serving_input_receiver_fn, assets_extra, as_text - ) + self._saved_model_exporter = _SavedModelExporter(name, serving_input_receiver_fn, assets_extra, as_text) self._event_file_pattern = event_file_pattern self._model_dir = None @@ -194,19 +179,13 @@ def make_train_and_eval_fn(): self._exports_to_keep = exports_to_keep if exports_to_keep is not None and exports_to_keep <= 0: - raise ValueError( - '`exports_to_keep`, if provided, must be a positive number. Got %s' % - exports_to_keep - ) + raise ValueError('`exports_to_keep`, if provided, must be a positive number. Got %s' % exports_to_keep) @property def name(self): return self._saved_model_exporter.name - def export( - self, estimator, export_path, checkpoint_path, eval_result, - is_the_final_export - ): + def export(self, estimator, export_path, checkpoint_path, eval_result, is_the_final_export): export_result = None if self._model_dir != estimator.model_dir and self._event_file_pattern: @@ -214,12 +193,8 @@ def export( tf_logging.info('Loading best metric from event files.') self._model_dir = estimator.model_dir - full_event_file_pattern = os.path.join( - self._model_dir, self._event_file_pattern - ) - self._best_eval_result = self._get_best_eval_result( - full_event_file_pattern, eval_result - ) + full_event_file_pattern = os.path.join(self._model_dir, self._event_file_pattern) + self._best_eval_result = self._get_best_eval_result(full_event_file_pattern, eval_result) if self._best_eval_result is None or self._compare_fn( best_eval_result=self._best_eval_result, current_eval_result=eval_result @@ -227,17 +202,18 @@ def export( tf_logging.info('Performing best model export.') self._best_eval_result = eval_result export_result = self._saved_model_exporter.export( - estimator, export_path, checkpoint_path, eval_result, - is_the_final_export + estimator, + export_path, + checkpoint_path, + eval_result, + is_the_final_export, ) self._garbage_collect_exports(export_path) # cp best checkpoints to best folder model_dir, _ = os.path.split(checkpoint_path) # add / is to be compatiable with oss best_dir = os.path.join(model_dir, 'best_ckpt/') - tf_logging.info( - 'Copy best checkpoint %s to %s' % (checkpoint_path, best_dir) - ) + tf_logging.info('Copy best checkpoint %s to %s' % (checkpoint_path, best_dir)) if not gfile.Exists(best_dir): gfile.MakeDirs(best_dir) for tmp_file in gfile.Glob(checkpoint_path + '.*'): @@ -250,9 +226,7 @@ def export( try: gfile.Copy(tmp_file, dst_path) except Exception as ex: - tf_logging.warn( - 'Copy file %s to %s failed: %s' % (tmp_file, dst_path, str(ex)) - ) + tf_logging.warn('Copy file %s to %s failed: %s' % (tmp_file, dst_path, str(ex))) self._garbage_collect_ckpts(best_dir) return export_result @@ -275,8 +249,8 @@ def _garbage_collect_ckpts(self, best_dir): tmp_steps = sorted(tmp_steps) drop_num = len(tmp_steps) - self._exports_to_keep tf_logging.info( - 'garbage_collect_ckpts: steps: %s export_to_keep: %d drop num: %d' % - (str(tmp_steps), self._exports_to_keep, drop_num) + 'garbage_collect_ckpts: steps: %s export_to_keep: %d drop num: %d' + % (str(tmp_steps), self._exports_to_keep, drop_num) ) for ver in tmp_steps[:drop_num]: tmp_prefix = os.path.join(best_dir, 'model.ckpt-%d.*' % ver) @@ -307,9 +281,7 @@ def _export_version_parser(path): # pylint: disable=protected-access keep_filter = gc._largest_export_versions(self._exports_to_keep) delete_filter = gc._negation(keep_filter) - for p in delete_filter( - gc._get_paths(export_dir_base, parser=_export_version_parser) - ): + for p in delete_filter(gc._get_paths(export_dir_base, parser=_export_version_parser)): try: gfile.DeleteRecursively(io_util.fix_oss_dir(p.path)) except errors_impl.NotFoundError as e: @@ -340,9 +312,7 @@ def _get_best_eval_result(self, event_files, curr_eval_result): if value.HasField('simple_value'): event_eval_result[value.tag] = value.simple_value if len(event_eval_result) >= 2: - if best_eval_result is None or self._compare_fn( - best_eval_result, event_eval_result - ): + if best_eval_result is None or self._compare_fn(best_eval_result, event_eval_result): best_eval_result = event_eval_result return best_eval_result @@ -353,9 +323,7 @@ class FinalExporter(Exporter): This class performs a single export at the end of training. """ - def __init__( - self, name, serving_input_receiver_fn, assets_extra=None, as_text=False - ): + def __init__(self, name, serving_input_receiver_fn, assets_extra=None, as_text=False): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. Args: @@ -376,26 +344,19 @@ def __init__( Raises: ValueError: if any arguments is invalid. """ - self._saved_model_exporter = _SavedModelExporter( - name, serving_input_receiver_fn, assets_extra, as_text - ) + self._saved_model_exporter = _SavedModelExporter(name, serving_input_receiver_fn, assets_extra, as_text) @property def name(self): return self._saved_model_exporter.name - def export( - self, estimator, export_path, checkpoint_path, eval_result, - is_the_final_export - ): + def export(self, estimator, export_path, checkpoint_path, eval_result, is_the_final_export): if not is_the_final_export: return None tf_logging.info('Performing the final export in the end of training.') - return self._saved_model_exporter.export( - estimator, export_path, checkpoint_path, eval_result, is_the_final_export - ) + return self._saved_model_exporter.export(estimator, export_path, checkpoint_path, eval_result, is_the_final_export) class LatestExporter(Exporter): @@ -410,7 +371,7 @@ def __init__( serving_input_receiver_fn, assets_extra=None, as_text=False, - exports_to_keep=5 + exports_to_keep=5, ): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. @@ -435,23 +396,16 @@ def __init__( Raises: ValueError: if any arguments is invalid. """ - self._saved_model_exporter = _SavedModelExporter( - name, serving_input_receiver_fn, assets_extra, as_text - ) + self._saved_model_exporter = _SavedModelExporter(name, serving_input_receiver_fn, assets_extra, as_text) self._exports_to_keep = exports_to_keep if exports_to_keep is not None and exports_to_keep <= 0: - raise ValueError( - '`exports_to_keep`, if provided, must be positive number' - ) + raise ValueError('`exports_to_keep`, if provided, must be positive number') @property def name(self): return self._saved_model_exporter.name - def export( - self, estimator, export_path, checkpoint_path, eval_result, - is_the_final_export - ): + def export(self, estimator, export_path, checkpoint_path, eval_result, is_the_final_export): export_result = self._saved_model_exporter.export( estimator, export_path, checkpoint_path, eval_result, is_the_final_export ) @@ -482,9 +436,7 @@ def _export_version_parser(path): # pylint: disable=protected-access keep_filter = gc._largest_export_versions(self._exports_to_keep) delete_filter = gc._negation(keep_filter) - for p in delete_filter( - gc._get_paths(export_dir_base, parser=_export_version_parser) - ): + for p in delete_filter(gc._get_paths(export_dir_base, parser=_export_version_parser)): try: gfile.DeleteRecursively(io_util.fix_oss_dir(p.path)) except errors_impl.NotFoundError as e: diff --git a/easy_rec/python/compat/feature_column/feature_column.py b/easy_rec/python/compat/feature_column/feature_column.py index 0345aeb6d..ccac194ac 100644 --- a/easy_rec/python/compat/feature_column/feature_column.py +++ b/easy_rec/python/compat/feature_column/feature_column.py @@ -143,6 +143,27 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.keras.engine import training from tensorflow.python.layers import base + +# from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import ( # NOQA + array_ops, + check_ops, + control_flow_ops, + data_flow_ops, + embedding_ops, + init_ops, + lookup_ops, + math_ops, + nn_ops, + parsing_ops, + resource_variable_ops, + sparse_ops, + string_ops, + template, + variable_scope, + variables, +) + # from tensorflow.python.ops.ragged import ragged_tensor # from tensorflow.python.ops.ragged import ragged_util from tensorflow.python.platform import gfile @@ -153,9 +174,6 @@ from easy_rec.python.compat.feature_column import utils as fc_utils from easy_rec.python.utils import conditional, constant, embedding_utils -# from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops, check_ops, control_flow_ops, data_flow_ops, embedding_ops, init_ops, lookup_ops, math_ops, nn_ops, parsing_ops, resource_variable_ops, sparse_ops, string_ops, template, variable_scope, variables # NOQA - try: from easy_rec.python.compat import dynamic_variable except Exception: @@ -167,22 +185,13 @@ hvd = None -def embedding_lookup_ragged( - embedding_weights, - ragged_ids, - ragged_weights, - combiner, - max_norm=None, - name=None -): +def embedding_lookup_ragged(embedding_weights, ragged_ids, ragged_weights, combiner, max_norm=None, name=None): segment_ids = ragged_ids.value_rowids() if segment_ids.dtype != dtypes.int32: segment_ids = math_ops.cast(segment_ids, dtypes.int32) ids = ragged_ids.flat_values ids, idx = array_ops.unique(ids) - embeddings = embedding_ops.embedding_lookup( - embedding_weights, ids, partition_strategy='mod', max_norm=max_norm - ) + embeddings = embedding_ops.embedding_lookup(embedding_weights, ids, partition_strategy='mod', max_norm=max_norm) if ragged_weights is not None: weights = ragged_weights.flat_values embeddings = array_ops.gather(embeddings, idx) @@ -215,17 +224,11 @@ def embedding_lookup_ragged( else: assert idx is not None if combiner == 'sum': - embeddings = math_ops.sparse_segment_sum( - embeddings, idx, segment_ids, name=name - ) + embeddings = math_ops.sparse_segment_sum(embeddings, idx, segment_ids, name=name) elif combiner == 'mean': - embeddings = math_ops.sparse_segment_mean( - embeddings, idx, segment_ids, name=name - ) + embeddings = math_ops.sparse_segment_mean(embeddings, idx, segment_ids, name=name) elif combiner == 'sqrtn': - embeddings = math_ops.sparse_segment_sqrt_n( - embeddings, idx, segment_ids, name=name - ) + embeddings = math_ops.sparse_segment_sqrt_n(embeddings, idx, segment_ids, name=name) else: assert False, 'Unrecognized combiner' return embeddings @@ -238,7 +241,7 @@ def embedding_parallel_lookup( output_ids, is_training, output_tensors=None, - batch_size=None + batch_size=None, ): N = len(output_ids) if batch_size is None: @@ -246,43 +249,36 @@ def embedding_parallel_lookup( else: num_segments = N * batch_size # first concat all the ids and unique - if isinstance(lookup_indices, - dict) and 'sparse_fea' in lookup_indices.keys(): + if isinstance(lookup_indices, dict) and 'sparse_fea' in lookup_indices.keys(): # all_uniq_ids, uniq_idx, segment_lens = features['sparse_fea'] all_ids, segment_lens = lookup_indices['sparse_fea'] all_uniq_ids, uniq_idx = array_ops.unique(all_ids) cumsum_lens = math_ops.cumsum(segment_lens) - segment_ids = array_ops.searchsorted( - cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right' + segment_ids = array_ops.searchsorted(cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') + elif ( + isinstance(lookup_indices, dict) + and 'ragged_ids' in lookup_indices.keys() + and 'ragged_lens' in lookup_indices.keys() + ): + all_ids, segment_lens = ( + lookup_indices['ragged_ids'], + lookup_indices['ragged_lens'], ) - elif isinstance(lookup_indices, - dict) and 'ragged_ids' in lookup_indices.keys( - ) and 'ragged_lens' in lookup_indices.keys(): - all_ids, segment_lens = lookup_indices['ragged_ids'], lookup_indices[ - 'ragged_lens'] all_uniq_ids, uniq_idx = array_ops.unique(all_ids) cumsum_lens = math_ops.cumsum(segment_lens) - segment_ids = array_ops.searchsorted( - cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right' - ) + segment_ids = array_ops.searchsorted(cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') elif isinstance(lookup_indices[0], sparse_tensor_lib.SparseTensor): with ops.device('/cpu:0'): all_ids = array_ops.concat([x.values for x in lookup_indices], axis=0) - segment_ids = array_ops.concat( - [x.indices[:, 0] for x in lookup_indices], axis=0 - ) + segment_ids = array_ops.concat([x.indices[:, 0] for x in lookup_indices], axis=0) all_uniq_ids, uniq_idx = array_ops.unique(all_ids) elif 'RaggedTensor' in str(type(lookup_indices[0])): with ops.device('/cpu:0'): all_ids = array_ops.concat([x.values for x in lookup_indices], axis=0) - segment_lens = array_ops.concat( - [x.row_lengths() for x in lookup_indices], axis=0 - ) + segment_lens = array_ops.concat([x.row_lengths() for x in lookup_indices], axis=0) all_uniq_ids, uniq_idx = array_ops.unique(all_ids) cumsum_lens = math_ops.cumsum(segment_lens) - segment_ids = array_ops.searchsorted( - cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right' - ) + segment_ids = array_ops.searchsorted(cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') else: assert False, 'invalid indices type: %s' % str(type(lookup_indices[0])) @@ -290,25 +286,17 @@ def embedding_parallel_lookup( if num_parts > 1: # dynamic partition p_assignments = math_ops.cast(all_uniq_ids % num_parts, dtypes.int32) - gather_ids = data_flow_ops.dynamic_partition( - all_uniq_ids, p_assignments, num_parts - ) + gather_ids = data_flow_ops.dynamic_partition(all_uniq_ids, p_assignments, num_parts) original_ids = math_ops.range(array_ops.size(all_uniq_ids)) - original_part_ids = data_flow_ops.dynamic_partition( - original_ids, p_assignments, num_parts - ) + original_part_ids = data_flow_ops.dynamic_partition(original_ids, p_assignments, num_parts) # all2all - split_sizes = array_ops.concat( - [array_ops.shape(x) for x in gather_ids], axis=0 - ) + split_sizes = array_ops.concat([array_ops.shape(x) for x in gather_ids], axis=0) send_ids = array_ops.concat(gather_ids, axis=0) recv_ids, recv_lens = hvd.alltoall(send_ids, split_sizes) # read embedding from dynamic variable if isinstance(embedding, dynamic_variable.DynamicVariable): - send_embed = embedding.sparse_read( - recv_ids, lookup_only=(not is_training) - ) + send_embed = embedding.sparse_read(recv_ids, lookup_only=(not is_training)) else: # find in subarray position # 0 2 4 6 8 10 ... @@ -318,9 +306,7 @@ def embedding_parallel_lookup( # all2all recv_embeddings, _ = hvd.alltoall(send_embed, recv_lens) - recv_embeddings = array_ops.split( - recv_embeddings, num_or_size_splits=split_sizes - ) + recv_embeddings = array_ops.split(recv_embeddings, num_or_size_splits=split_sizes) recv_embeddings = data_flow_ops.parallel_dynamic_stitch( original_part_ids, recv_embeddings, name='parallel_dynamic_stitch' ) @@ -329,13 +315,11 @@ def embedding_parallel_lookup( uniq_idx, segment_ids, num_segments=num_segments, - name='sparse_segment_sum' + name='sparse_segment_sum', ) else: if isinstance(embedding, dynamic_variable.DynamicVariable): - recv_embeddings = embedding.sparse_read( - all_uniq_ids, lookup_only=(not is_training) - ) + recv_embeddings = embedding.sparse_read(all_uniq_ids, lookup_only=(not is_training)) else: recv_embeddings = array_ops.gather(embedding, all_uniq_ids) embeddings = math_ops.sparse_segment_sum( @@ -343,7 +327,7 @@ def embedding_parallel_lookup( uniq_idx, segment_ids, num_segments=num_segments, - name='sparse_segment_sum' + name='sparse_segment_sum', ) embed_dim = embedding.get_shape()[-1] @@ -356,10 +340,7 @@ def embedding_parallel_lookup( if batch_size is None: batch_size = -1 - return array_ops.reshape( - array_ops.transpose(output_tensor, perm=[1, 0, 2]), - [batch_size, N * embed_dim] - ) + return array_ops.reshape(array_ops.transpose(output_tensor, perm=[1, 0, 2]), [batch_size, N * embed_dim]) def _internal_input_layer( @@ -372,7 +353,7 @@ def _internal_input_layer( cols_to_output_tensors=None, from_template=False, feature_name_to_output_tensors=None, - is_training=True + is_training=True, ): """See input_layer, `scope` is a name or variable scope to use.""" feature_columns = _normalize_feature_columns(feature_columns) @@ -389,7 +370,7 @@ def _internal_input_layer( if ops.GraphKeys.MODEL_VARIABLES not in weight_collections: weight_collections.append(ops.GraphKeys.MODEL_VARIABLES) - def _get_logits(): # pylint: disable=missing-docstring + def _get_logits(): builder = _LazyBuilder(features) output_tensors = [] @@ -398,25 +379,18 @@ def _get_logits(): # pylint: disable=missing-docstring logging.info('will sort columns[len=%d] by name' % len(tmp_cols)) tmp_cols = sorted(tmp_cols, key=lambda x: x.name) for column in tmp_cols: - with variable_scope.variable_scope( - None, default_name=column._var_scope_name - ): # pylint: disable=protected-access - tensor = column._get_dense_tensor( # pylint: disable=protected-access - builder, - weight_collections=weight_collections, - trainable=trainable) - num_elements = column._variable_shape.num_elements() # pylint: disable=protected-access + with variable_scope.variable_scope(None, default_name=column._var_scope_name): + tensor = column._get_dense_tensor(builder, weight_collections=weight_collections, trainable=trainable) + num_elements = column._variable_shape.num_elements() batch_size = array_ops.shape(tensor)[0] - output_tensor = array_ops.reshape( - tensor, shape=(batch_size, num_elements) - ) + output_tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements)) output_tensors.append(output_tensor) if cols_to_vars is not None: # Retrieve any variables created (some _DenseColumn's don't create # variables, in which case an empty list is returned). cols_to_vars[column] = ops.get_collection( ops.GraphKeys.GLOBAL_VARIABLES, - scope=variable_scope.get_variable_scope().name + scope=variable_scope.get_variable_scope().name, ) if cols_to_output_tensors is not None: cols_to_output_tensors[column] = output_tensor @@ -424,7 +398,7 @@ def _get_logits(): # pylint: disable=missing-docstring feature_name_to_output_tensors[column.raw_name] = output_tensor return array_ops.concat(output_tensors, 1) - def _get_logits_embedding_parallel(): # pylint: disable=missing-docstring + def _get_logits_embedding_parallel(): assert hvd is not None, 'horovod is not installed' builder = _LazyBuilder(features) @@ -458,9 +432,7 @@ def _get_var_type(column): batch_sizes = [] for column in feature_columns: ordered_columns.append(column) - with variable_scope.variable_scope( - None, default_name=column._var_scope_name - ): # pylint: disable=protected-access + with variable_scope.variable_scope(None, default_name=column._var_scope_name): # for features which does not require embedding if 'Embedding' not in str(type(column)): dense_cols.append(column) @@ -489,7 +461,7 @@ def _get_var_type(column): trainable=column.trainable and trainable, dtype=dtypes.float32, init_capacity=column.ev_params.init_capacity, - max_capacity=column.ev_params.max_capacity + max_capacity=column.ev_params.max_capacity, ) else: embedding_weights = variable_scope.get_variable( @@ -499,7 +471,7 @@ def _get_var_type(column): initializer=column.initializer, trainable=column.trainable and trainable, partitioner=None, - collections=weight_collections + collections=weight_collections, ) shared_weights[shared_name] = embedding_weights else: @@ -514,7 +486,7 @@ def _get_var_type(column): trainable=column.trainable and trainable, dtype=dtypes.float32, init_capacity=column.ev_params.init_capacity, - max_capacity=column.ev_params.max_capacity + max_capacity=column.ev_params.max_capacity, ) else: embedding_weights = variable_scope.get_variable( @@ -524,7 +496,7 @@ def _get_var_type(column): initializer=column.initializer, trainable=column.trainable and trainable, partitioner=None, - collections=weight_collections + collections=weight_collections, ) lookup_embeddings.append(embedding_weights) output_id = len(output_tensors) @@ -543,7 +515,7 @@ def _get_var_type(column): if lookup_indices is None: lookup_indices = { 'ragged_ids': features['ragged_ids'], - 'ragged_lens': features['ragged_lens'] + 'ragged_lens': features['ragged_lens'], } if 'ragged_wgts' in features: lookup_indices['ragged_wgts'] = features['ragged_wgts'] @@ -554,7 +526,7 @@ def _get_var_type(column): sparse_tensors = column.categorical_column._get_sparse_tensors( builder, weight_collections=weight_collections, - trainable=trainable + trainable=trainable, ) lookup_indices.append(sparse_tensors.id_tensor) if sparse_tensors.weight_tensor is not None: @@ -562,7 +534,7 @@ def _get_var_type(column): if cols_to_vars is not None: cols_to_vars[column] = ops.get_collection( ops.GraphKeys.GLOBAL_VARIABLES, - scope=variable_scope.get_variable_scope().name + scope=variable_scope.get_variable_scope().name, ) if dense_cnt > 0: @@ -570,8 +542,7 @@ def _get_var_type(column): fea_dim_s = 0 for dense_output_id, dense_col in zip(dense_output_ids, dense_cols): fea_dim_e = fea_dim_s + dense_col.shape[0] - output_tensors[dense_output_id] = features['dense_fea' - ][:, fea_dim_s:fea_dim_e] + output_tensors[dense_output_id] = features['dense_fea'][:, fea_dim_s:fea_dim_e] fea_dim_s = fea_dim_e batch_sizes.append(array_ops.shape(features['dense_fea'])[0]) else: @@ -588,13 +559,17 @@ def _get_var_type(column): batch_size = batch_sizes[0] # do embedding parallel lookup if len(lookup_output_ids) > 0: - packed_input = ('sparse_fea' in features or 'ragged_ids' in features) + packed_input = 'sparse_fea' in features or 'ragged_ids' in features if packed_input: uniq_embed_cnt = len(set(lookup_embeddings)) assert uniq_embed_cnt == 1, 'only one uniq embed is support for packed inputs' outputs = embedding_parallel_lookup( - lookup_embeddings[0], lookup_indices, lookup_output_ids, is_training, - output_tensors, batch_size + lookup_embeddings[0], + lookup_indices, + lookup_output_ids, + is_training, + output_tensors, + batch_size, ) else: if batch_size is None: @@ -605,13 +580,11 @@ def _get_var_type(column): batch_size = math_ops.reduce_max(all_indices) + 1 # group lookup_embeddings grouped_inputs = {} - for embedding, lookup_indice, output_id in zip( - lookup_embeddings, lookup_indices, lookup_output_ids - ): + for embedding, lookup_indice, output_id in zip(lookup_embeddings, lookup_indices, lookup_output_ids): if embedding not in grouped_inputs: grouped_inputs[embedding] = { 'lookup_indice': [lookup_indice], - 'output_id': [output_id] + 'output_id': [output_id], } else: grouped_inputs[embedding]['lookup_indice'].append(lookup_indice) @@ -621,8 +594,12 @@ def _get_var_type(column): lookup_indices = grouped_inputs[embedding]['lookup_indice'] output_ids = grouped_inputs[embedding]['output_id'] outputs = embedding_parallel_lookup( - embedding, lookup_indices, output_ids, is_training, output_tensors, - batch_size + embedding, + lookup_indices, + output_ids, + is_training, + output_tensors, + batch_size, ) for output_tensor, col in zip(output_tensors, feature_columns): @@ -649,15 +626,11 @@ def _get_var_type(column): if from_template: return _get_logits() else: - with variable_scope.variable_scope( - scope, default_name='input_layer', values=features.values() - ): + with variable_scope.variable_scope(scope, default_name='input_layer', values=features.values()): if embedding_utils.is_embedding_parallel(): return _get_logits_embedding_parallel() else: - with conditional( - embedding_utils.embedding_on_cpu(), ops.device('/cpu:0') - ): + with conditional(embedding_utils.embedding_on_cpu(), ops.device('/cpu:0')): return _get_logits() @@ -669,7 +642,7 @@ def input_layer( cols_to_vars=None, cols_to_output_tensors=None, feature_name_to_output_tensors=None, - is_training=True + is_training=True, ): """Returns a dense `Tensor` as input layer based on given `feature_columns`. @@ -735,7 +708,7 @@ def input_layer( cols_to_vars=cols_to_vars, cols_to_output_tensors=cols_to_output_tensors, feature_name_to_output_tensors=feature_name_to_output_tensors, - is_training=is_training + is_training=is_training, ) @@ -753,7 +726,7 @@ def __init__( trainable=True, cols_to_vars=None, name='feature_column_input_layer', - create_scope_now=True + create_scope_now=True, ): """See `input_layer`.""" self._feature_columns = feature_columns @@ -773,7 +746,7 @@ def __call__(self, features): weight_collections=self._weight_collections, trainable=self._trainable, cols_to_vars=None, - from_template=True + from_template=True, ) @property @@ -812,7 +785,7 @@ def linear_model( sparse_combiner='sum', weight_collections=None, trainable=True, - cols_to_vars=None + cols_to_vars=None, ): """Returns a linear prediction `Tensor` based on given `feature_columns`. @@ -937,9 +910,9 @@ def linear_model( sparse_combiner=sparse_combiner, weight_collections=weight_collections, trainable=trainable, - name=model_name + name=model_name, ) - retval = linear_model_layer(features) # pylint: disable=not-callable + retval = linear_model_layer(features) if cols_to_vars is not None: cols_to_vars.update(linear_model_layer.cols_to_vars()) return retval @@ -981,10 +954,9 @@ def __init__( weight_collections=None, trainable=True, name=None, - **kwargs + **kwargs, ): - super(_FCLinearWrapper, - self).__init__(trainable=trainable, name=name, **kwargs) + super(_FCLinearWrapper, self).__init__(trainable=trainable, name=name, **kwargs) self._feature_column = feature_column self._units = units self._sparse_combiner = sparse_combiner @@ -994,17 +966,17 @@ def build(self, _): if isinstance(self._feature_column, _CategoricalColumn): weight = self.add_variable( name='weights', - shape=(self._feature_column._num_buckets, self._units), # pylint: disable=protected-access + shape=(self._feature_column._num_buckets, self._units), initializer=init_ops.zeros_initializer(), - trainable=self.trainable + trainable=self.trainable, ) else: - num_elements = self._feature_column._variable_shape.num_elements() # pylint: disable=protected-access + num_elements = self._feature_column._variable_shape.num_elements() weight = self.add_variable( name='weights', shape=[num_elements, self._units], initializer=init_ops.zeros_initializer(), - trainable=self.trainable + trainable=self.trainable, ) _add_to_collections(weight, self._weight_collections) self._weight_var = weight @@ -1018,7 +990,7 @@ def call(self, builder): sparse_combiner=self._sparse_combiner, weight_collections=self._weight_collections, trainable=self.trainable, - weight_var=self._weight_var + weight_var=self._weight_var, ) return weighted_sum @@ -1026,14 +998,7 @@ def call(self, builder): class _BiasLayer(base.Layer): """A layer for the bias term.""" - def __init__( - self, - units=1, - trainable=True, - weight_collections=None, - name=None, - **kwargs - ): + def __init__(self, units=1, trainable=True, weight_collections=None, name=None, **kwargs): super(_BiasLayer, self).__init__(trainable=trainable, name=name, **kwargs) self._units = units self._weight_collections = weight_collections @@ -1043,7 +1008,7 @@ def build(self, _): 'bias_weights', shape=[self._units], initializer=init_ops.zeros_initializer(), - trainable=self.trainable + trainable=self.trainable, ) _add_to_collections(self._bias_variable, self._weight_collections) self.built = True @@ -1053,10 +1018,7 @@ def call(self, _): def _get_expanded_variable_list(variable): - if ( - isinstance(variable, variables.Variable) - or resource_variable_ops.is_resource_variable(variable) - ): + if isinstance(variable, variables.Variable) or resource_variable_ops.is_resource_variable(variable): return [variable] # Single variable case. else: # Must be a PartitionedVariable, so convert into a list. return list(variable) @@ -1080,7 +1042,7 @@ def __init__( weight_collections=None, trainable=True, name=None, - **kwargs + **kwargs, ): super(_LinearModel, self).__init__(name=name, **kwargs) self._feature_columns = _normalize_feature_columns(feature_columns) @@ -1092,16 +1054,19 @@ def __init__( column_layers = {} for column in sorted(self._feature_columns, key=lambda x: x.name): - with variable_scope.variable_scope( - None, default_name=column._var_scope_name - ) as vs: # pylint: disable=protected-access + with variable_scope.variable_scope(None, default_name=column._var_scope_name) as vs: # Having the fully expressed variable scope name ends up doubly # expressing the outer scope (scope with which this method was called) # in the name of the variable that would get created. column_name = _strip_leading_slashes(vs.name) column_layer = _FCLinearWrapper( - column, units, sparse_combiner, self._weight_collections, trainable, - column_name, **kwargs + column, + units, + sparse_combiner, + self._weight_collections, + trainable, + column_name, + **kwargs, ) column_layers[column_name] = column_layer self._column_layers = self._add_layers(column_layers) @@ -1110,7 +1075,7 @@ def __init__( trainable=trainable, weight_collections=self._weight_collections, name='bias_layer', - **kwargs + **kwargs, ) self._cols_to_vars = {} @@ -1127,31 +1092,25 @@ def call(self, features): for column in self._feature_columns: if not isinstance(column, (_DenseColumn, _CategoricalColumn)): raise ValueError( - 'Items of feature_columns must be either a ' - '_DenseColumn or _CategoricalColumn. Given: {}'.format(column) + 'Items of feature_columns must be either a ' '_DenseColumn or _CategoricalColumn. Given: {}'.format(column) ) weighted_sums = [] ordered_columns = [] builder = _LazyBuilder(features) for layer in sorted(self._column_layers.values(), key=lambda x: x.name): - column = layer._feature_column # pylint: disable=protected-access + column = layer._feature_column ordered_columns.append(column) weighted_sum = layer(builder) weighted_sums.append(weighted_sum) - self._cols_to_vars[column] = ops.get_collection( - ops.GraphKeys.GLOBAL_VARIABLES, scope=layer.scope_name - ) + self._cols_to_vars[column] = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, scope=layer.scope_name) _verify_static_batch_size_equality(weighted_sums, ordered_columns) - predictions_no_bias = math_ops.add_n( - weighted_sums, name='weighted_sum_no_bias' - ) + predictions_no_bias = math_ops.add_n(weighted_sums, name='weighted_sum_no_bias') predictions = nn_ops.bias_add( - predictions_no_bias, - self._bias_layer( # pylint: disable=not-callable - builder, - scope=variable_scope.get_variable_scope()), # pylint: disable=not-callable - name='weighted_sum') + predictions_no_bias, + self._bias_layer(builder, scope=variable_scope.get_variable_scope()), + name='weighted_sum', + ) bias = self._bias_layer.variables[0] self._cols_to_vars['bias'] = _get_expanded_variable_list(bias) return predictions @@ -1200,9 +1159,7 @@ def _transform_features(features, feature_columns): """ feature_columns = _normalize_feature_columns(feature_columns) outputs = {} - with ops.name_scope( - None, default_name='transform_features', values=features.values() - ): + with ops.name_scope(None, default_name='transform_features', values=features.values()): builder = _LazyBuilder(features) for column in sorted(feature_columns, key=lambda x: x.name): with ops.name_scope(None, default_name=column.name): @@ -1258,16 +1215,12 @@ def make_parse_example_spec(feature_columns): result = {} for column in feature_columns: if not isinstance(column, _FeatureColumn): - raise ValueError( - 'All feature_columns must be _FeatureColumn instances. ' - 'Given: {}'.format(column) - ) - config = column._parse_example_spec # pylint: disable=protected-access + raise ValueError('All feature_columns must be _FeatureColumn instances. ' 'Given: {}'.format(column)) + config = column._parse_example_spec for key, value in six.iteritems(config): if key in result and value != result[key]: raise ValueError( - 'feature_columns contain different parse_spec for key ' - '{}. Given {} and {}'.format(key, value, result[key]) + 'feature_columns contain different parse_spec for key ' '{}. Given {} and {}'.format(key, value, result[key]) ) result.update(config) return result @@ -1281,7 +1234,7 @@ def _embedding_column( ckpt_to_load_from=None, tensor_name_in_ckpt=None, max_norm=None, - trainable=True + trainable=True, ): """`_DenseColumn` that converts from sparse, categorical input. @@ -1357,22 +1310,16 @@ def model_fn(features, ...): if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError( - 'Must specify both `ckpt_to_load_from` and ' - '`tensor_name_in_ckpt` or none of them.' - ) + raise ValueError('Must specify both `ckpt_to_load_from` and ' '`tensor_name_in_ckpt` or none of them.') if (initializer is not None) and (not callable(initializer)): raise ValueError( - 'initializer must be callable if specified. ' - 'Embedding of column_name: {}'.format(categorical_column.name) + 'initializer must be callable if specified. ' 'Embedding of column_name: {}'.format(categorical_column.name) ) if initializer is None: - initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=0.01 / math.sqrt(dimension) - ) + initializer = init_ops.truncated_normal_initializer(mean=0.0, stddev=0.01 / math.sqrt(dimension)) - embedding_shape = categorical_column._num_buckets, dimension # pylint: disable=protected-access + embedding_shape = categorical_column._num_buckets, dimension def _creator(weight_collections, scope): embedding_column_layer = _EmbeddingColumnLayer( @@ -1380,9 +1327,9 @@ def _creator(weight_collections, scope): initializer=initializer, weight_collections=weight_collections, trainable=trainable, - name='embedding_column_layer' + name='embedding_column_layer', ) - return embedding_column_layer(None, scope=scope) # pylint: disable=not-callable + return embedding_column_layer(None, scope=scope) return _EmbeddingColumn( categorical_column=categorical_column, @@ -1392,17 +1339,11 @@ def _creator(weight_collections, scope): ckpt_to_load_from=ckpt_to_load_from, tensor_name_in_ckpt=tensor_name_in_ckpt, max_norm=max_norm, - trainable=trainable + trainable=trainable, ) -def _numeric_column( - key, - shape=(1, ), - default_value=None, - dtype=dtypes.float32, - normalizer_fn=None -): +def _numeric_column(key, shape=(1,), default_value=None, dtype=dtypes.float32, normalizer_fn=None): """Represents real valued or numerical features. Example: @@ -1456,18 +1397,11 @@ def _numeric_column( """ shape = _check_shape(shape, key) if not (dtype.is_integer or dtype.is_floating): - raise ValueError( - 'dtype must be convertible to float. ' - 'dtype: {}, key: {}'.format(dtype, key) - ) - default_value = fc_utils.check_default_value( - shape, default_value, dtype, key - ) + raise ValueError('dtype must be convertible to float. ' 'dtype: {}, key: {}'.format(dtype, key)) + default_value = fc_utils.check_default_value(shape, default_value, dtype, key) if normalizer_fn is not None and not callable(normalizer_fn): - raise TypeError( - 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn) - ) + raise TypeError('normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) fc_utils.assert_key_is_string(key) return _NumericColumn( @@ -1475,7 +1409,7 @@ def _numeric_column( shape=shape, default_value=default_value, dtype=dtype, - normalizer_fn=normalizer_fn + normalizer_fn=normalizer_fn, ) @@ -1547,18 +1481,11 @@ def _bucketized_column(source_column, boundaries): """ if not isinstance(source_column, _NumericColumn): raise ValueError( - 'source_column must be a column generated with numeric_column(). ' - 'Given: {}'.format(source_column) + 'source_column must be a column generated with numeric_column(). ' 'Given: {}'.format(source_column) ) if len(source_column.shape) > 1: - raise ValueError( - 'source_column must be one-dimensional column. ' - 'Given: {}'.format(source_column) - ) - if ( - not boundaries - or not (isinstance(boundaries, list) or isinstance(boundaries, tuple)) - ): + raise ValueError('source_column must be one-dimensional column. ' 'Given: {}'.format(source_column)) + if not boundaries or not (isinstance(boundaries, list) or isinstance(boundaries, tuple)): raise ValueError('boundaries must be a sorted list.') for i in range(len(boundaries) - 1): if boundaries[i] >= boundaries[i + 1]: @@ -1566,9 +1493,7 @@ def _bucketized_column(source_column, boundaries): return _BucketizedColumn(source_column, tuple(boundaries)) -def _categorical_column_with_hash_bucket( - key, hash_bucket_size, dtype=dtypes.string -): +def _categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dtypes.string): """Represents sparse feature where ids are set by hashing. Use this when your sparse features are in string or integer format, and you @@ -1611,13 +1536,11 @@ def _categorical_column_with_hash_bucket( ValueError: `dtype` is neither string nor integer. """ if hash_bucket_size is None: - raise ValueError('hash_bucket_size must be set. ' - 'key: {}'.format(key)) + raise ValueError('hash_bucket_size must be set. ' 'key: {}'.format(key)) if hash_bucket_size < 1: raise ValueError( - 'hash_bucket_size must be at least 1. ' - 'hash_bucket_size: {}, key: {}'.format(hash_bucket_size, key) + 'hash_bucket_size must be at least 1. ' 'hash_bucket_size: {}, key: {}'.format(hash_bucket_size, key) ) fc_utils.assert_key_is_string(key) @@ -1632,7 +1555,7 @@ def _categorical_column_with_vocabulary_file( vocabulary_size=None, num_oov_buckets=0, default_value=None, - dtype=dtypes.string + dtype=dtypes.string, ): """A `_CategoricalColumn` with a vocabulary file. @@ -1722,8 +1645,10 @@ def _categorical_column_with_vocabulary_file( with gfile.GFile(vocabulary_file) as f: vocabulary_size = sum(1 for _ in f) logging.info( - 'vocabulary_size = %d in %s is inferred from the number of elements ' - 'in the vocabulary_file %s.', vocabulary_size, key, vocabulary_file + 'vocabulary_size = %d in %s is inferred from the number of elements ' 'in the vocabulary_file %s.', + vocabulary_size, + key, + vocabulary_file, ) # `vocabulary_size` isn't required for lookup, but it is for `_num_buckets`. @@ -1731,14 +1656,9 @@ def _categorical_column_with_vocabulary_file( raise ValueError('Invalid vocabulary_size in {}.'.format(key)) if num_oov_buckets: if default_value is not None: - raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'. - format(key) - ) + raise ValueError("Can't specify both num_oov_buckets and default_value in {}.".format(key)) if num_oov_buckets < 0: - raise ValueError( - 'Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key) - ) + raise ValueError('Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key)) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) return _VocabularyFileCategoricalColumn( @@ -1747,13 +1667,11 @@ def _categorical_column_with_vocabulary_file( vocabulary_size=vocabulary_size, num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, default_value=-1 if default_value is None else default_value, - dtype=dtype + dtype=dtype, ) -def _categorical_column_with_vocabulary_list( - key, vocabulary_list, dtype=None, default_value=-1, num_oov_buckets=0 -): +def _categorical_column_with_vocabulary_list(key, vocabulary_list, dtype=None, default_value=-1, num_oov_buckets=0): """A `_CategoricalColumn` with in-memory vocabulary. Use this when your inputs are in string or integer format, and you have an @@ -1830,38 +1748,21 @@ def _categorical_column_with_vocabulary_list( ValueError: if `dtype` is not integer or string. """ if (vocabulary_list is None) or (len(vocabulary_list) < 1): - raise ValueError( - 'vocabulary_list {} must be non-empty, column_name: {}'.format( - vocabulary_list, key - ) - ) + raise ValueError('vocabulary_list {} must be non-empty, column_name: {}'.format(vocabulary_list, key)) if len(set(vocabulary_list)) != len(vocabulary_list): - raise ValueError( - 'Duplicate keys in vocabulary_list {}, column_name: {}'.format( - vocabulary_list, key - ) - ) + raise ValueError('Duplicate keys in vocabulary_list {}, column_name: {}'.format(vocabulary_list, key)) vocabulary_dtype = dtypes.as_dtype(np.array(vocabulary_list).dtype) if num_oov_buckets: if default_value != -1: - raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'. - format(key) - ) + raise ValueError("Can't specify both num_oov_buckets and default_value in {}.".format(key)) if num_oov_buckets < 0: - raise ValueError( - 'Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key) - ) - fc_utils.assert_string_or_int( - vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key) - ) + raise ValueError('Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key)) + fc_utils.assert_string_or_int(vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key)) if dtype is None: dtype = vocabulary_dtype elif dtype.is_integer != vocabulary_dtype.is_integer: raise ValueError( - 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format( - dtype, vocabulary_dtype, key - ) + 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format(dtype, vocabulary_dtype, key) ) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) @@ -1871,7 +1772,7 @@ def _categorical_column_with_vocabulary_list( vocabulary_list=tuple(vocabulary_list), dtype=dtype, default_value=default_value, - num_oov_buckets=num_oov_buckets + num_oov_buckets=num_oov_buckets, ) @@ -1930,20 +1831,11 @@ def _categorical_column_with_identity(key, num_buckets, default_value=None): ValueError: if `default_value` is not in range `[0, num_buckets)`. """ if num_buckets < 1: - raise ValueError( - 'num_buckets {} < 1, column_name {}'.format(num_buckets, key) - ) - if (default_value is not None - ) and ((default_value < 0) or (default_value >= num_buckets)): - raise ValueError( - 'default_value {} not in range [0, {}), column_name {}'.format( - default_value, num_buckets, key - ) - ) + raise ValueError('num_buckets {} < 1, column_name {}'.format(num_buckets, key)) + if (default_value is not None) and ((default_value < 0) or (default_value >= num_buckets)): + raise ValueError('default_value {} not in range [0, {}), column_name {}'.format(default_value, num_buckets, key)) fc_utils.assert_key_is_string(key) - return _IdentityCategoricalColumn( - key=key, num_buckets=num_buckets, default_value=default_value - ) + return _IdentityCategoricalColumn(key=key, num_buckets=num_buckets, default_value=default_value) def _indicator_column(categorical_column): @@ -1980,9 +1872,7 @@ def _indicator_column(categorical_column): return _IndicatorColumn(categorical_column) -def _weighted_categorical_column( - categorical_column, weight_feature_key, dtype=dtypes.float32 -): +def _weighted_categorical_column(categorical_column, weight_feature_key, dtype=dtypes.float32): """Applies weight values to a `_CategoricalColumn`. Use this when each of your sparse inputs has both an ID and a value. For @@ -2052,7 +1942,7 @@ def _weighted_categorical_column( return _WeightedCategoricalColumn( categorical_column=categorical_column, weight_feature_key=weight_feature_key, - dtype=dtype + dtype=dtype, ) @@ -2161,19 +2051,11 @@ def _crossed_column(keys, hash_bucket_size, hash_key=None): ValueError: If `hash_bucket_size < 1`. """ if not hash_bucket_size or hash_bucket_size < 1: - raise ValueError( - 'hash_bucket_size must be > 1. ' - 'hash_bucket_size: {}'.format(hash_bucket_size) - ) + raise ValueError('hash_bucket_size must be > 1. ' 'hash_bucket_size: {}'.format(hash_bucket_size)) if not keys or len(keys) < 2: - raise ValueError( - 'keys must be a list with length > 1. Given: {}'.format(keys) - ) + raise ValueError('keys must be a list with length > 1. Given: {}'.format(keys)) for key in keys: - if ( - not isinstance(key, six.string_types) - and not isinstance(key, _CategoricalColumn) - ): + if not isinstance(key, six.string_types) and not isinstance(key, _CategoricalColumn): raise ValueError( 'Unsupported key type. All keys must be either string, or ' 'categorical column except _HashedCategoricalColumn. ' @@ -2185,9 +2067,7 @@ def _crossed_column(keys, hash_bucket_size, hash_key=None): 'Hashing before crossing will increase probability of collision. ' 'Instead, use the feature name as a string. Given: {}'.format(key) ) - return _CrossedColumn( - keys=tuple(keys), hash_bucket_size=hash_bucket_size, hash_key=hash_key - ) + return _CrossedColumn(keys=tuple(keys), hash_bucket_size=hash_bucket_size, hash_key=hash_key) # TODO(rohanj): Clearly define semantics of this layer. @@ -2201,7 +2081,7 @@ def __init__( weight_collections=None, trainable=True, name=None, - **kwargs + **kwargs, ): """Constructor. @@ -2217,8 +2097,7 @@ def __init__( name: Name of the layer **kwargs: keyword named properties. """ - super(_EmbeddingColumnLayer, - self).__init__(trainable=trainable, name=name, **kwargs) + super(_EmbeddingColumnLayer, self).__init__(trainable=trainable, name=name, **kwargs) self._embedding_shape = embedding_shape self._initializer = initializer self._weight_collections = weight_collections @@ -2238,7 +2117,7 @@ def build(self, _): shape=self._embedding_shape, dtype=dtypes.float32, initializer=self._initializer, - trainable=self.trainable + trainable=self.trainable, ) if self._weight_collections and not context.executing_eagerly(): _add_to_collections(self._embedding_weight_var, self._weight_collections) @@ -2268,7 +2147,6 @@ class _FeatureColumn(object): @abc.abstractproperty def name(self): """Returns string, Used for naming and for name_scope.""" - pass @property def raw_name(self): @@ -2302,7 +2180,6 @@ def _transform_feature(self, inputs): Returns: Transformed feature `Tensor`. """ - pass @abc.abstractproperty def _parse_example_spec(self): @@ -2323,7 +2200,6 @@ def _parse_example_spec(self): return spec ``` """ - pass def _reset_config(self): """Resets the configuration in the column. @@ -2347,7 +2223,6 @@ class _DenseColumn(_FeatureColumn): @abc.abstractproperty def _variable_shape(self): """`TensorShape` of `_get_dense_tensor`, without batch dimension.""" - pass @abc.abstractmethod def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): @@ -2372,7 +2247,6 @@ def input_layer(features, feature_columns, ...): Returns: `Tensor` of shape [batch_size] + `_variable_shape`. """ - pass def _create_weighted_sum( @@ -2382,7 +2256,7 @@ def _create_weighted_sum( sparse_combiner, weight_collections, trainable, - weight_var=None + weight_var=None, ): """Creates a weighted sum for a dense/categorical column for linear_model.""" if isinstance(column, _CategoricalColumn): @@ -2393,7 +2267,7 @@ def _create_weighted_sum( sparse_combiner=sparse_combiner, weight_collections=weight_collections, trainable=trainable, - weight_var=weight_var + weight_var=weight_var, ) else: return _create_dense_column_weighted_sum( @@ -2402,19 +2276,14 @@ def _create_weighted_sum( units=units, weight_collections=weight_collections, trainable=trainable, - weight_var=weight_var + weight_var=weight_var, ) -def _create_dense_column_weighted_sum( - column, builder, units, weight_collections, trainable, weight_var=None -): +def _create_dense_column_weighted_sum(column, builder, units, weight_collections, trainable, weight_var=None): """Create a weighted sum of a dense column for linear_model.""" - tensor = column._get_dense_tensor( # pylint: disable=protected-access - builder, - weight_collections=weight_collections, - trainable=trainable) - num_elements = column._variable_shape.num_elements() # pylint: disable=protected-access + tensor = column._get_dense_tensor(builder, weight_collections=weight_collections, trainable=trainable) + num_elements = column._variable_shape.num_elements() batch_size = array_ops.shape(tensor)[0] tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements)) if weight_var is not None: @@ -2425,7 +2294,7 @@ def _create_dense_column_weighted_sum( shape=[num_elements, units], initializer=init_ops.zeros_initializer(), trainable=trainable, - collections=weight_collections + collections=weight_collections, ) return math_ops.matmul(tensor, weight, name='weighted_sum') @@ -2439,18 +2308,14 @@ class _CategoricalColumn(_FeatureColumn): A categorical feature typically handled with a `tf.SparseTensor` of IDs. """ - IdWeightPair = collections.namedtuple( # pylint: disable=invalid-name - 'IdWeightPair', ['id_tensor', 'weight_tensor']) + IdWeightPair = collections.namedtuple('IdWeightPair', ['id_tensor', 'weight_tensor']) @abc.abstractproperty def _num_buckets(self): """Returns number of buckets in this sparse feature.""" - pass @abc.abstractmethod - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): """Returns an IdWeightPair. `IdWeightPair` is a pair of `SparseTensor`s which represents ids and @@ -2471,7 +2336,6 @@ def _get_sparse_tensors( trainable: If `True` also add variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see `tf.compat.v1.get_variable`). """ - pass def _create_categorical_column_weighted_sum( @@ -2481,9 +2345,8 @@ def _create_categorical_column_weighted_sum( sparse_combiner, weight_collections, trainable, - weight_var=None + weight_var=None, ): - # pylint: disable=g-doc-return-or-yield,g-doc-args """Create a weighted sum of a categorical column for linear_model. Note to maintainer: As implementation details, the weighted sum is @@ -2510,51 +2373,39 @@ def _create_categorical_column_weighted_sum( For both cases, we can implement weighted sum via embedding_lookup with sparse_combiner = "sum". """ - sparse_tensors = column._get_sparse_tensors( # pylint: disable=protected-access - builder, - weight_collections=weight_collections, - trainable=trainable) - id_tensor = sparse_ops.sparse_reshape( - sparse_tensors.id_tensor, - [array_ops.shape(sparse_tensors.id_tensor)[0], -1] - ) + sparse_tensors = column._get_sparse_tensors(builder, weight_collections=weight_collections, trainable=trainable) + id_tensor = sparse_ops.sparse_reshape(sparse_tensors.id_tensor, [array_ops.shape(sparse_tensors.id_tensor)[0], -1]) weight_tensor = sparse_tensors.weight_tensor if weight_tensor is not None: - weight_tensor = sparse_ops.sparse_reshape( - weight_tensor, [array_ops.shape(weight_tensor)[0], -1] - ) + weight_tensor = sparse_ops.sparse_reshape(weight_tensor, [array_ops.shape(weight_tensor)[0], -1]) if weight_var is not None: weight = weight_var else: weight = variable_scope.get_variable( name='weights', - shape=(column._num_buckets, units), # pylint: disable=protected-access + shape=(column._num_buckets, units), initializer=init_ops.zeros_initializer(), trainable=trainable, - collections=weight_collections + collections=weight_collections, ) return embedding_ops.safe_embedding_lookup_sparse( weight, id_tensor, sparse_weights=weight_tensor, combiner=sparse_combiner, - name='weighted_sum' + name='weighted_sum', ) class _SequenceDenseColumn(_FeatureColumn): """Represents dense sequence data.""" - TensorSequenceLengthPair = collections.namedtuple( # pylint: disable=invalid-name - 'TensorSequenceLengthPair', ['dense_tensor', 'sequence_length']) + TensorSequenceLengthPair = collections.namedtuple('TensorSequenceLengthPair', ['dense_tensor', 'sequence_length']) @abc.abstractmethod - def _get_sequence_dense_tensor( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): """Returns a `TensorSequenceLengthPair`.""" - pass class _LazyBuilder(object): @@ -2631,14 +2482,11 @@ def get(self, key): raise ValueError('Feature {} is not in features dictionary.'.format(key)) if not isinstance(key, _FeatureColumn): - raise TypeError( - '"key" must be either a "str" or "_FeatureColumn". ' - 'Provided: {}'.format(key) - ) + raise TypeError('"key" must be either a "str" or "_FeatureColumn". ' 'Provided: {}'.format(key)) column = key logging.debug('Transforming feature_column %s.', column) - transformed = column._transform_feature(self) # pylint: disable=protected-access + transformed = column._transform_feature(self) if transformed is None: raise ValueError('Column {} is not supported.'.format(column.name)) self._feature_tensors[column] = transformed @@ -2666,27 +2514,19 @@ def _get_raw_feature_as_tensor(self, key): if 'RaggedTensor' in str(type(raw_feature)): return raw_feature - feature_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - raw_feature - ) + feature_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(raw_feature) def expand_dims(input_tensor): # Input_tensor must have rank 1. if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): - return sparse_ops.sparse_reshape( - input_tensor, [array_ops.shape(input_tensor)[0], 1] - ) + return sparse_ops.sparse_reshape(input_tensor, [array_ops.shape(input_tensor)[0], 1]) else: return array_ops.expand_dims(input_tensor, -1) rank = feature_tensor.get_shape().ndims if rank is not None: if rank == 0: - raise ValueError( - 'Feature (key: {}) cannot have rank 0. Give: {}'.format( - key, feature_tensor - ) - ) + raise ValueError('Feature (key: {}) cannot have rank 0. Give: {}'.format(key, feature_tensor)) return feature_tensor if rank != 1 else expand_dims(feature_tensor) # Handle dynamic rank. @@ -2694,15 +2534,14 @@ def expand_dims(input_tensor): [ check_ops.assert_positive( array_ops.rank(feature_tensor), - message='Feature (key: {}) cannot have rank 0. Given: {}'.format( - key, feature_tensor - ) + message='Feature (key: {}) cannot have rank 0. Given: {}'.format(key, feature_tensor), ) ] ): return control_flow_ops.cond( math_ops.equal(1, array_ops.rank(feature_tensor)), - lambda: expand_dims(feature_tensor), lambda: feature_tensor + lambda: expand_dims(feature_tensor), + lambda: feature_tensor, ) @@ -2737,16 +2576,16 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): Raises: ValueError: when `input_tensor`'s rank is `None`. """ - input_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - input_tensor - ) + input_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(input_tensor) if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): return input_tensor with ops.name_scope( - None, 'to_sparse_input', ( + None, + 'to_sparse_input', + ( input_tensor, ignore_value, - ) + ), ): if ignore_value is None: if input_tensor.dtype == dtypes.string: @@ -2759,18 +2598,12 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): # constructing a new numpy object of the given type, which yields the # default value for that type. ignore_value = input_tensor.dtype.as_numpy_dtype() - ignore_value = math_ops.cast( - ignore_value, input_tensor.dtype, name='ignore_value' - ) - indices = array_ops.where( - math_ops.not_equal(input_tensor, ignore_value), name='indices' - ) + ignore_value = math_ops.cast(ignore_value, input_tensor.dtype, name='ignore_value') + indices = array_ops.where(math_ops.not_equal(input_tensor, ignore_value), name='indices') return sparse_tensor_lib.SparseTensor( indices=indices, values=array_ops.gather_nd(input_tensor, indices, name='values'), - dense_shape=array_ops.shape( - input_tensor, out_type=dtypes.int64, name='dense_shape' - ) + dense_shape=array_ops.shape(input_tensor, out_type=dtypes.int64, name='dense_shape'), ) @@ -2802,8 +2635,7 @@ def _normalize_feature_columns(feature_columns): for column in feature_columns: if not isinstance(column, _FeatureColumn): raise ValueError( - 'Items of feature_columns must be a _FeatureColumn. ' - 'Given (type {}): {}.'.format(type(column), column) + 'Items of feature_columns must be a _FeatureColumn. ' 'Given (type {}): {}.'.format(type(column), column) ) if not feature_columns: raise ValueError('feature_columns must not be empty.') @@ -2824,10 +2656,7 @@ def _normalize_feature_columns(feature_columns): class _NumericColumn( _DenseColumn, - collections.namedtuple( - '_NumericColumn', - ['key', 'shape', 'default_value', 'dtype', 'normalizer_fn'] - ) + collections.namedtuple('_NumericColumn', ['key', 'shape', 'default_value', 'dtype', 'normalizer_fn']), ): """See `numeric_column`.""" @@ -2837,10 +2666,7 @@ def name(self): @property def _parse_example_spec(self): - return { - self.key: - parsing_ops.FixedLenFeature(self.shape, self.dtype, self.default_value) - } + return {self.key: parsing_ops.FixedLenFeature(self.shape, self.dtype, self.default_value)} def _transform_feature(self, inputs): input_tensor = inputs.get(self.key) @@ -2880,8 +2706,9 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): class _BucketizedColumn( - _DenseColumn, _CategoricalColumn, - collections.namedtuple('_BucketizedColumn', ['source_column', 'boundaries']) + _DenseColumn, + _CategoricalColumn, + collections.namedtuple('_BucketizedColumn', ['source_column', 'boundaries']), ): """See `bucketized_column`.""" @@ -2895,19 +2722,15 @@ def raw_name(self): @property def _parse_example_spec(self): - return self.source_column._parse_example_spec # pylint: disable=protected-access + return self.source_column._parse_example_spec def _transform_feature(self, inputs): source_tensor = inputs.get(self.source_column) - return math_ops._bucketize( # pylint: disable=protected-access - source_tensor, - boundaries=self.boundaries) + return math_ops._bucketize(source_tensor, boundaries=self.boundaries) @property def _variable_shape(self): - return tensor_shape.TensorShape( - tuple(self.source_column.shape) + (len(self.boundaries) + 1, ) - ) + return tensor_shape.TensorShape(tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections @@ -2916,8 +2739,8 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): return array_ops.one_hot( indices=math_ops.cast(input_tensor, dtypes.int64), depth=len(self.boundaries) + 1, - on_value=1., - off_value=0. + on_value=1.0, + off_value=0.0, ) @property @@ -2925,9 +2748,7 @@ def _num_buckets(self): # By construction, source_column is always one-dimensional. return (len(self.boundaries) + 1) * self.source_column.shape[0] - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): """Converts dense inputs to SparseTensor so downstream code can use it.""" input_tensor = inputs.get(self) batch_size = array_ops.shape(input_tensor)[0] @@ -2937,36 +2758,37 @@ def _get_sparse_tensors( i1 = array_ops.reshape( array_ops.tile( array_ops.expand_dims(math_ops.range(0, batch_size), 1), - [1, source_dimension] - ), (-1, ) + [1, source_dimension], + ), + (-1,), ) i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size]) # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets - bucket_indices = ( - array_ops.reshape(input_tensor, (-1, )) + (len(self.boundaries) + 1) * i2 - ) + bucket_indices = array_ops.reshape(input_tensor, (-1,)) + (len(self.boundaries) + 1) * i2 - indices = math_ops.cast( - array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64 - ) - dense_shape = math_ops.cast( - array_ops.stack([batch_size, source_dimension]), dtypes.int64 - ) - sparse_tensor = sparse_tensor_lib.SparseTensor( - indices=indices, values=bucket_indices, dense_shape=dense_shape - ) + indices = math_ops.cast(array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64) + dense_shape = math_ops.cast(array_ops.stack([batch_size, source_dimension]), dtypes.int64) + sparse_tensor = sparse_tensor_lib.SparseTensor(indices=indices, values=bucket_indices, dense_shape=dense_shape) return _CategoricalColumn.IdWeightPair(sparse_tensor, None) class _EmbeddingColumn( - _DenseColumn, _SequenceDenseColumn, + _DenseColumn, + _SequenceDenseColumn, collections.namedtuple( - '_EmbeddingColumn', ( - 'categorical_column', 'dimension', 'combiner', 'layer_creator', - 'ckpt_to_load_from', 'tensor_name_in_ckpt', 'max_norm', 'trainable' - ) - ) + '_EmbeddingColumn', + ( + 'categorical_column', + 'dimension', + 'combiner', + 'layer_creator', + 'ckpt_to_load_from', + 'tensor_name_in_ckpt', + 'max_norm', + 'trainable', + ), + ), ): """See `embedding_column`.""" @@ -2978,7 +2800,7 @@ def name(self): @property def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec def _transform_feature(self, inputs): return inputs.get(self.categorical_column) @@ -2989,30 +2811,25 @@ def _variable_shape(self): self._shape = tensor_shape.TensorShape([self.dimension]) return self._shape - def _get_dense_tensor_internal( - self, inputs, weight_collections=None, trainable=None - ): + def _get_dense_tensor_internal(self, inputs, weight_collections=None, trainable=None): """Private method that follows the signature of _get_dense_tensor.""" # Get sparse IDs and weights. - sparse_tensors = self.categorical_column._get_sparse_tensors( # pylint: disable=protected-access - inputs, - weight_collections=weight_collections, - trainable=trainable) + sparse_tensors = self.categorical_column._get_sparse_tensors( + inputs, weight_collections=weight_collections, trainable=trainable + ) sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor embedding_weights = self.layer_creator( weight_collections=weight_collections, - scope=variable_scope.get_variable_scope() + scope=variable_scope.get_variable_scope(), ) if self.ckpt_to_load_from is not None: to_restore = embedding_weights if isinstance(to_restore, variables.PartitionedVariable): - to_restore = to_restore._get_variable_list() # pylint: disable=protected-access - checkpoint_utils.init_from_checkpoint( - self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore} - ) + to_restore = to_restore._get_variable_list() + checkpoint_utils.init_from_checkpoint(self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) # Return embedding lookup result. return embedding_ops.safe_embedding_lookup_sparse( @@ -3021,7 +2838,7 @@ def _get_dense_tensor_internal( sparse_weights=sparse_weights, combiner=self.combiner, name='%s_weights' % self.name, - max_norm=self.max_norm + max_norm=self.max_norm, ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): @@ -3033,42 +2850,27 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): 'non-sequence categorical_column_with_*. ' 'Suggested fix B: If you wish to create sequence input, use ' 'sequence_input_layer instead of input_layer. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) - return self._get_dense_tensor_internal( - inputs=inputs, - weight_collections=weight_collections, - trainable=trainable - ) + return self._get_dense_tensor_internal(inputs=inputs, weight_collections=weight_collections, trainable=trainable) - def _get_sequence_dense_tensor( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): if not isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( 'In embedding_column: {}. ' 'categorical_column must be of type _SequenceCategoricalColumn ' 'to use sequence_input_layer. ' 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) - dense_tensor = self._get_dense_tensor_internal( # pylint: disable=protected-access - inputs=inputs, - weight_collections=weight_collections, - trainable=trainable) - - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access - sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor - ) - return _SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length + dense_tensor = self._get_dense_tensor_internal( + inputs=inputs, weight_collections=weight_collections, trainable=trainable ) + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) + sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) + return _SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) + def _get_graph_for_variable(var): if isinstance(var, variables.PartitionedVariable): @@ -3078,15 +2880,24 @@ def _get_graph_for_variable(var): class _SharedEmbeddingColumn( - _DenseColumn, _SequenceDenseColumn, + _DenseColumn, + _SequenceDenseColumn, collections.namedtuple( - '_SharedEmbeddingColumn', ( - 'categorical_column', 'dimension', 'combiner', 'initializer', - 'shared_embedding_collection_name', 'ckpt_to_load_from', - 'tensor_name_in_ckpt', 'max_norm', 'trainable', 'partitioner', - 'ev_params' - ) - ) + '_SharedEmbeddingColumn', + ( + 'categorical_column', + 'dimension', + 'combiner', + 'initializer', + 'shared_embedding_collection_name', + 'ckpt_to_load_from', + 'tensor_name_in_ckpt', + 'max_norm', + 'trainable', + 'partitioner', + 'ev_params', + ), + ), ): """See `embedding_column`.""" @@ -3106,7 +2917,7 @@ def _var_scope_name(self): @property def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec def _transform_feature(self, inputs): return inputs.get(self.categorical_column) @@ -3117,26 +2928,21 @@ def _variable_shape(self): self._shape = tensor_shape.TensorShape([self.dimension]) return self._shape - def _get_dense_tensor_internal( - self, inputs, weight_collections=None, trainable=None - ): + def _get_dense_tensor_internal(self, inputs, weight_collections=None, trainable=None): """Private method that follows the signature of _get_dense_tensor.""" # This method is called from a variable_scope with name _var_scope_name, # which is shared among all shared embeddings. Open a name_scope here, so # that the ops for different columns have distinct names. with ops.name_scope(None, default_name=self.name): # Get sparse IDs and weights. - sparse_tensors = self.categorical_column._get_sparse_tensors( # pylint: disable=protected-access - inputs, - weight_collections=weight_collections, - trainable=trainable) + sparse_tensors = self.categorical_column._get_sparse_tensors( + inputs, weight_collections=weight_collections, trainable=trainable + ) sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor - embedding_shape = (self.categorical_column._num_buckets, self.dimension) # pylint: disable=protected-access - shared_embedding_collection = ops.get_collection( - self.shared_embedding_collection_name - ) + embedding_shape = (self.categorical_column._num_buckets, self.dimension) + shared_embedding_collection = ops.get_collection(self.shared_embedding_collection_name) if shared_embedding_collection: if len(shared_embedding_collection) > 1: raise ValueError( @@ -3147,8 +2953,7 @@ def _get_dense_tensor_internal( 'hood.'.format(shared_embedding_collection) ) embedding_weights = shared_embedding_collection[0] - if embedding_weights.get_shape( - ) != embedding_shape and not self.ev_params is not None: # noqa : E714 + if embedding_weights.get_shape() != embedding_shape and self.ev_params is None: raise ValueError( 'Shared embedding collection {} contains variable {} of ' 'unexpected shape {}. Expected shape is {}. ' @@ -3156,8 +2961,10 @@ def _get_dense_tensor_internal( 'Suggested fix B: Do not add any variables to this collection. ' 'The feature_column library already adds a variable under the ' 'hood.'.format( - self.shared_embedding_collection_name, embedding_weights.name, - embedding_weights.get_shape(), embedding_shape + self.shared_embedding_collection_name, + embedding_weights.name, + embedding_weights.get_shape(), + embedding_shape, ) ) else: @@ -3169,28 +2976,23 @@ def _get_dense_tensor_internal( initializer=self.initializer, trainable=self.trainable and trainable, partitioner=self.partitioner, - collections=weight_collections + collections=weight_collections, ) else: # at eval or inference time, it is necessary to set # the initializers to zeros, so that new key will # get zero embedding - if os.environ.get('tf.estimator.mode', '') != \ - os.environ.get('tf.estimator.ModeKeys.TRAIN', 'train'): + if os.environ.get('tf.estimator.mode', '') != os.environ.get('tf.estimator.ModeKeys.TRAIN', 'train'): initializer = init_ops.zeros_initializer() else: initializer = self.initializer extra_args = {} if 'EmbeddingVariableConfig' in dir(variables): ev_option = variables.EmbeddingVariableOption() - ev_option.filter_strategy = variables.CounterFilter( - filter_freq=self.ev_params.filter_freq - ) + ev_option.filter_strategy = variables.CounterFilter(filter_freq=self.ev_params.filter_freq) extra_args['ev_option'] = ev_option else: - extra_args['filter_options'] = variables.CounterFilterOptions( - self.ev_params.filter_freq - ) + extra_args['filter_options'] = variables.CounterFilterOptions(self.ev_params.filter_freq) embedding_weights = variable_scope.get_embedding_variable( name='embedding_weights', embedding_dim=self.dimension, @@ -3199,19 +3001,15 @@ def _get_dense_tensor_internal( partitioner=self.partitioner, collections=weight_collections, steps_to_live=self.ev_params.steps_to_live, - **extra_args + **extra_args, ) - ops.add_to_collection( - self.shared_embedding_collection_name, embedding_weights - ) + ops.add_to_collection(self.shared_embedding_collection_name, embedding_weights) if self.ckpt_to_load_from is not None: to_restore = embedding_weights if isinstance(to_restore, variables.PartitionedVariable): - to_restore = to_restore._get_variable_list() # pylint: disable=protected-access - checkpoint_utils.init_from_checkpoint( - self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore} - ) + to_restore = to_restore._get_variable_list() + checkpoint_utils.init_from_checkpoint(self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) if 'RaggedTensor' in str(type(sparse_ids)): assert sparse_weights is None @@ -3221,7 +3019,7 @@ def _get_dense_tensor_internal( ragged_weights=sparse_weights, combiner=self.combiner, max_norm=self.max_norm, - name='%s_weights' % self.name + name='%s_weights' % self.name, ) # Return embedding lookup result. @@ -3231,7 +3029,7 @@ def _get_dense_tensor_internal( sparse_weights=sparse_weights, combiner=self.combiner, name='%s_weights' % self.name, - max_norm=self.max_norm + max_norm=self.max_norm, ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): @@ -3243,40 +3041,25 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): 'non-sequence categorical_column_with_*. ' 'Suggested fix B: If you wish to create sequence input, use ' 'sequence_input_layer instead of input_layer. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) - return self._get_dense_tensor_internal( - inputs=inputs, - weight_collections=weight_collections, - trainable=trainable - ) + return self._get_dense_tensor_internal(inputs=inputs, weight_collections=weight_collections, trainable=trainable) - def _get_sequence_dense_tensor( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): if not isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( 'In embedding_column: {}. ' 'categorical_column must be of type _SequenceCategoricalColumn ' 'to use sequence_input_layer. ' 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) - dense_tensor = self._get_dense_tensor_internal( # pylint: disable=protected-access - inputs=inputs, - weight_collections=weight_collections, - trainable=trainable) - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access - sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor - ) - return _SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length + dense_tensor = self._get_dense_tensor_internal( + inputs=inputs, weight_collections=weight_collections, trainable=trainable ) + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) + sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) + return _SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) def _check_shape(shape, key): @@ -3287,23 +3070,15 @@ def _check_shape(shape, key): shape = tuple(shape) for dimension in shape: if not isinstance(dimension, six.integer_types): - raise TypeError( - 'shape dimensions must be integer. ' - 'shape: {}, key: {}'.format(shape, key) - ) + raise TypeError('shape dimensions must be integer. ' 'shape: {}, key: {}'.format(shape, key)) if dimension < 1: - raise ValueError( - 'shape dimensions must be greater than 0. ' - 'shape: {}, key: {}'.format(shape, key) - ) + raise ValueError('shape dimensions must be greater than 0. ' 'shape: {}, key: {}'.format(shape, key)) return shape class _HashedCategoricalColumn( _CategoricalColumn, - collections.namedtuple( - '_HashedCategoricalColumn', ['key', 'hash_bucket_size', 'dtype'] - ) + collections.namedtuple('_HashedCategoricalColumn', ['key', 'hash_bucket_size', 'dtype']), ): """See `categorical_column_with_hash_bucket`.""" @@ -3320,23 +3095,16 @@ def _parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values( - inputs.get(self.key) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) if not isinstance(input_tensor, sparse_tensor_lib.SparseTensor): raise ValueError('SparseColumn input must be a SparseTensor.') - fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key) - ) + fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype - ) + 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) ) if self.dtype == dtypes.string: @@ -3344,32 +3112,31 @@ def _transform_feature(self, inputs): else: sparse_values = string_ops.as_string(input_tensor.values) - sparse_id_values = string_ops.string_to_hash_bucket_fast( - sparse_values, self.hash_bucket_size, name='lookup' - ) - return sparse_tensor_lib.SparseTensor( - input_tensor.indices, sparse_id_values, input_tensor.dense_shape - ) + sparse_id_values = string_ops.string_to_hash_bucket_fast(sparse_values, self.hash_bucket_size, name='lookup') + return sparse_tensor_lib.SparseTensor(input_tensor.indices, sparse_id_values, input_tensor.dense_shape) @property def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.hash_bucket_size - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) class _VocabularyFileCategoricalColumn( _CategoricalColumn, collections.namedtuple( - '_VocabularyFileCategoricalColumn', ( - 'key', 'vocabulary_file', 'vocabulary_size', 'num_oov_buckets', 'dtype', - 'default_value' - ) - ) + '_VocabularyFileCategoricalColumn', + ( + 'key', + 'vocabulary_file', + 'vocabulary_size', + 'num_oov_buckets', + 'dtype', + 'default_value', + ), + ), ): """See `categorical_column_with_vocabulary_file`.""" @@ -3382,22 +3149,15 @@ def _parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values( - inputs.get(self.key) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype - ) + 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) ) - fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key) - ) + fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -3411,7 +3171,7 @@ def _transform_feature(self, inputs): vocab_size=self.vocabulary_size, default_value=self.default_value, key_dtype=key_dtype, - name='{}_lookup'.format(self.key) + name='{}_lookup'.format(self.key), ).lookup(input_tensor) @property @@ -3419,9 +3179,7 @@ def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.vocabulary_size + self.num_oov_buckets - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -3429,8 +3187,8 @@ class _VocabularyListCategoricalColumn( _CategoricalColumn, collections.namedtuple( '_VocabularyListCategoricalColumn', - ('key', 'vocabulary_list', 'dtype', 'default_value', 'num_oov_buckets') - ) + ('key', 'vocabulary_list', 'dtype', 'default_value', 'num_oov_buckets'), + ), ): """See `categorical_column_with_vocabulary_list`.""" @@ -3443,22 +3201,15 @@ def _parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values( - inputs.get(self.key) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype - ) + 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) ) - fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key) - ) + fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -3471,7 +3222,7 @@ def _transform_feature(self, inputs): default_value=self.default_value, num_oov_buckets=self.num_oov_buckets, dtype=key_dtype, - name='{}_lookup'.format(self.key) + name='{}_lookup'.format(self.key), ).lookup(input_tensor) @property @@ -3479,17 +3230,13 @@ def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return len(self.vocabulary_list) + self.num_oov_buckets - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) class _IdentityCategoricalColumn( _CategoricalColumn, - collections.namedtuple( - '_IdentityCategoricalColumn', ('key', 'num_buckets', 'default_value') - ) + collections.namedtuple('_IdentityCategoricalColumn', ('key', 'num_buckets', 'default_value')), ): """See `categorical_column_with_identity`.""" @@ -3502,21 +3249,13 @@ def _parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(dtypes.int64)} def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values( - inputs.get(self.key) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) if not input_tensor.dtype.is_integer: - raise ValueError( - 'Invalid input, not integer. key: {} dtype: {}'.format( - self.key, input_tensor.dtype - ) - ) + raise ValueError('Invalid input, not integer. key: {} dtype: {}'.format(self.key, input_tensor.dtype)) values = math_ops.cast(input_tensor.values, dtypes.int64, name='values') - num_buckets = math_ops.cast( - self.num_buckets, dtypes.int64, name='num_buckets' - ) + num_buckets = math_ops.cast(self.num_buckets, dtypes.int64, name='num_buckets') zero = math_ops.cast(0, dtypes.int64, name='zero') if self.default_value is None: # Fail if values are out-of-range. @@ -3524,30 +3263,27 @@ def _transform_feature(self, inputs): values, num_buckets, data=(values, num_buckets), - name='assert_less_than_num_buckets' - ) - assert_greater = check_ops.assert_greater_equal( - values, zero, data=(values, ), name='assert_greater_or_equal_0' + name='assert_less_than_num_buckets', ) + assert_greater = check_ops.assert_greater_equal(values, zero, data=(values,), name='assert_greater_or_equal_0') with ops.control_dependencies((assert_less, assert_greater)): values = array_ops.identity(values) else: # Assign default for out-of-range values. values = array_ops.where( - math_ops.logical_or( - values < zero, values >= num_buckets, name='out_of_range' - ), + math_ops.logical_or(values < zero, values >= num_buckets, name='out_of_range'), array_ops.fill( dims=array_ops.shape(values), value=math_ops.cast(self.default_value, dtypes.int64), - name='default_values' - ), values + name='default_values', + ), + values, ) return sparse_tensor_lib.SparseTensor( indices=input_tensor.indices, values=values, - dense_shape=input_tensor.dense_shape + dense_shape=input_tensor.dense_shape, ) @property @@ -3555,9 +3291,7 @@ def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.num_buckets - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -3565,58 +3299,44 @@ class _WeightedCategoricalColumn( _CategoricalColumn, collections.namedtuple( '_WeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype') - ) + ('categorical_column', 'weight_feature_key', 'dtype'), + ), ): """See `weighted_categorical_column`.""" @property def name(self): - return '{}_weighted_by_{}'.format( - self.categorical_column.name, self.weight_feature_key - ) + return '{}_weighted_by_{}'.format(self.categorical_column.name, self.weight_feature_key) @property def _parse_example_spec(self): - config = self.categorical_column._parse_example_spec # pylint: disable=protected-access + config = self.categorical_column._parse_example_spec if self.weight_feature_key in config: raise ValueError( - 'Parse config {} already exists for {}.'.format( - config[self.weight_feature_key], self.weight_feature_key - ) + 'Parse config {} already exists for {}.'.format(config[self.weight_feature_key], self.weight_feature_key) ) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @property def _num_buckets(self): - return self.categorical_column._num_buckets # pylint: disable=protected-access + return self.categorical_column._num_buckets def _transform_feature(self, inputs): weight_tensor = inputs.get(self.weight_feature_key) if weight_tensor is None: raise ValueError('Missing weights {}.'.format(self.weight_feature_key)) - weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - weight_tensor - ) + weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(weight_tensor) if self.dtype != weight_tensor.dtype.base_dtype: - raise ValueError( - 'Bad dtype, expected {}, but got {}.'.format( - self.dtype, weight_tensor.dtype - ) - ) + raise ValueError('Bad dtype, expected {}, but got {}.'.format(self.dtype, weight_tensor.dtype)) if not isinstance(weight_tensor, sparse_tensor_lib.SparseTensor): # The weight tensor can be a regular Tensor. In this case, sparsify it. - weight_tensor = _to_sparse_input_and_drop_ignore_values( - weight_tensor, ignore_value=0.0 - ) + weight_tensor = _to_sparse_input_and_drop_ignore_values(weight_tensor, ignore_value=0.0) if not weight_tensor.dtype.is_floating: weight_tensor = math_ops.cast(weight_tensor, dtypes.float32) return (inputs.get(self.categorical_column), weight_tensor) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable tensors = inputs.get(self) @@ -3625,9 +3345,7 @@ def _get_sparse_tensors( class _CrossedColumn( _CategoricalColumn, - collections.namedtuple( - '_CrossedColumn', ['keys', 'hash_bucket_size', 'hash_key'] - ) + collections.namedtuple('_CrossedColumn', ['keys', 'hash_bucket_size', 'hash_key']), ): """See `crossed_column`.""" @@ -3646,7 +3364,7 @@ def _parse_example_spec(self): config = {} for key in self.keys: if isinstance(key, _FeatureColumn): - config.update(key._parse_example_spec) # pylint: disable=protected-access + config.update(key._parse_example_spec) else: # key must be a string config.update({key: parsing_ops.VarLenFeature(dtypes.string)}) return config @@ -3657,7 +3375,7 @@ def _transform_feature(self, inputs): if isinstance(key, six.string_types): feature_tensors.append(inputs.get(key)) elif isinstance(key, _CategoricalColumn): - ids_and_weights = key._get_sparse_tensors(inputs) # pylint: disable=protected-access + ids_and_weights = key._get_sparse_tensors(inputs) if ids_and_weights.weight_tensor is not None: raise ValueError( 'crossed_column does not support weight_tensor, but the given ' @@ -3670,7 +3388,7 @@ def _transform_feature(self, inputs): return sparse_ops.sparse_cross_hashed( inputs=feature_tensors, num_buckets=self.hash_bucket_size, - hash_key=self.hash_key + hash_key=self.hash_key, ) @property @@ -3678,9 +3396,7 @@ def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.hash_bucket_size - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -3703,8 +3419,9 @@ def _collect_leaf_level_keys(cross): class _IndicatorColumn( - _DenseColumn, _SequenceDenseColumn, - collections.namedtuple('_IndicatorColumn', ['categorical_column']) + _DenseColumn, + _SequenceDenseColumn, + collections.namedtuple('_IndicatorColumn', ['categorical_column']), ): """Represents a one-hot column for use in deep networks. @@ -3729,7 +3446,7 @@ def _transform_feature(self, inputs): Raises: ValueError: if input rank is not known at graph building time. """ - id_weight_pair = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + id_weight_pair = self.categorical_column._get_sparse_tensors(inputs) id_tensor = id_weight_pair.id_tensor weight_tensor = id_weight_pair.weight_tensor @@ -3738,43 +3455,35 @@ def _transform_feature(self, inputs): weighted_column = sparse_ops.sparse_merge( sp_ids=id_tensor, sp_values=weight_tensor, - vocab_size=int(self._variable_shape[-1]) + vocab_size=int(self._variable_shape[-1]), ) # Remove (?, -1) index. - weighted_column = sparse_ops.sparse_slice( - weighted_column, [0, 0], weighted_column.dense_shape - ) + weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0], weighted_column.dense_shape) # Use scatter_nd to merge duplicated indices if existed, # instead of sparse_tensor_to_dense. return array_ops.scatter_nd( - weighted_column.indices, weighted_column.values, - weighted_column.dense_shape + weighted_column.indices, + weighted_column.values, + weighted_column.dense_shape, ) - dense_id_tensor = sparse_ops.sparse_tensor_to_dense( - id_tensor, default_value=-1 - ) + dense_id_tensor = sparse_ops.sparse_tensor_to_dense(id_tensor, default_value=-1) # One hot must be float for tf.concat reasons since all other inputs to # input_layer are float32. - one_hot_id_tensor = array_ops.one_hot( - dense_id_tensor, - depth=self._variable_shape[-1], - on_value=1.0, - off_value=0.0 - ) + one_hot_id_tensor = array_ops.one_hot(dense_id_tensor, depth=self._variable_shape[-1], on_value=1.0, off_value=0.0) # Reduce to get a multi-hot per example. return math_ops.reduce_sum(one_hot_id_tensor, axis=[-2]) @property def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec @property def _variable_shape(self): """Returns a `TensorShape` representing the shape of the dense `Tensor`.""" - return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) # pylint: disable=protected-access + return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): """Returns dense `Tensor` representing feature. @@ -3804,17 +3513,13 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): 'non-sequence categorical_column_with_*. ' 'Suggested fix B: If you wish to create sequence input, use ' 'sequence_input_layer instead of input_layer. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) # Feature has been already transformed. Return the intermediate # representation created by _transform_feature. return inputs.get(self) - def _get_sequence_dense_tensor( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): # Do nothing with weight_collections and trainable since no variables are # created in this function. del weight_collections @@ -3825,20 +3530,14 @@ def _get_sequence_dense_tensor( 'categorical_column must be of type _SequenceCategoricalColumn ' 'to use sequence_input_layer. ' 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) # Feature has been already transformed. Return the intermediate # representation created by _transform_feature. dense_tensor = inputs.get(self) - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access - sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor - ) - return _SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length - ) + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) + sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) + return _SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) def _verify_static_batch_size_equality(tensors, columns): @@ -3859,21 +3558,21 @@ def _verify_static_batch_size_equality(tensors, columns): if expected_batch_size is None: bath_size_column_index = i expected_batch_size = tensors[i].shape.dims[0] - elif not expected_batch_size.is_compatible_with( - tensors[i].shape.dims[0] - ): + elif not expected_batch_size.is_compatible_with(tensors[i].shape.dims[0]): raise ValueError( 'Batch size (first dimension) of each feature must be same. ' 'Batch size of columns ({}, {}): ({}, {})'.format( - columns[bath_size_column_index].name, columns[i].name, - expected_batch_size, tensors[i].shape.dims[0] + columns[bath_size_column_index].name, + columns[i].name, + expected_batch_size, + tensors[i].shape.dims[0], ) ) class _SequenceCategoricalColumn( _CategoricalColumn, - collections.namedtuple('_SequenceCategoricalColumn', ['categorical_column']) + collections.namedtuple('_SequenceCategoricalColumn', ['categorical_column']), ): """Represents sequences of categorical data.""" @@ -3883,19 +3582,17 @@ def name(self): @property def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec def _transform_feature(self, inputs): - return self.categorical_column._transform_feature(inputs) # pylint: disable=protected-access + return self.categorical_column._transform_feature(inputs) @property def _num_buckets(self): - return self.categorical_column._num_buckets # pylint: disable=protected-access + return self.categorical_column._num_buckets - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) id_tensor = sparse_tensors.id_tensor weight_tensor = sparse_tensors.weight_tensor diff --git a/easy_rec/python/compat/feature_column/feature_column_v2.py b/easy_rec/python/compat/feature_column/feature_column_v2.py index 2bb8aa2e2..7772f60a2 100644 --- a/easy_rec/python/compat/feature_column/feature_column_v2.py +++ b/easy_rec/python/compat/feature_column/feature_column_v2.py @@ -139,31 +139,45 @@ from tensorflow.python.framework import dtypes, ops from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib from tensorflow.python.framework import tensor_shape + # TODO(b/118385027): Dependency on keras can be problematic if Keras moves out # of the main repo. from tensorflow.python.keras import utils from tensorflow.python.keras.engine import training from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.ops import ( # NOQA + array_ops, + check_ops, + control_flow_ops, + embedding_ops, + init_ops, + lookup_ops, + math_ops, + nn_ops, + parsing_ops, + sparse_ops, + string_ops, + variable_scope, + variables, +) from tensorflow.python.platform import gfile from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import checkpoint_utils + # from tensorflow.python.training.tracking import tracking from tensorflow.python.util import deprecation, nest from easy_rec.python.compat import ops as compat_ops from easy_rec.python.compat.feature_column import feature_column as fc_old from easy_rec.python.compat.feature_column import utils as fc_utils +from easy_rec.python.compat.feature_column.feature_column import ( # NOQA + embedding_lookup_ragged, +) from easy_rec.python.layers import utils as layer_utils -from tensorflow.python.ops import array_ops, check_ops, control_flow_ops, embedding_ops, init_ops, lookup_ops, math_ops, nn_ops, parsing_ops, sparse_ops, string_ops, variable_scope, variables # NOQA - -from easy_rec.python.compat.feature_column.feature_column import embedding_lookup_ragged # NOQA - _FEATURE_COLUMN_DEPRECATION_DATE = None _FEATURE_COLUMN_DEPRECATION = ( - 'The old _FeatureColumn APIs are being ' - 'deprecated. Please use the new FeatureColumn ' - 'APIs instead.' + 'The old _FeatureColumn APIs are being ' 'deprecated. Please use the new FeatureColumn ' 'APIs instead.' ) if os.getenv('SAFE_EMBEDDING', 'TRUE') == 'TRUE': @@ -189,7 +203,7 @@ def create_variable( dtype=None, trainable=True, use_resource=True, - initializer=None + initializer=None, ): """Creates a new variable. @@ -280,7 +294,7 @@ def create_variable( dtype=None, trainable=True, use_resource=True, - initializer=None + initializer=None, ): if name in self._cols_to_vars_map[feature_column]: raise ValueError('Variable already exists.') @@ -295,7 +309,7 @@ def create_variable( # TODO(rohanj): Get rid of this hack once we have a mechanism for # specifying a default partitioner for an entire layer. In that case, # the default getter for Layers should work. - getter=variable_scope.get_variable + getter=variable_scope.get_variable, ) self._cols_to_vars_map[feature_column][name] = var return var @@ -325,11 +339,8 @@ class _BaseFeaturesLayer(Layer): `expected_column_type`. """ - def __init__( - self, feature_columns, expected_column_type, trainable, name, **kwargs - ): - super(_BaseFeaturesLayer, - self).__init__(name=name, trainable=trainable, **kwargs) + def __init__(self, feature_columns, expected_column_type, trainable, name, **kwargs): + super(_BaseFeaturesLayer, self).__init__(name=name, trainable=trainable, **kwargs) self._feature_columns = _normalize_feature_columns(feature_columns) self._state_manager = _StateManagerImpl(self, self.trainable) for column in self._feature_columns: @@ -337,15 +348,13 @@ def __init__( raise ValueError( 'Items of feature_columns must be a {}. ' 'You can wrap a categorical column with an ' - 'embedding_column or indicator_column. Given: {}'.format( - expected_column_type, column - ) + 'embedding_column or indicator_column. Given: {}'.format(expected_column_type, column) ) def build(self, _): for column in self._feature_columns: - with variable_scope._pure_variable_scope(self.name): # pylint: disable=protected-access - with variable_scope._pure_variable_scope(column.name): # pylint: disable=protected-access + with variable_scope._pure_variable_scope(self.name): + with variable_scope._pure_variable_scope(column.name): column.create_state(self._state_manager) super(_BaseFeaturesLayer, self).build(None) @@ -435,7 +444,7 @@ def __init__(self, feature_columns, trainable=True, name=None, **kwargs): trainable=trainable, name=name, expected_column_type=DenseColumn, - **kwargs + **kwargs, ) @property @@ -465,16 +474,12 @@ def call(self, features, cols_to_output_tensors=None): ValueError: If features are not a dictionary. """ if not isinstance(features, dict): - raise ValueError( - 'We expected a dictionary here. Instead we got: ', features - ) + raise ValueError('We expected a dictionary here. Instead we got: ', features) transformation_cache = FeatureTransformationCache(features) output_tensors = [] for column in self._feature_columns: with ops.name_scope(column.name): - tensor = column.get_dense_tensor( - transformation_cache, self._state_manager - ) + tensor = column.get_dense_tensor(transformation_cache, self._state_manager) processed_tensors = self._process_dense_tensor(column, tensor) if cols_to_output_tensors is not None: cols_to_output_tensors[column] = processed_tensors @@ -492,17 +497,15 @@ def __init__( sparse_combiner='sum', trainable=True, name=None, - **kwargs + **kwargs, ): - super(_LinearModelLayer, - self).__init__(name=name, trainable=trainable, **kwargs) + super(_LinearModelLayer, self).__init__(name=name, trainable=trainable, **kwargs) self._feature_columns = _normalize_feature_columns(feature_columns) for column in self._feature_columns: if not isinstance(column, (DenseColumn, CategoricalColumn)): raise ValueError( - 'Items of feature_columns must be either a ' - 'DenseColumn or CategoricalColumn. Given: {}'.format(column) + 'Items of feature_columns must be either a ' 'DenseColumn or CategoricalColumn. Given: {}'.format(column) ) self._units = units @@ -516,9 +519,9 @@ def build(self, _): # information to percolate down. We also use _pure_variable_scope's here # since we want to open up a name_scope in the `call` method while creating # the ops. - with variable_scope._pure_variable_scope(self.name): # pylint: disable=protected-access + with variable_scope._pure_variable_scope(self.name): for column in self._feature_columns: - with variable_scope._pure_variable_scope(column.name): # pylint: disable=protected-access + with variable_scope._pure_variable_scope(column.name): # Create the state for each feature column column.create_state(self._state_manager) @@ -533,7 +536,7 @@ def build(self, _): dtype=dtypes.float32, shape=(first_dim, self._units), initializer=init_ops.zeros_initializer(), - trainable=self.trainable + trainable=self.trainable, ) # Create a bias variable. @@ -547,16 +550,14 @@ def build(self, _): # TODO(rohanj): Get rid of this hack once we have a mechanism for # specifying a default partitioner for an entire layer. In that case, # the default getter for Layers should work. - getter=variable_scope.get_variable + getter=variable_scope.get_variable, ) super(_LinearModelLayer, self).build(None) def call(self, features): if not isinstance(features, dict): - raise ValueError( - 'We expected a dictionary here. Instead we got: {}'.format(features) - ) + raise ValueError('We expected a dictionary here. Instead we got: {}'.format(features)) with ops.name_scope(self.name): transformation_cache = FeatureTransformationCache(features) weighted_sums = [] @@ -571,17 +572,13 @@ def call(self, features): transformation_cache=transformation_cache, state_manager=self._state_manager, sparse_combiner=self._sparse_combiner, - weight_var=weight_var + weight_var=weight_var, ) weighted_sums.append(weighted_sum) _verify_static_batch_size_equality(weighted_sums, self._feature_columns) - predictions_no_bias = math_ops.add_n( - weighted_sums, name='weighted_sum_no_bias' - ) - predictions = nn_ops.bias_add( - predictions_no_bias, self.bias, name='weighted_sum' - ) + predictions_no_bias = math_ops.add_n(weighted_sums, name='weighted_sum_no_bias') + predictions = nn_ops.bias_add(predictions_no_bias, self.bias, name='weighted_sum') return predictions @@ -631,7 +628,7 @@ def __init__( sparse_combiner='sum', trainable=True, name=None, - **kwargs + **kwargs, ): """Constructs a LinearLayer. @@ -690,14 +687,7 @@ def __init__( nor `CategoricalColumn`. """ super(LinearModel, self).__init__(name=name, **kwargs) - self.layer = _LinearModelLayer( - feature_columns, - units, - sparse_combiner, - trainable, - name=self.name, - **kwargs - ) + self.layer = _LinearModelLayer(feature_columns, units, sparse_combiner, trainable, name=self.name, **kwargs) def call(self, features): """Returns a `Tensor` the represents the predictions of a linear model. @@ -758,9 +748,7 @@ def _transform_features_v2(features, feature_columns, state_manager): """ feature_columns = _normalize_feature_columns(feature_columns) outputs = {} - with ops.name_scope( - None, default_name='transform_features', values=features.values() - ): + with ops.name_scope(None, default_name='transform_features', values=features.values()): transformation_cache = FeatureTransformationCache(features) for column in feature_columns: with ops.name_scope(None, default_name=column.name): @@ -816,16 +804,12 @@ def make_parse_example_spec_v2(feature_columns): result = {} for column in feature_columns: if not isinstance(column, FeatureColumn): - raise ValueError( - 'All feature_columns must be FeatureColumn instances. ' - 'Given: {}'.format(column) - ) + raise ValueError('All feature_columns must be FeatureColumn instances. ' 'Given: {}'.format(column)) config = column.parse_example_spec for key, value in six.iteritems(config): if key in result and value != result[key]: raise ValueError( - 'feature_columns contain different parse_spec for key ' - '{}. Given {} and {}'.format(key, value, result[key]) + 'feature_columns contain different parse_spec for key ' '{}. Given {} and {}'.format(key, value, result[key]) ) result.update(config) return result @@ -841,7 +825,7 @@ def embedding_column( max_norm=None, trainable=True, partitioner=None, - ev_params=None + ev_params=None, ): """`DenseColumn` that converts from sparse, categorical input. @@ -917,20 +901,14 @@ def model_fn(features, ...): if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError( - 'Must specify both `ckpt_to_load_from` and ' - '`tensor_name_in_ckpt` or none of them.' - ) + raise ValueError('Must specify both `ckpt_to_load_from` and ' '`tensor_name_in_ckpt` or none of them.') if (initializer is not None) and (not callable(initializer)): raise ValueError( - 'initializer must be callable if specified. ' - 'Embedding of column_name: {}'.format(categorical_column.name) + 'initializer must be callable if specified. ' 'Embedding of column_name: {}'.format(categorical_column.name) ) if initializer is None: - initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=0.01 / math.sqrt(dimension) - ) + initializer = init_ops.truncated_normal_initializer(mean=0.0, stddev=0.01 / math.sqrt(dimension)) return EmbeddingColumn( categorical_column=categorical_column, @@ -942,7 +920,7 @@ def model_fn(features, ...): max_norm=max_norm, trainable=trainable, partitioner=partitioner, - ev_params=ev_params + ev_params=ev_params, ) @@ -957,7 +935,7 @@ def shared_embedding_columns( max_norm=None, trainable=True, partitioner=None, - ev_params=None + ev_params=None, ): """List of dense columns that convert from sparse, categorical input. @@ -1056,54 +1034,39 @@ def model_fn(features, ...): RuntimeError: if eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'shared_embedding_columns are not supported when eager ' - 'execution is enabled.' - ) + raise RuntimeError('shared_embedding_columns are not supported when eager ' 'execution is enabled.') if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError( - 'Must specify both `ckpt_to_load_from` and ' - '`tensor_name_in_ckpt` or none of them.' - ) + raise ValueError('Must specify both `ckpt_to_load_from` and ' '`tensor_name_in_ckpt` or none of them.') if (initializer is not None) and (not callable(initializer)): raise ValueError('initializer must be callable if specified.') if initializer is None: - initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=0.01 / math.sqrt(dimension) - ) + initializer = init_ops.truncated_normal_initializer(mean=0.0, stddev=0.01 / math.sqrt(dimension)) # Sort the columns so the default collection name is deterministic even if the # user passes columns from an unsorted collection, such as dict.values(). sorted_columns = sorted(categorical_columns, key=lambda x: x.name) c0 = sorted_columns[0] - num_buckets = c0._num_buckets # pylint: disable=protected-access - if not isinstance(c0, fc_old._CategoricalColumn): # pylint: disable=protected-access + num_buckets = c0._num_buckets + if not isinstance(c0, fc_old._CategoricalColumn): raise ValueError( - 'All categorical_columns must be subclasses of _CategoricalColumn. ' - 'Given: {}, of type: {}'.format(c0, type(c0)) + 'All categorical_columns must be subclasses of _CategoricalColumn. ' 'Given: {}, of type: {}'.format(c0, type(c0)) ) - if isinstance( - c0, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn) - ): # pylint: disable=protected-access + if isinstance(c0, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): c0 = c0.categorical_column for c in sorted_columns[1:]: - if isinstance( - c, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn) - ): # pylint: disable=protected-access + if isinstance(c, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): c = c.categorical_column - if num_buckets != c._num_buckets: # pylint: disable=protected-access + if num_buckets != c._num_buckets: raise ValueError( 'To use shared_embedding_column, all categorical_columns must have ' 'the same number of buckets. Given column: {} with buckets: {} does ' - 'not match column: {} with buckets: {}'.format( - c0, num_buckets, c, c._num_buckets - ) - ) # pylint: disable=protected-access + 'not match column: {} with buckets: {}'.format(c0, num_buckets, c, c._num_buckets) + ) if not shared_embedding_collection_name: shared_embedding_collection_name = '_'.join(c.name for c in sorted_columns) @@ -1112,18 +1075,20 @@ def model_fn(features, ...): result = [] for column in categorical_columns: result.append( - fc_old._SharedEmbeddingColumn( # pylint: disable=protected-access - categorical_column=column, - initializer=initializer, - dimension=dimension, - combiner=combiner, - shared_embedding_collection_name=shared_embedding_collection_name, - ckpt_to_load_from=ckpt_to_load_from, - tensor_name_in_ckpt=tensor_name_in_ckpt, - max_norm=max_norm, - trainable=trainable, - partitioner=partitioner, - ev_params=ev_params)) + fc_old._SharedEmbeddingColumn( + categorical_column=column, + initializer=initializer, + dimension=dimension, + combiner=combiner, + shared_embedding_collection_name=shared_embedding_collection_name, + ckpt_to_load_from=ckpt_to_load_from, + tensor_name_in_ckpt=tensor_name_in_ckpt, + max_norm=max_norm, + trainable=trainable, + partitioner=partitioner, + ev_params=ev_params, + ) + ) return result @@ -1137,7 +1102,7 @@ def shared_embedding_columns_v2( ckpt_to_load_from=None, tensor_name_in_ckpt=None, max_norm=None, - trainable=True + trainable=True, ): """List of dense columns that convert from sparse, categorical input. @@ -1235,25 +1200,17 @@ def model_fn(features, ...): RuntimeError: if eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'shared_embedding_columns are not supported when eager ' - 'execution is enabled.' - ) + raise RuntimeError('shared_embedding_columns are not supported when eager ' 'execution is enabled.') if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError( - 'Must specify both `ckpt_to_load_from` and ' - '`tensor_name_in_ckpt` or none of them.' - ) + raise ValueError('Must specify both `ckpt_to_load_from` and ' '`tensor_name_in_ckpt` or none of them.') if (initializer is not None) and (not callable(initializer)): raise ValueError('initializer must be callable if specified.') if initializer is None: - initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=0.01 / math.sqrt(dimension) - ) + initializer = init_ops.truncated_normal_initializer(mean=0.0, stddev=0.01 / math.sqrt(dimension)) # Sort the columns so the default collection name is deterministic even if the # user passes columns from an unsorted collection, such as dict.values(). @@ -1263,8 +1220,7 @@ def model_fn(features, ...): num_buckets = c0.num_buckets if not isinstance(c0, CategoricalColumn): raise ValueError( - 'All categorical_columns must be subclasses of CategoricalColumn. ' - 'Given: {}, of type: {}'.format(c0, type(c0)) + 'All categorical_columns must be subclasses of CategoricalColumn. ' 'Given: {}, of type: {}'.format(c0, type(c0)) ) if isinstance(c0, WeightedCategoricalColumn): c0 = c0.categorical_column @@ -1282,9 +1238,7 @@ def model_fn(features, ...): raise ValueError( 'To use shared_embedding_column, all categorical_columns must have ' 'the same number of buckets. Given column: {} with buckets: {} does ' - 'not match column: {} with buckets: {}'.format( - c0, num_buckets, c, c.num_buckets - ) + 'not match column: {} with buckets: {}'.format(c0, num_buckets, c, c.num_buckets) ) if not shared_embedding_collection_name: @@ -1292,28 +1246,29 @@ def model_fn(features, ...): shared_embedding_collection_name += '_shared_embedding' column_creator = SharedEmbeddingColumnCreator( - dimension, initializer, ckpt_to_load_from, tensor_name_in_ckpt, - num_buckets, trainable, shared_embedding_collection_name + dimension, + initializer, + ckpt_to_load_from, + tensor_name_in_ckpt, + num_buckets, + trainable, + shared_embedding_collection_name, ) result = [] for column in categorical_columns: - result.append( - column_creator( - categorical_column=column, combiner=combiner, max_norm=max_norm - ) - ) + result.append(column_creator(categorical_column=column, combiner=combiner, max_norm=max_norm)) return result def numeric_column( key, - shape=(1, ), + shape=(1,), default_value=None, dtype=dtypes.float32, normalizer_fn=None, - feature_name=None + feature_name=None, ): """Represents real valued or numerical features. @@ -1368,18 +1323,11 @@ def numeric_column( """ shape = _check_shape(shape, key) if not (dtype.is_integer or dtype.is_floating): - raise ValueError( - 'dtype must be convertible to float. ' - 'dtype: {}, key: {}'.format(dtype, key) - ) - default_value = fc_utils.check_default_value( - shape, default_value, dtype, key - ) + raise ValueError('dtype must be convertible to float. ' 'dtype: {}, key: {}'.format(dtype, key)) + default_value = fc_utils.check_default_value(shape, default_value, dtype, key) if normalizer_fn is not None and not callable(normalizer_fn): - raise TypeError( - 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn) - ) + raise TypeError('normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) fc_utils.assert_key_is_string(key) return NumericColumn( @@ -1388,7 +1336,7 @@ def numeric_column( shape=shape, default_value=default_value, dtype=dtype, - normalizer_fn=normalizer_fn + normalizer_fn=normalizer_fn, ) @@ -1458,16 +1406,12 @@ def bucketized_column(source_column, boundaries): one-dimensional. ValueError: If `boundaries` is not a sorted list or tuple. """ - if not isinstance(source_column, (NumericColumn, fc_old._NumericColumn)): # pylint: disable=protected-access + if not isinstance(source_column, (NumericColumn, fc_old._NumericColumn)): raise ValueError( - 'source_column must be a column generated with numeric_column(). ' - 'Given: {}'.format(source_column) + 'source_column must be a column generated with numeric_column(). ' 'Given: {}'.format(source_column) ) if len(source_column.shape) > 1: - raise ValueError( - 'source_column must be one-dimensional column. ' - 'Given: {}'.format(source_column) - ) + raise ValueError('source_column must be one-dimensional column. ' 'Given: {}'.format(source_column)) if not boundaries: raise ValueError('boundaries must not be empty.') if not (isinstance(boundaries, list) or isinstance(boundaries, tuple)): @@ -1478,9 +1422,7 @@ def bucketized_column(source_column, boundaries): return BucketizedColumn(source_column, tuple(boundaries)) -def categorical_column_with_hash_bucket( - key, hash_bucket_size, dtype=dtypes.string, feature_name=None -): +def categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dtypes.string, feature_name=None): """Represents sparse feature where ids are set by hashing. Use this when your sparse features are in string or integer format, and you @@ -1523,13 +1465,11 @@ def categorical_column_with_hash_bucket( ValueError: `dtype` is neither string nor integer. """ if hash_bucket_size is None: - raise ValueError('hash_bucket_size must be set. ' - 'key: {}'.format(key)) + raise ValueError('hash_bucket_size must be set. ' 'key: {}'.format(key)) if hash_bucket_size < 1: raise ValueError( - 'hash_bucket_size must be at least 1. ' - 'hash_bucket_size: {}, key: {}'.format(hash_bucket_size, key) + 'hash_bucket_size must be at least 1. ' 'hash_bucket_size: {}, key: {}'.format(hash_bucket_size, key) ) fc_utils.assert_key_is_string(key) @@ -1545,7 +1485,7 @@ def categorical_column_with_vocabulary_file_v2( dtype=dtypes.string, default_value=None, num_oov_buckets=0, - feature_name=None + feature_name=None, ): """A `CategoricalColumn` with a vocabulary file. @@ -1635,8 +1575,10 @@ def categorical_column_with_vocabulary_file_v2( with gfile.GFile(vocabulary_file) as f: vocabulary_size = sum(1 for _ in f) logging.info( - 'vocabulary_size = %d in %s is inferred from the number of elements ' - 'in the vocabulary_file %s.', vocabulary_size, key, vocabulary_file + 'vocabulary_size = %d in %s is inferred from the number of elements ' 'in the vocabulary_file %s.', + vocabulary_size, + key, + vocabulary_file, ) # `vocabulary_size` isn't required for lookup, but it is for `_num_buckets`. @@ -1644,14 +1586,9 @@ def categorical_column_with_vocabulary_file_v2( raise ValueError('Invalid vocabulary_size in {}.'.format(key)) if num_oov_buckets: if default_value is not None: - raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'. - format(key) - ) + raise ValueError("Can't specify both num_oov_buckets and default_value in {}.".format(key)) if num_oov_buckets < 0: - raise ValueError( - 'Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key) - ) + raise ValueError('Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key)) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) return VocabularyFileCategoricalColumn( @@ -1661,7 +1598,7 @@ def categorical_column_with_vocabulary_file_v2( vocabulary_size=vocabulary_size, num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, default_value=-1 if default_value is None else default_value, - dtype=dtype + dtype=dtype, ) @@ -1671,7 +1608,7 @@ def categorical_column_with_vocabulary_list( dtype=None, default_value=-1, num_oov_buckets=0, - feature_name=None + feature_name=None, ): """A `CategoricalColumn` with in-memory vocabulary. @@ -1749,38 +1686,21 @@ def categorical_column_with_vocabulary_list( ValueError: if `dtype` is not integer or string. """ if (vocabulary_list is None) or (len(vocabulary_list) < 1): - raise ValueError( - 'vocabulary_list {} must be non-empty, column_name: {}'.format( - vocabulary_list, key - ) - ) + raise ValueError('vocabulary_list {} must be non-empty, column_name: {}'.format(vocabulary_list, key)) if len(set(vocabulary_list)) != len(vocabulary_list): - raise ValueError( - 'Duplicate keys in vocabulary_list {}, column_name: {}'.format( - vocabulary_list, key - ) - ) + raise ValueError('Duplicate keys in vocabulary_list {}, column_name: {}'.format(vocabulary_list, key)) vocabulary_dtype = dtypes.as_dtype(np.array(vocabulary_list).dtype) if num_oov_buckets: if default_value != -1: - raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'. - format(key) - ) + raise ValueError("Can't specify both num_oov_buckets and default_value in {}.".format(key)) if num_oov_buckets < 0: - raise ValueError( - 'Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key) - ) - fc_utils.assert_string_or_int( - vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key) - ) + raise ValueError('Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key)) + fc_utils.assert_string_or_int(vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key)) if dtype is None: dtype = vocabulary_dtype elif dtype.is_integer != vocabulary_dtype.is_integer: raise ValueError( - 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format( - dtype, vocabulary_dtype, key - ) + 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format(dtype, vocabulary_dtype, key) ) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) @@ -1791,13 +1711,11 @@ def categorical_column_with_vocabulary_list( vocabulary_list=tuple(vocabulary_list), dtype=dtype, default_value=default_value, - num_oov_buckets=num_oov_buckets + num_oov_buckets=num_oov_buckets, ) -def categorical_column_with_identity( - key, num_buckets, default_value=None, feature_name=None -): +def categorical_column_with_identity(key, num_buckets, default_value=None, feature_name=None): """A `CategoricalColumn` that returns identity values. Use this when your inputs are integers in the range `[0, num_buckets)`, and @@ -1852,22 +1770,15 @@ def categorical_column_with_identity( ValueError: if `default_value` is not in range `[0, num_buckets)`. """ if num_buckets < 1: - raise ValueError( - 'num_buckets {} < 1, column_name {}'.format(num_buckets, key) - ) - if (default_value is not None - ) and ((default_value < 0) or (default_value >= num_buckets)): - raise ValueError( - 'default_value {} not in range [0, {}), column_name {}'.format( - default_value, num_buckets, key - ) - ) + raise ValueError('num_buckets {} < 1, column_name {}'.format(num_buckets, key)) + if (default_value is not None) and ((default_value < 0) or (default_value >= num_buckets)): + raise ValueError('default_value {} not in range [0, {}), column_name {}'.format(default_value, num_buckets, key)) fc_utils.assert_key_is_string(key) return IdentityCategoricalColumn( feature_name=feature_name, key=key, number_buckets=num_buckets, - default_value=default_value + default_value=default_value, ) @@ -1905,9 +1816,7 @@ def indicator_column(categorical_column): return IndicatorColumn(categorical_column) -def weighted_categorical_column( - categorical_column, weight_feature_key, dtype=dtypes.float32 -): +def weighted_categorical_column(categorical_column, weight_feature_key, dtype=dtypes.float32): """Applies weight values to a `CategoricalColumn`. Use this when each of your sparse inputs has both an ID and a value. For @@ -1977,7 +1886,7 @@ def weighted_categorical_column( return WeightedCategoricalColumn( categorical_column=categorical_column, weight_feature_key=weight_feature_key, - dtype=dtype + dtype=dtype, ) @@ -2086,27 +1995,17 @@ def crossed_column(keys, hash_bucket_size, hash_key=None, feature_name=None): ValueError: If `hash_bucket_size < 1`. """ if not hash_bucket_size or hash_bucket_size < 1: - raise ValueError( - 'hash_bucket_size must be > 1. ' - 'hash_bucket_size: {}'.format(hash_bucket_size) - ) + raise ValueError('hash_bucket_size must be > 1. ' 'hash_bucket_size: {}'.format(hash_bucket_size)) if not keys or len(keys) < 2: - raise ValueError( - 'keys must be a list with length > 1. Given: {}'.format(keys) - ) + raise ValueError('keys must be a list with length > 1. Given: {}'.format(keys)) for key in keys: - if ( - not isinstance(key, six.string_types) - and not isinstance(key, (CategoricalColumn, fc_old._CategoricalColumn)) - ): # pylint: disable=protected-access + if not isinstance(key, six.string_types) and not isinstance(key, (CategoricalColumn, fc_old._CategoricalColumn)): raise ValueError( 'Unsupported key type. All keys must be either string, or ' 'categorical column except HashedCategoricalColumn. ' 'Given: {}'.format(key) ) - if isinstance( - key, (HashedCategoricalColumn, fc_old._HashedCategoricalColumn) - ): # pylint: disable=protected-access + if isinstance(key, (HashedCategoricalColumn, fc_old._HashedCategoricalColumn)): raise ValueError( 'categorical_column_with_hash_bucket is not supported for crossing. ' 'Hashing before crossing will increase probability of collision. ' @@ -2116,7 +2015,7 @@ def crossed_column(keys, hash_bucket_size, hash_key=None, feature_name=None): feature_name=feature_name, keys=tuple(keys), hash_bucket_size=hash_bucket_size, - hash_key=hash_key + hash_key=hash_key, ) @@ -2140,7 +2039,6 @@ class FeatureColumn(object): @abc.abstractproperty def name(self): """Returns string, Used for naming.""" - pass @property def raw_name(self): @@ -2172,7 +2070,6 @@ def transform_feature(self, transformation_cache, state_manager): Returns: Transformed feature `Tensor`. """ - pass @abc.abstractproperty def parse_example_spec(self): @@ -2193,7 +2090,6 @@ def parse_example_spec(self): return spec ``` """ - pass def create_state(self, state_manager): """Uses the `state_manager` to create state for the FeatureColumn. @@ -2202,7 +2098,6 @@ def create_state(self, state_manager): state_manager: A `StateManager` to create / access resources such as lookup tables and variables. """ - pass @abc.abstractproperty def _is_v2_column(self): @@ -2212,7 +2107,6 @@ def _is_v2_column(self): might take in old categorical columns as input and then we want to use the old API. """ - pass @abc.abstractproperty def parents(self): @@ -2226,7 +2120,6 @@ def parents(self): a.parents = ['f1'] c.parents = [a, 'f2'] """ - pass @abc.abstractmethod def _get_config(self): @@ -2283,7 +2176,6 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): A serializable Dict that can be used to deserialize the object with from_config. """ - pass @classmethod def _from_config(cls, config, custom_objects=None, columns_by_name=None): @@ -2307,7 +2199,6 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): Returns: A FeatureColumn for the input config. """ - pass class DenseColumn(FeatureColumn): @@ -2320,7 +2211,6 @@ class DenseColumn(FeatureColumn): @abc.abstractproperty def variable_shape(self): """`TensorShape` of `get_dense_tensor`, without batch dimension.""" - pass @abc.abstractmethod def get_dense_tensor(self, transformation_cache, state_manager): @@ -2344,7 +2234,6 @@ def input_layer(features, feature_columns, ...): Returns: `Tensor` of shape [batch_size] + `variable_shape`. """ - pass def is_feature_column_v2(feature_columns): @@ -2352,14 +2241,12 @@ def is_feature_column_v2(feature_columns): for feature_column in feature_columns: if not isinstance(feature_column, FeatureColumn): return False - if not feature_column._is_v2_column: # pylint: disable=protected-access + if not feature_column._is_v2_column: return False return True -def _create_weighted_sum( - column, transformation_cache, state_manager, sparse_combiner, weight_var -): +def _create_weighted_sum(column, transformation_cache, state_manager, sparse_combiner, weight_var): """Creates a weighted sum for a dense/categorical column for linear_model.""" if isinstance(column, CategoricalColumn): return _create_categorical_column_weighted_sum( @@ -2367,20 +2254,18 @@ def _create_weighted_sum( transformation_cache=transformation_cache, state_manager=state_manager, sparse_combiner=sparse_combiner, - weight_var=weight_var + weight_var=weight_var, ) else: return _create_dense_column_weighted_sum( column=column, transformation_cache=transformation_cache, state_manager=state_manager, - weight_var=weight_var + weight_var=weight_var, ) -def _create_dense_column_weighted_sum( - column, transformation_cache, state_manager, weight_var -): +def _create_dense_column_weighted_sum(column, transformation_cache, state_manager, weight_var): """Create a weighted sum of a dense column for linear_model.""" tensor = column.get_dense_tensor(transformation_cache, state_manager) num_elements = column.variable_shape.num_elements() @@ -2395,13 +2280,11 @@ class CategoricalColumn(FeatureColumn): A categorical feature typically handled with a `tf.SparseTensor` of IDs. """ - IdWeightPair = collections.namedtuple( # pylint: disable=invalid-name - 'IdWeightPair', ('id_tensor', 'weight_tensor')) + IdWeightPair = collections.namedtuple('IdWeightPair', ('id_tensor', 'weight_tensor')) @abc.abstractproperty def num_buckets(self): """Returns number of buckets in this sparse feature.""" - pass @abc.abstractmethod def get_sparse_tensors(self, transformation_cache, state_manager): @@ -2423,13 +2306,9 @@ def get_sparse_tensors(self, transformation_cache, state_manager): state_manager: A `StateManager` to create / access resources such as lookup tables. """ - pass -def _create_categorical_column_weighted_sum( - column, transformation_cache, state_manager, sparse_combiner, weight_var -): - # pylint: disable=g-doc-return-or-yield,g-doc-args +def _create_categorical_column_weighted_sum(column, transformation_cache, state_manager, sparse_combiner, weight_var): """Create a weighted sum of a categorical column for linear_model. Note to maintainer: As implementation details, the weighted sum is @@ -2456,33 +2335,25 @@ def _create_categorical_column_weighted_sum( For both cases, we can implement weighted sum via embedding_lookup with sparse_combiner = "sum". """ - sparse_tensors = column.get_sparse_tensors( - transformation_cache, state_manager - ) - id_tensor = sparse_ops.sparse_reshape( - sparse_tensors.id_tensor, - [array_ops.shape(sparse_tensors.id_tensor)[0], -1] - ) + sparse_tensors = column.get_sparse_tensors(transformation_cache, state_manager) + id_tensor = sparse_ops.sparse_reshape(sparse_tensors.id_tensor, [array_ops.shape(sparse_tensors.id_tensor)[0], -1]) weight_tensor = sparse_tensors.weight_tensor if weight_tensor is not None: - weight_tensor = sparse_ops.sparse_reshape( - weight_tensor, [array_ops.shape(weight_tensor)[0], -1] - ) + weight_tensor = sparse_ops.sparse_reshape(weight_tensor, [array_ops.shape(weight_tensor)[0], -1]) return embedding_lookup_sparse( weight_var, id_tensor, sparse_weights=weight_tensor, combiner=sparse_combiner, - name='weighted_sum' + name='weighted_sum', ) class SequenceDenseColumn(FeatureColumn): """Represents dense sequence data.""" - TensorSequenceLengthPair = collections.namedtuple( # pylint: disable=invalid-name - 'TensorSequenceLengthPair', ('dense_tensor', 'sequence_length')) + TensorSequenceLengthPair = collections.namedtuple('TensorSequenceLengthPair', ('dense_tensor', 'sequence_length')) @abc.abstractmethod def get_sequence_dense_tensor(self, transformation_cache, state_manager): @@ -2494,7 +2365,6 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): state_manager: A `StateManager` to create / access resources such as lookup tables. """ - pass class FeatureTransformationCache(object): @@ -2572,10 +2442,7 @@ def get(self, key, state_manager): raise ValueError('Feature {} is not in features dictionary.'.format(key)) if not isinstance(key, FeatureColumn): - raise TypeError( - '"key" must be either a "str" or "FeatureColumn". ' - 'Provided: {}'.format(key) - ) + raise TypeError('"key" must be either a "str" or "FeatureColumn". ' 'Provided: {}'.format(key)) column = key logging.debug('Transforming feature_column %s.', column) @@ -2604,27 +2471,19 @@ def _get_raw_feature_as_tensor(self, key): ValueError: if the raw feature has rank 0. """ raw_feature = self._features[key] - feature_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - raw_feature - ) + feature_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(raw_feature) def expand_dims(input_tensor): # Input_tensor must have rank 1. if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): - return sparse_ops.sparse_reshape( - input_tensor, [array_ops.shape(input_tensor)[0], 1] - ) + return sparse_ops.sparse_reshape(input_tensor, [array_ops.shape(input_tensor)[0], 1]) else: return array_ops.expand_dims(input_tensor, -1) rank = feature_tensor.get_shape().ndims if rank is not None: if rank == 0: - raise ValueError( - 'Feature (key: {}) cannot have rank 0. Give: {}'.format( - key, feature_tensor - ) - ) + raise ValueError('Feature (key: {}) cannot have rank 0. Give: {}'.format(key, feature_tensor)) return feature_tensor if rank != 1 else expand_dims(feature_tensor) # Handle dynamic rank. @@ -2632,15 +2491,14 @@ def expand_dims(input_tensor): [ check_ops.assert_positive( array_ops.rank(feature_tensor), - message='Feature (key: {}) cannot have rank 0. Given: {}'.format( - key, feature_tensor - ) + message='Feature (key: {}) cannot have rank 0. Given: {}'.format(key, feature_tensor), ) ] ): return control_flow_ops.cond( math_ops.equal(1, array_ops.rank(feature_tensor)), - lambda: expand_dims(feature_tensor), lambda: feature_tensor + lambda: expand_dims(feature_tensor), + lambda: feature_tensor, ) @@ -2665,16 +2523,16 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): if 'RaggedTensor' in str(type(input_tensor)): return input_tensor - input_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - input_tensor - ) + input_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(input_tensor) if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): return input_tensor with ops.name_scope( - None, 'to_sparse_input', ( + None, + 'to_sparse_input', + ( input_tensor, ignore_value, - ) + ), ): if ignore_value is None: if input_tensor.dtype == dtypes.string: @@ -2687,18 +2545,12 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): # constructing a new numpy object of the given type, which yields the # default value for that type. ignore_value = input_tensor.dtype.as_numpy_dtype() - ignore_value = math_ops.cast( - ignore_value, input_tensor.dtype, name='ignore_value' - ) - indices = array_ops.where( - math_ops.not_equal(input_tensor, ignore_value), name='indices' - ) + ignore_value = math_ops.cast(ignore_value, input_tensor.dtype, name='ignore_value') + indices = array_ops.where(math_ops.not_equal(input_tensor, ignore_value), name='indices') return sparse_tensor_lib.SparseTensor( indices=indices, values=array_ops.gather_nd(input_tensor, indices, name='values'), - dense_shape=array_ops.shape( - input_tensor, out_type=dtypes.int64, name='dense_shape' - ) + dense_shape=array_ops.shape(input_tensor, out_type=dtypes.int64, name='dense_shape'), ) @@ -2730,8 +2582,7 @@ def _normalize_feature_columns(feature_columns): for column in feature_columns: if not isinstance(column, FeatureColumn): raise ValueError( - 'Items of feature_columns must be a FeatureColumn. ' - 'Given (type {}): {}.'.format(type(column), column) + 'Items of feature_columns must be a FeatureColumn. ' 'Given (type {}): {}.'.format(type(column), column) ) if not feature_columns: raise ValueError('feature_columns must not be empty.') @@ -2752,12 +2603,11 @@ def _normalize_feature_columns(feature_columns): class NumericColumn( DenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access + fc_old._DenseColumn, collections.namedtuple( - 'NumericColumn', ( - 'feature_name', 'key', 'shape', 'default_value', 'dtype', 'normalizer_fn' - ) - ) + 'NumericColumn', + ('feature_name', 'key', 'shape', 'default_value', 'dtype', 'normalizer_fn'), + ), ): """See `numeric_column`.""" @@ -2778,15 +2628,10 @@ def raw_name(self): @property def parse_example_spec(self): """See `FeatureColumn` base class.""" - return { - self.key: - parsing_ops.FixedLenFeature(self.shape, self.dtype, self.default_value) - } + return {self.key: parsing_ops.FixedLenFeature(self.shape, self.dtype, self.default_value)} @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec @@ -2800,9 +2645,7 @@ def _transform_input_tensor(self, input_tensor): input_tensor = self.normalizer_fn(input_tensor) return math_ops.cast(input_tensor, dtypes.float32) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): input_tensor = inputs.get(self.key) return self._transform_input_tensor(input_tensor) @@ -2833,9 +2676,7 @@ def variable_shape(self): return tensor_shape.TensorShape(self.shape) @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): return self.variable_shape @@ -2855,9 +2696,7 @@ def get_dense_tensor(self, transformation_cache, state_manager): # representation created by _transform_feature. return transformation_cache.get(self, state_manager) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -2880,9 +2719,7 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): """See 'FeatureColumn` base class.""" _check_config_keys(config, cls._fields) kwargs = config.copy() - kwargs['normalizer_fn'] = utils.deserialize_keras_object( - config['normalizer_fn'], custom_objects=custom_objects - ) + kwargs['normalizer_fn'] = utils.deserialize_keras_object(config['normalizer_fn'], custom_objects=custom_objects) kwargs['dtype'] = dtypes.as_dtype(config['dtype']) return cls(**kwargs) @@ -2890,18 +2727,15 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class BucketizedColumn( DenseColumn, CategoricalColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('BucketizedColumn', ('source_column', 'boundaries')) + fc_old._DenseColumn, + fc_old._CategoricalColumn, + collections.namedtuple('BucketizedColumn', ('source_column', 'boundaries')), ): """See `bucketized_column`.""" @property def _is_v2_column(self): - return ( - isinstance(self.source_column, FeatureColumn) - and self.source_column._is_v2_column - ) # pylint: disable=protected-access + return isinstance(self.source_column, FeatureColumn) and self.source_column._is_v2_column @property def name(self): @@ -2919,40 +2753,28 @@ def parse_example_spec(self): return self.source_column.parse_example_spec @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.source_column._parse_example_spec # pylint: disable=protected-access + return self.source_column._parse_example_spec - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Returns bucketized categorical `source_column` tensor.""" source_tensor = inputs.get(self.source_column) - return math_ops._bucketize( # pylint: disable=protected-access - source_tensor, - boundaries=self.boundaries) + return math_ops._bucketize(source_tensor, boundaries=self.boundaries) def transform_feature(self, transformation_cache, state_manager): """Returns bucketized categorical `source_column` tensor.""" source_tensor = transformation_cache.get(self.source_column, state_manager) - return math_ops._bucketize( # pylint: disable=protected-access - source_tensor, - boundaries=self.boundaries) + return math_ops._bucketize(source_tensor, boundaries=self.boundaries) @property def variable_shape(self): """See `DenseColumn` base class.""" - return tensor_shape.TensorShape( - tuple(self.source_column.shape) + (len(self.boundaries) + 1, ) - ) + return tensor_shape.TensorShape(tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): return self.variable_shape @@ -2960,8 +2782,8 @@ def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( indices=math_ops.cast(input_tensor, dtypes.int64), depth=len(self.boundaries) + 1, - on_value=1., - off_value=0. + on_value=1.0, + off_value=0.0, ) def get_dense_tensor(self, transformation_cache, state_manager): @@ -2969,9 +2791,7 @@ def get_dense_tensor(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_dense_tensor_for_input_tensor(input_tensor) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -2985,9 +2805,7 @@ def num_buckets(self): return (len(self.boundaries) + 1) * self.source_column.shape[0] @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets @@ -2999,25 +2817,18 @@ def _get_sparse_tensors_for_input_tensor(self, input_tensor): i1 = array_ops.reshape( array_ops.tile( array_ops.expand_dims(math_ops.range(0, batch_size), 1), - [1, source_dimension] - ), (-1, ) + [1, source_dimension], + ), + (-1,), ) i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size]) # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets - bucket_indices = ( - array_ops.reshape(input_tensor, (-1, )) + (len(self.boundaries) + 1) * i2 - ) + bucket_indices = array_ops.reshape(input_tensor, (-1,)) + (len(self.boundaries) + 1) * i2 - indices = math_ops.cast( - array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64 - ) - dense_shape = math_ops.cast( - array_ops.stack([batch_size, source_dimension]), dtypes.int64 - ) - sparse_tensor = sparse_tensor_lib.SparseTensor( - indices=indices, values=bucket_indices, dense_shape=dense_shape - ) + indices = math_ops.cast(array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64) + dense_shape = math_ops.cast(array_ops.stack([batch_size, source_dimension]), dtypes.int64) + sparse_tensor = sparse_tensor_lib.SparseTensor(indices=indices, values=bucket_indices, dense_shape=dense_shape) return CategoricalColumn.IdWeightPair(sparse_tensor, None) def get_sparse_tensors(self, transformation_cache, state_manager): @@ -3025,12 +2836,8 @@ def get_sparse_tensors(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_sparse_tensors_for_input_tensor(input_tensor) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): """Converts dense inputs to SparseTensor so downstream code can use it.""" del weight_collections del trainable @@ -3053,29 +2860,22 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): """See 'FeatureColumn` base class.""" _check_config_keys(config, cls._fields) kwargs = config.copy() - kwargs['source_column'] = deserialize_feature_column( - config['source_column'], custom_objects, columns_by_name - ) + kwargs['source_column'] = deserialize_feature_column(config['source_column'], custom_objects, columns_by_name) return cls(**kwargs) class SequenceBucketizedColumn( DenseColumn, CategoricalColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple( - 'SequenceBucketizedColumn', ('source_column', 'boundaries') - ) + fc_old._DenseColumn, + fc_old._CategoricalColumn, + collections.namedtuple('SequenceBucketizedColumn', ('source_column', 'boundaries')), ): """See `bucketized_column`.""" @property def _is_v2_column(self): - return ( - isinstance(self.source_column, FeatureColumn) - and self.source_column._is_v2_column - ) # pylint: disable=protected-access + return isinstance(self.source_column, FeatureColumn) and self.source_column._is_v2_column @property def name(self): @@ -3093,46 +2893,34 @@ def parse_example_spec(self): return self.source_column.parse_example_spec @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.source_column._parse_example_spec # pylint: disable=protected-access + return self.source_column._parse_example_spec - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Returns bucketized categorical `source_column` tensor.""" source_tensor = inputs.get(self.source_column) - bucketize_values = math_ops._bucketize( - source_tensor.values, boundaries=self.boundaries - ) + bucketize_values = math_ops._bucketize(source_tensor.values, boundaries=self.boundaries) bucketize_tensor = sparse_tensor_lib.SparseTensor( indices=source_tensor.indices, values=bucketize_values, - dense_shape=source_tensor.dense_shape + dense_shape=source_tensor.dense_shape, ) return bucketize_tensor def transform_feature(self, transformation_cache, state_manager): """Returns bucketized categorical `source_column` tensor.""" source_tensor = transformation_cache.get(self.source_column, state_manager) - return math_ops._bucketize( # pylint: disable=protected-access - source_tensor, - boundaries=self.boundaries) + return math_ops._bucketize(source_tensor, boundaries=self.boundaries) @property def variable_shape(self): """See `DenseColumn` base class.""" - return tensor_shape.TensorShape( - tuple(self.source_column.shape) + (len(self.boundaries) + 1, ) - ) + return tensor_shape.TensorShape(tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): return self.variable_shape @@ -3140,8 +2928,8 @@ def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( indices=math_ops.cast(input_tensor, dtypes.int64), depth=len(self.boundaries) + 1, - on_value=1., - off_value=0. + on_value=1.0, + off_value=0.0, ) def get_dense_tensor(self, transformation_cache, state_manager): @@ -3149,9 +2937,7 @@ def get_dense_tensor(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_dense_tensor_for_input_tensor(input_tensor) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -3165,9 +2951,7 @@ def num_buckets(self): return (len(self.boundaries) + 1) * self.source_column.shape[0] @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets @@ -3181,14 +2965,12 @@ def _get_sparse_tensors_for_input_tensor(self, input_sparse_tensor): i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size]) # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets - bucket_indices = ( - array_ops.reshape(input_tensor, (-1, )) + (len(self.boundaries) + 1) * i2 - ) + bucket_indices = array_ops.reshape(input_tensor, (-1,)) + (len(self.boundaries) + 1) * i2 sparse_tensor = sparse_tensor_lib.SparseTensor( indices=input_indices, values=bucket_indices, - dense_shape=input_sparse_tensor.dense_shape + dense_shape=input_sparse_tensor.dense_shape, ) # Compute the third dimension explicitly instead of setting it to -1, as # that doesn't work for dynamically shaped tensors with 0-length at runtime. @@ -3203,12 +2985,8 @@ def get_sparse_tensors(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_sparse_tensors_for_input_tensor(input_tensor) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): """Converts dense inputs to SparseTensor so downstream code can use it.""" del weight_collections del trainable @@ -3231,29 +3009,22 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): """See 'FeatureColumn` base class.""" _check_config_keys(config, cls._fields) kwargs = config.copy() - kwargs['source_column'] = deserialize_feature_column( - config['source_column'], custom_objects, columns_by_name - ) + kwargs['source_column'] = deserialize_feature_column(config['source_column'], custom_objects, columns_by_name) return cls(**kwargs) class SequenceNumericColumn( DenseColumn, CategoricalColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple( - 'SequenceNumericColumn', ('source_column', 'sequence_length') - ) + fc_old._DenseColumn, + fc_old._CategoricalColumn, + collections.namedtuple('SequenceNumericColumn', ('source_column', 'sequence_length')), ): """See `SequenceNumericColumn`.""" @property def _is_v2_column(self): - return ( - isinstance(self.source_column, FeatureColumn) - and self.source_column._is_v2_column - ) # pylint: disable=protected-access + return isinstance(self.source_column, FeatureColumn) and self.source_column._is_v2_column @property def name(self): @@ -3271,15 +3042,11 @@ def parse_example_spec(self): return self.source_column.parse_example_spec @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.source_column._parse_example_spec # pylint: disable=protected-access + return self.source_column._parse_example_spec - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Returns bucketized categorical `source_column` tensor.""" source_tensor = inputs.get(self.source_column) @@ -3293,14 +3060,10 @@ def transform_feature(self, transformation_cache, state_manager): @property def variable_shape(self): """See `DenseColumn` base class.""" - return tensor_shape.TensorShape( - tuple(self.source_column.shape) + (self.sequence_length, ) - ) + return tensor_shape.TensorShape(tuple(self.source_column.shape) + (self.sequence_length,)) @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): return self.variable_shape @@ -3308,8 +3071,8 @@ def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( indices=math_ops.cast(input_tensor, dtypes.int64), depth=self.sequence_length, - on_value=1., - off_value=0. + on_value=1.0, + off_value=0.0, ) def get_dense_tensor(self, transformation_cache, state_manager): @@ -3317,9 +3080,7 @@ def get_dense_tensor(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_dense_tensor_for_input_tensor(input_tensor) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -3328,18 +3089,12 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): def _get_sequence_dense_tensor(self, inputs): input_tensor = inputs.get(self) - sparse_tensors = self._get_sparse_tensors_for_input_tensor( - input_tensor - ).id_tensor - sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors - ) + sparse_tensors = self._get_sparse_tensors_for_input_tensor(input_tensor).id_tensor + sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors) sequence_length = tf.cast(sequence_length, tf.int32) shape = array_ops.shape(sparse_tensors) target_shape = [shape[0], shape[1], math_ops.reduce_prod(shape[2:])] - ret_tensor = tf.sparse_to_dense( - sparse_tensors.indices, target_shape, sparse_tensors.values - ) + ret_tensor = tf.sparse_to_dense(sparse_tensors.indices, target_shape, sparse_tensors.values) return CategoricalColumn.IdWeightPair(ret_tensor, sequence_length) @property @@ -3349,9 +3104,7 @@ def num_buckets(self): return self.sequence_length * self.source_column.shape[0] @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets @@ -3369,12 +3122,8 @@ def get_sparse_tensors(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_sparse_tensors_for_input_tensor(input_tensor) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): """Converts dense inputs to SparseTensor so downstream code can use it.""" del weight_collections del trainable @@ -3397,35 +3146,28 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): """See 'FeatureColumn` base class.""" _check_config_keys(config, cls._fields) kwargs = config.copy() - kwargs['source_column'] = deserialize_feature_column( - config['source_column'], custom_objects, columns_by_name - ) + kwargs['source_column'] = deserialize_feature_column(config['source_column'], custom_objects, columns_by_name) return cls(**kwargs) class SequenceWeightedCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, collections.namedtuple( 'SequenceWeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype') - ) + ('categorical_column', 'weight_feature_key', 'dtype'), + ), ): """See `weighted_categorical_column`.""" @property def _is_v2_column(self): - return ( - isinstance(self.categorical_column, FeatureColumn) - and self.categorical_column._is_v2_column - ) # pylint: disable=protected-access + return isinstance(self.categorical_column, FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): """See `FeatureColumn` base class.""" - return '{}_weighted_by_{}'.format( - self.categorical_column.name, self.weight_feature_key - ) + return '{}_weighted_by_{}'.format(self.categorical_column.name, self.weight_feature_key) @property def raw_name(self): @@ -3438,24 +3180,18 @@ def parse_example_spec(self): config = self.categorical_column.parse_example_spec if self.weight_feature_key in config: raise ValueError( - 'Parse config {} already exists for {}.'.format( - config[self.weight_feature_key], self.weight_feature_key - ) + 'Parse config {} already exists for {}.'.format(config[self.weight_feature_key], self.weight_feature_key) ) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - config = self.categorical_column._parse_example_spec # pylint: disable=protected-access + config = self.categorical_column._parse_example_spec if self.weight_feature_key in config: raise ValueError( - 'Parse config {} already exists for {}.'.format( - config[self.weight_feature_key], self.weight_feature_key - ) + 'Parse config {} already exists for {}.'.format(config[self.weight_feature_key], self.weight_feature_key) ) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @@ -3466,29 +3202,19 @@ def num_buckets(self): return self.categorical_column.num_buckets @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): - return self.categorical_column._num_buckets # pylint: disable=protected-access + return self.categorical_column._num_buckets def _transform_weight_tensor(self, weight_tensor): if weight_tensor is None: raise ValueError('Missing weights {}.'.format(self.weight_feature_key)) - weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - weight_tensor - ) + weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(weight_tensor) if self.dtype != weight_tensor.dtype.base_dtype: - raise ValueError( - 'Bad dtype, expected {}, but got {}.'.format( - self.dtype, weight_tensor.dtype - ) - ) + raise ValueError('Bad dtype, expected {}, but got {}.'.format(self.dtype, weight_tensor.dtype)) if not isinstance(weight_tensor, sparse_tensor_lib.SparseTensor): # The weight tensor can be a regular Tensor. In this case, sparsify it. - weight_tensor = _to_sparse_input_and_drop_ignore_values( - weight_tensor, ignore_value=0.0 - ) + weight_tensor = _to_sparse_input_and_drop_ignore_values(weight_tensor, ignore_value=0.0) if not weight_tensor.dtype.is_floating: weight_tensor = math_ops.cast(weight_tensor, dtypes.float32) shape = tf.shape(weight_tensor) @@ -3498,18 +3224,14 @@ def _transform_weight_tensor(self, weight_tensor): def transform_feature(self, transformation_cache, state_manager): """Applies weights to tensor generated from `categorical_column`'.""" - weight_tensor = transformation_cache.get( - self.weight_feature_key, state_manager - ) + weight_tensor = transformation_cache.get(self.weight_feature_key, state_manager) weight_tensor = self._transform_weight_tensor(weight_tensor) return ( - transformation_cache.get(self.categorical_column, - state_manager), weight_tensor + transformation_cache.get(self.categorical_column, state_manager), + weight_tensor, ) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Applies weights to tensor generated from `categorical_column`'.""" weight_tensor = inputs.get(self.weight_feature_key) @@ -3521,12 +3243,8 @@ def get_sparse_tensors(self, transformation_cache, state_manager): tensors = transformation_cache.get(self, state_manager) return CategoricalColumn.IdWeightPair(tensors[0], tensors[1]) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable tensors = inputs.get(self) @@ -3540,9 +3258,7 @@ def parents(self): def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) - config['categorical_column'] = serialize_feature_column( - self.categorical_column - ) + config['categorical_column'] = serialize_feature_column(self.categorical_column) config['dtype'] = self.dtype.name return config @@ -3561,24 +3277,29 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class EmbeddingColumn( DenseColumn, SequenceDenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._SequenceDenseColumn, # pylint: disable=protected-access + fc_old._DenseColumn, + fc_old._SequenceDenseColumn, collections.namedtuple( - 'EmbeddingColumn', ( - 'categorical_column', 'dimension', 'combiner', 'initializer', - 'ckpt_to_load_from', 'tensor_name_in_ckpt', 'max_norm', 'trainable', - 'partitioner', 'ev_params' - ) - ) + 'EmbeddingColumn', + ( + 'categorical_column', + 'dimension', + 'combiner', + 'initializer', + 'ckpt_to_load_from', + 'tensor_name_in_ckpt', + 'max_norm', + 'trainable', + 'partitioner', + 'ev_params', + ), + ), ): """See `embedding_column`.""" @property def _is_v2_column(self): - return ( - isinstance(self.categorical_column, FeatureColumn) - and self.categorical_column._is_v2_column - ) # pylint: disable=protected-access + return isinstance(self.categorical_column, FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): @@ -3596,19 +3317,15 @@ def parse_example_spec(self): return self.categorical_column.parse_example_spec @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec def transform_feature(self, transformation_cache, state_manager): """Transforms underlying `categorical_column`.""" return transformation_cache.get(self.categorical_column, state_manager) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): return inputs.get(self.categorical_column) @@ -3618,15 +3335,13 @@ def variable_shape(self): return tensor_shape.TensorShape([self.dimension]) @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): return self.variable_shape def create_state(self, state_manager): """Creates the embedding lookup variable.""" - embedding_shape = (self.categorical_column._num_buckets, self.dimension) # pylint: disable=protected-access + embedding_shape = (self.categorical_column._num_buckets, self.dimension) state_manager.create_variable( self, name='embedding_weights', @@ -3634,22 +3349,18 @@ def create_state(self, state_manager): dtype=dtypes.float32, trainable=self.trainable, use_resource=True, - initializer=self.initializer + initializer=self.initializer, ) - def _get_dense_tensor_internal_helper( - self, sparse_tensors, embedding_weights - ): + def _get_dense_tensor_internal_helper(self, sparse_tensors, embedding_weights): sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor if self.ckpt_to_load_from is not None: to_restore = embedding_weights if isinstance(to_restore, variables.PartitionedVariable): - to_restore = to_restore._get_variable_list() # pylint: disable=protected-access - checkpoint_utils.init_from_checkpoint( - self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore} - ) + to_restore = to_restore._get_variable_list() + checkpoint_utils.init_from_checkpoint(self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) if 'RaggedTensor' in str(type(sparse_ids)): return embedding_lookup_ragged( @@ -3658,7 +3369,7 @@ def _get_dense_tensor_internal_helper( sparse_weights, combiner=self.combiner, max_norm=self.max_norm, - name='%s_weights' % self.name + name='%s_weights' % self.name, ) # Return embedding lookup result. @@ -3668,27 +3379,18 @@ def _get_dense_tensor_internal_helper( sparse_weights, combiner=self.combiner, name='%s_weights' % self.name, - max_norm=self.max_norm + max_norm=self.max_norm, ) def _get_dense_tensor_internal(self, sparse_tensors, state_manager): """Private method that follows the signature of get_dense_tensor.""" - embedding_weights = state_manager.get_variable( - self, name='embedding_weights' - ) - return self._get_dense_tensor_internal_helper( - sparse_tensors, embedding_weights - ) + embedding_weights = state_manager.get_variable(self, name='embedding_weights') + return self._get_dense_tensor_internal_helper(sparse_tensors, embedding_weights) - def _old_get_dense_tensor_internal( - self, sparse_tensors, weight_collections, trainable - ): + def _old_get_dense_tensor_internal(self, sparse_tensors, weight_collections, trainable): """Private method that follows the signature of _get_dense_tensor.""" - embedding_shape = (self.categorical_column._num_buckets, self.dimension) # pylint: disable=protected-access - if ( - weight_collections - and ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections - ): + embedding_shape = (self.categorical_column._num_buckets, self.dimension) + if weight_collections and ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections: weight_collections.append(ops.GraphKeys.GLOBAL_VARIABLES) if self.ev_params is None: embedding_weights = variable_scope.get_variable( @@ -3698,28 +3400,23 @@ def _old_get_dense_tensor_internal( initializer=self.initializer, trainable=self.trainable and trainable, partitioner=self.partitioner, - collections=weight_collections + collections=weight_collections, ) else: # at eval or inference time, it is necessary to set # the initializers to zeros, so that new key will # get zero embedding - if os.environ.get('tf.estimator.mode', '') != \ - os.environ.get('tf.estimator.ModeKeys.TRAIN', 'train'): + if os.environ.get('tf.estimator.mode', '') != os.environ.get('tf.estimator.ModeKeys.TRAIN', 'train'): initializer = init_ops.zeros_initializer() else: initializer = self.initializer extra_args = {} if 'EmbeddingVariableConfig' in dir(variables): ev_option = variables.EmbeddingVariableOption() - ev_option.filter_strategy = variables.CounterFilter( - filter_freq=self.ev_params.filter_freq - ) + ev_option.filter_strategy = variables.CounterFilter(filter_freq=self.ev_params.filter_freq) extra_args['ev_option'] = ev_option else: - extra_args['filter_options'] = variables.CounterFilterOptions( - self.ev_params.filter_freq - ) + extra_args['filter_options'] = variables.CounterFilterOptions(self.ev_params.filter_freq) embedding_weights = variable_scope.get_embedding_variable( name='embedding_weights', embedding_dim=self.dimension, @@ -3728,7 +3425,7 @@ def _old_get_dense_tensor_internal( partitioner=self.partitioner, collections=weight_collections, steps_to_live=self.ev_params.steps_to_live, - **extra_args + **extra_args, ) # Write the embedding configuration to RTP-specified collections. This will inform RTP to @@ -3738,31 +3435,31 @@ def _old_get_dense_tensor_internal( variable=embedding_weights, bucket_size=self.categorical_column._num_buckets, combiner=self.combiner, - is_embedding_var=(self.ev_params is not None) + is_embedding_var=(self.ev_params is not None), ) embedding_attrs['name'] = layer_utils.unique_name_in_collection( compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name'] ) - layer_utils.update_attr_to_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs - ) + layer_utils.update_attr_to_collection(compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs) # operate embedding - predictions = self._get_dense_tensor_internal_helper( - sparse_tensors, embedding_weights - ) + predictions = self._get_dense_tensor_internal_helper(sparse_tensors, embedding_weights) # Update the information about the output and input nodes of embedding operation to the # previous written RTP-specific collection entry. RTP uses these informations to extract # the embedding subgraph. if isinstance(sparse_tensors.id_tensor, sparse_tensor_lib.SparseTensor): layer_utils.append_tensor_to_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name'], - 'tensor', predictions + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, + embedding_attrs['name'], + 'tensor', + predictions, ) layer_utils.append_tensor_to_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name'], - 'input', sparse_tensors.id_tensor + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, + embedding_attrs['name'], + 'input', + sparse_tensors.id_tensor, ) return predictions @@ -3790,24 +3487,18 @@ def get_dense_tensor(self, transformation_cache, state_manager): 'non-sequence categorical_column_with_*. ' 'Suggested fix B: If you wish to create sequence input, use ' 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) # Get sparse IDs and weights. - sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager - ) + sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) return self._get_dense_tensor_internal(sparse_tensors, state_manager) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): if isinstance( self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn) - ): # pylint: disable=protected-access + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), + ): raise ValueError( 'In embedding_column: {}. ' 'categorical_column must not be of type _SequenceCategoricalColumn. ' @@ -3815,15 +3506,10 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): 'non-sequence categorical_column_with_*. ' 'Suggested fix B: If you wish to create sequence input, use ' 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) - sparse_tensors = self.categorical_column._get_sparse_tensors( # pylint: disable=protected-access - inputs, weight_collections, trainable) - return self._old_get_dense_tensor_internal( - sparse_tensors, weight_collections, trainable - ) + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs, weight_collections, trainable) + return self._old_get_dense_tensor_internal(sparse_tensors, weight_collections, trainable) def get_sequence_dense_tensor(self, transformation_cache, state_manager): """See `SequenceDenseColumn` base class.""" @@ -3833,57 +3519,38 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): 'categorical_column must be of type SequenceCategoricalColumn ' 'to use SequenceFeatures. ' 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) - sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager - ) - dense_tensor = self._get_dense_tensor_internal( - sparse_tensors, state_manager - ) - sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor - ) - return SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length - ) + sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) + dense_tensor = self._get_dense_tensor_internal(sparse_tensors, state_manager) + sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) + return SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sequence_dense_tensor( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): if not isinstance( - self.categorical_column, ( - SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn, - SequenceBucketizedColumn, SequenceNumericColumn, - SequenceWeightedCategoricalColumn - ) - ): # pylint: disable=protected-access + self.categorical_column, + ( + SequenceCategoricalColumn, + fc_old._SequenceCategoricalColumn, + SequenceBucketizedColumn, + SequenceNumericColumn, + SequenceWeightedCategoricalColumn, + ), + ): raise ValueError( 'In embedding_column: {}. ' 'categorical_column must be of type SequenceCategoricalColumn ' 'to use SequenceFeatures. ' 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) dense_tensor = self._old_get_dense_tensor_internal( - sparse_tensors, - weight_collections=weight_collections, - trainable=trainable - ) - sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor - ) - return SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length + sparse_tensors, weight_collections=weight_collections, trainable=trainable ) + sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) + return SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) @property def parents(self): @@ -3893,9 +3560,7 @@ def parents(self): def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) - config['categorical_column'] = serialize_feature_column( - self.categorical_column - ) + config['categorical_column'] = serialize_feature_column(self.categorical_column) config['initializer'] = utils.serialize_keras_object(self.initializer) return config @@ -3907,9 +3572,7 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): kwargs['categorical_column'] = deserialize_feature_column( config['categorical_column'], custom_objects, columns_by_name ) - kwargs['initializer'] = utils.deserialize_keras_object( - config['initializer'], custom_objects=custom_objects - ) + kwargs['initializer'] = utils.deserialize_keras_object(config['initializer'], custom_objects=custom_objects) return cls(**kwargs) @@ -3923,7 +3586,6 @@ def _raise_shared_embedding_column_error(): # class SharedEmbeddingColumnCreator(tracking.AutoTrackable): class SharedEmbeddingColumnCreator: - def __init__( self, dimension, @@ -3932,7 +3594,7 @@ def __init__( tensor_name_in_ckpt, num_buckets, trainable, - name='shared_embedding_column_creator' + name='shared_embedding_column_creator', ): self._dimension = dimension self._initializer = initializer @@ -3949,7 +3611,7 @@ def __call__(self, categorical_column, combiner, max_norm): @property def embedding_weights(self): - key = ops.get_default_graph()._graph_key # pylint: disable=protected-access + key = ops.get_default_graph()._graph_key if key not in self._embedding_weights: embedding_shape = (self._num_buckets, self._dimension) var = variable_scope.get_variable( @@ -3957,16 +3619,14 @@ def embedding_weights(self): shape=embedding_shape, dtype=dtypes.float32, initializer=self._initializer, - trainable=self._trainable + trainable=self._trainable, ) if self._ckpt_to_load_from is not None: to_restore = var if isinstance(to_restore, variables.PartitionedVariable): - to_restore = to_restore._get_variable_list() # pylint: disable=protected-access - checkpoint_utils.init_from_checkpoint( - self._ckpt_to_load_from, {self._tensor_name_in_ckpt: to_restore} - ) + to_restore = to_restore._get_variable_list() + checkpoint_utils.init_from_checkpoint(self._ckpt_to_load_from, {self._tensor_name_in_ckpt: to_restore}) self._embedding_weights[key] = var return self._embedding_weights[key] @@ -3978,14 +3638,17 @@ def dimension(self): class SharedEmbeddingColumn( DenseColumn, SequenceDenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._SequenceDenseColumn, # pylint: disable=protected-access + fc_old._DenseColumn, + fc_old._SequenceDenseColumn, collections.namedtuple( - 'SharedEmbeddingColumn', ( - 'categorical_column', 'shared_embedding_column_creator', 'combiner', - 'max_norm' - ) - ) + 'SharedEmbeddingColumn', + ( + 'categorical_column', + 'shared_embedding_column_creator', + 'combiner', + 'max_norm', + ), + ), ): """See `embedding_column`.""" @@ -4022,9 +3685,7 @@ def _transform_feature(self, inputs): @property def variable_shape(self): """See `DenseColumn` base class.""" - return tensor_shape.TensorShape( - [self.shared_embedding_column_creator.dimension] - ) + return tensor_shape.TensorShape([self.shared_embedding_column_creator.dimension]) @property def _variable_shape(self): @@ -4037,9 +3698,7 @@ def _get_dense_tensor_internal(self, transformation_cache, state_manager): # that the ops for different columns have distinct names. with ops.name_scope(None, default_name=self.name): # Get sparse IDs and weights. - sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager - ) + sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor @@ -4052,7 +3711,7 @@ def _get_dense_tensor_internal(self, transformation_cache, state_manager): sparse_weights=sparse_weights, combiner=self.combiner, name='%s_weights' % self.name, - max_norm=self.max_norm + max_norm=self.max_norm, ) def get_dense_tensor(self, transformation_cache, state_manager): @@ -4065,9 +3724,7 @@ def get_dense_tensor(self, transformation_cache, state_manager): 'non-sequence categorical_column_with_*. ' 'Suggested fix B: If you wish to create sequence input, use ' 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) return self._get_dense_tensor_internal(transformation_cache, state_manager) @@ -4082,26 +3739,14 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): 'categorical_column must be of type SequenceCategoricalColumn ' 'to use SequenceFeatures. ' 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) - dense_tensor = self._get_dense_tensor_internal( - transformation_cache, state_manager - ) - sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager - ) - sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor - ) - return SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length - ) + dense_tensor = self._get_dense_tensor_internal(transformation_cache, state_manager) + sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) + sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) + return SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) - def _get_sequence_dense_tensor( - self, inputs, weight_collections=None, trainable=None - ): + def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): return _raise_shared_embedding_column_error() @property @@ -4127,25 +3772,16 @@ def _check_shape(shape, key): shape = tuple(shape) for dimension in shape: if not isinstance(dimension, int): - raise TypeError( - 'shape dimensions must be integer. ' - 'shape: {}, key: {}'.format(shape, key) - ) + raise TypeError('shape dimensions must be integer. ' 'shape: {}, key: {}'.format(shape, key)) if dimension < 1: - raise ValueError( - 'shape dimensions must be greater than 0. ' - 'shape: {}, key: {}'.format(shape, key) - ) + raise ValueError('shape dimensions must be greater than 0. ' 'shape: {}, key: {}'.format(shape, key)) return shape class HashedCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple( - 'HashedCategoricalColumn', - ('feature_name', 'key', 'hash_bucket_size', 'dtype') - ) + fc_old._CategoricalColumn, + collections.namedtuple('HashedCategoricalColumn', ('feature_name', 'key', 'hash_bucket_size', 'dtype')), ): """See `categorical_column_with_hash_bucket`.""" @@ -4169,25 +3805,18 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec def _transform_input_tensor(self, input_tensor): """Hashes the values in the feature_column.""" - fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key) - ) + fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype - ) + 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) ) if input_tensor.dtype == dtypes.string: @@ -4195,34 +3824,23 @@ def _transform_input_tensor(self, input_tensor): else: sparse_values = string_ops.as_string(input_tensor.values) - sparse_id_values = string_ops.string_to_hash_bucket_fast( - sparse_values, self.hash_bucket_size, name='lookup' - ) + sparse_id_values = string_ops.string_to_hash_bucket_fast(sparse_values, self.hash_bucket_size, name='lookup') if 'RaggedTensor' in str(type(input_tensor)): from tensorflow.python.ops.ragged import ragged_tensor - return ragged_tensor.RaggedTensor.from_row_splits( - values=sparse_id_values, row_splits=input_tensor.row_splits - ) - return sparse_tensor_lib.SparseTensor( - input_tensor.indices, sparse_id_values, input_tensor.dense_shape - ) + return ragged_tensor.RaggedTensor.from_row_splits(values=sparse_id_values, row_splits=input_tensor.row_splits) + + return sparse_tensor_lib.SparseTensor(input_tensor.indices, sparse_id_values, input_tensor.dense_shape) def transform_feature(self, transformation_cache, state_manager): """Hashes the values in the feature_column.""" - input_tensor = _to_sparse_input_and_drop_ignore_values( - transformation_cache.get(self.key, state_manager) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(transformation_cache.get(self.key, state_manager)) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values( - inputs.get(self.key) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) return self._transform_input_tensor(input_tensor) @property @@ -4231,24 +3849,16 @@ def num_buckets(self): return self.hash_bucket_size @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" - return CategoricalColumn.IdWeightPair( - transformation_cache.get(self, state_manager), None - ) + return CategoricalColumn.IdWeightPair(transformation_cache.get(self, state_manager), None) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -4275,13 +3885,19 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class VocabularyFileCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, collections.namedtuple( - 'VocabularyFileCategoricalColumn', ( - 'feature_name', 'key', 'vocabulary_file', 'vocabulary_size', - 'num_oov_buckets', 'dtype', 'default_value' - ) - ) + 'VocabularyFileCategoricalColumn', + ( + 'feature_name', + 'key', + 'vocabulary_file', + 'vocabulary_size', + 'num_oov_buckets', + 'dtype', + 'default_value', + ), + ), ): """See `categorical_column_with_vocabulary_file`.""" @@ -4305,9 +3921,7 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec @@ -4316,15 +3930,10 @@ def _transform_input_tensor(self, input_tensor): if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype - ) + 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) ) - fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key) - ) + fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -4339,23 +3948,17 @@ def _transform_input_tensor(self, input_tensor): vocab_size=self.vocabulary_size, default_value=self.default_value, key_dtype=key_dtype, - name='{}_lookup'.format(self.key) + name='{}_lookup'.format(self.key), ).lookup(input_tensor) def transform_feature(self, transformation_cache, state_manager): """Creates a lookup table for the vocabulary.""" - input_tensor = _to_sparse_input_and_drop_ignore_values( - transformation_cache.get(self.key, state_manager) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(transformation_cache.get(self.key, state_manager)) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values( - inputs.get(self.key) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) return self._transform_input_tensor(input_tensor) @property @@ -4364,24 +3967,16 @@ def num_buckets(self): return self.vocabulary_size + self.num_oov_buckets @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" - return CategoricalColumn.IdWeightPair( - transformation_cache.get(self, state_manager), None - ) + return CategoricalColumn.IdWeightPair(transformation_cache.get(self, state_manager), None) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -4408,13 +4003,18 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class VocabularyListCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, collections.namedtuple( - 'VocabularyListCategoricalColumn', ( - 'feature_name', 'key', 'vocabulary_list', 'dtype', 'default_value', - 'num_oov_buckets' - ) - ) + 'VocabularyListCategoricalColumn', + ( + 'feature_name', + 'key', + 'vocabulary_list', + 'dtype', + 'default_value', + 'num_oov_buckets', + ), + ), ): """See `categorical_column_with_vocabulary_list`.""" @@ -4438,9 +4038,7 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec @@ -4449,15 +4047,10 @@ def _transform_input_tensor(self, input_tensor): if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format( - self.key, self.dtype, input_tensor.dtype - ) + 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) ) - fc_utils.assert_string_or_int( - input_tensor.dtype, - prefix='column_name: {} input_tensor'.format(self.key) - ) + fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -4471,23 +4064,17 @@ def _transform_input_tensor(self, input_tensor): default_value=self.default_value, num_oov_buckets=self.num_oov_buckets, dtype=key_dtype, - name='{}_lookup'.format(self.key) + name='{}_lookup'.format(self.key), ).lookup(input_tensor) def transform_feature(self, transformation_cache, state_manager): """Creates a lookup table for the vocabulary list.""" - input_tensor = _to_sparse_input_and_drop_ignore_values( - transformation_cache.get(self.key, state_manager) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(transformation_cache.get(self.key, state_manager)) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values( - inputs.get(self.key) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) return self._transform_input_tensor(input_tensor) @property @@ -4496,24 +4083,16 @@ def num_buckets(self): return len(self.vocabulary_list) + self.num_oov_buckets @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" - return CategoricalColumn.IdWeightPair( - transformation_cache.get(self, state_manager), None - ) + return CategoricalColumn.IdWeightPair(transformation_cache.get(self, state_manager), None) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -4540,11 +4119,11 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class IdentityCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, collections.namedtuple( 'IdentityCategoricalColumn', - ('feature_name', 'key', 'number_buckets', 'default_value') - ) + ('feature_name', 'key', 'number_buckets', 'default_value'), + ), ): """See `categorical_column_with_identity`.""" @@ -4568,29 +4147,21 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(dtypes.int64)} @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec def _transform_input_tensor(self, input_tensor): """Returns a SparseTensor with identity values.""" if not input_tensor.dtype.is_integer: - raise ValueError( - 'Invalid input, not integer. key: {} dtype: {}'.format( - self.key, input_tensor.dtype - ) - ) + raise ValueError('Invalid input, not integer. key: {} dtype: {}'.format(self.key, input_tensor.dtype)) if 'RaggedTensor' in str(type(input_tensor)): return input_tensor values = math_ops.cast(input_tensor.values, dtypes.int64, name='values') if self.num_buckets < sys.maxsize: - num_buckets = math_ops.cast( - self.num_buckets, dtypes.int64, name='num_buckets' - ) + num_buckets = math_ops.cast(self.num_buckets, dtypes.int64, name='num_buckets') zero = math_ops.cast(0, dtypes.int64, name='zero') if self.default_value is None: # Fail if values are out-of-range. @@ -4598,46 +4169,37 @@ def _transform_input_tensor(self, input_tensor): values, num_buckets, data=(values, num_buckets), - name='assert_less_than_num_buckets' - ) - assert_greater = check_ops.assert_greater_equal( - values, zero, data=(values, ), name='assert_greater_or_equal_0' + name='assert_less_than_num_buckets', ) + assert_greater = check_ops.assert_greater_equal(values, zero, data=(values,), name='assert_greater_or_equal_0') with ops.control_dependencies((assert_less, assert_greater)): values = array_ops.identity(values) else: # Assign default for out-of-range values. values = array_ops.where( - math_ops.logical_or( - values < zero, values >= num_buckets, name='out_of_range' - ), + math_ops.logical_or(values < zero, values >= num_buckets, name='out_of_range'), array_ops.fill( dims=array_ops.shape(values), value=math_ops.cast(self.default_value, dtypes.int64), - name='default_values' - ), values + name='default_values', + ), + values, ) return sparse_tensor_lib.SparseTensor( indices=input_tensor.indices, values=values, - dense_shape=input_tensor.dense_shape + dense_shape=input_tensor.dense_shape, ) def transform_feature(self, transformation_cache, state_manager): """Returns a SparseTensor with identity values.""" - input_tensor = _to_sparse_input_and_drop_ignore_values( - transformation_cache.get(self.key, state_manager) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(transformation_cache.get(self.key, state_manager)) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): - input_tensor = _to_sparse_input_and_drop_ignore_values( - inputs.get(self.key) - ) + input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) return self._transform_input_tensor(input_tensor) @property @@ -4646,24 +4208,16 @@ def num_buckets(self): return self.number_buckets @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" - return CategoricalColumn.IdWeightPair( - transformation_cache.get(self, state_manager), None - ) + return CategoricalColumn.IdWeightPair(transformation_cache.get(self, state_manager), None) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -4686,27 +4240,22 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class WeightedCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, collections.namedtuple( 'WeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype') - ) + ('categorical_column', 'weight_feature_key', 'dtype'), + ), ): """See `weighted_categorical_column`.""" @property def _is_v2_column(self): - return ( - isinstance(self.categorical_column, FeatureColumn) - and self.categorical_column._is_v2_column - ) # pylint: disable=protected-access + return isinstance(self.categorical_column, FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): """See `FeatureColumn` base class.""" - return '{}_weighted_by_{}'.format( - self.categorical_column.name, self.weight_feature_key - ) + return '{}_weighted_by_{}'.format(self.categorical_column.name, self.weight_feature_key) @property def raw_name(self): @@ -4719,24 +4268,18 @@ def parse_example_spec(self): config = self.categorical_column.parse_example_spec if self.weight_feature_key in config: raise ValueError( - 'Parse config {} already exists for {}.'.format( - config[self.weight_feature_key], self.weight_feature_key - ) + 'Parse config {} already exists for {}.'.format(config[self.weight_feature_key], self.weight_feature_key) ) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - config = self.categorical_column._parse_example_spec # pylint: disable=protected-access + config = self.categorical_column._parse_example_spec if self.weight_feature_key in config: raise ValueError( - 'Parse config {} already exists for {}.'.format( - config[self.weight_feature_key], self.weight_feature_key - ) + 'Parse config {} already exists for {}.'.format(config[self.weight_feature_key], self.weight_feature_key) ) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @@ -4747,47 +4290,33 @@ def num_buckets(self): return self.categorical_column.num_buckets @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): - return self.categorical_column._num_buckets # pylint: disable=protected-access + return self.categorical_column._num_buckets def _transform_weight_tensor(self, weight_tensor): if weight_tensor is None: raise ValueError('Missing weights {}.'.format(self.weight_feature_key)) - weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( - weight_tensor - ) + weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(weight_tensor) if self.dtype != weight_tensor.dtype.base_dtype: - raise ValueError( - 'Bad dtype, expected {}, but got {}.'.format( - self.dtype, weight_tensor.dtype - ) - ) + raise ValueError('Bad dtype, expected {}, but got {}.'.format(self.dtype, weight_tensor.dtype)) if not isinstance(weight_tensor, sparse_tensor_lib.SparseTensor): # The weight tensor can be a regular Tensor. In this case, sparsify it. - weight_tensor = _to_sparse_input_and_drop_ignore_values( - weight_tensor, ignore_value=0.0 - ) + weight_tensor = _to_sparse_input_and_drop_ignore_values(weight_tensor, ignore_value=0.0) if not weight_tensor.dtype.is_floating: weight_tensor = math_ops.cast(weight_tensor, dtypes.float32) return weight_tensor def transform_feature(self, transformation_cache, state_manager): """Applies weights to tensor generated from `categorical_column`'.""" - weight_tensor = transformation_cache.get( - self.weight_feature_key, state_manager - ) + weight_tensor = transformation_cache.get(self.weight_feature_key, state_manager) weight_tensor = self._transform_weight_tensor(weight_tensor) return ( - transformation_cache.get(self.categorical_column, - state_manager), weight_tensor + transformation_cache.get(self.categorical_column, state_manager), + weight_tensor, ) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Applies weights to tensor generated from `categorical_column`'.""" weight_tensor = inputs.get(self.weight_feature_key) @@ -4799,12 +4328,8 @@ def get_sparse_tensors(self, transformation_cache, state_manager): tensors = transformation_cache.get(self, state_manager) return CategoricalColumn.IdWeightPair(tensors[0], tensors[1]) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable tensors = inputs.get(self) @@ -4818,9 +4343,7 @@ def parents(self): def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) - config['categorical_column'] = serialize_feature_column( - self.categorical_column - ) + config['categorical_column'] = serialize_feature_column(self.categorical_column) config['dtype'] = self.dtype.name return config @@ -4838,10 +4361,8 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class CrossedColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple( - 'CrossedColumn', ('feature_name', 'keys', 'hash_bucket_size', 'hash_key') - ) + fc_old._CategoricalColumn, + collections.namedtuple('CrossedColumn', ('feature_name', 'keys', 'hash_bucket_size', 'hash_key')), ): """See `crossed_column`.""" @@ -4852,7 +4373,7 @@ def _is_v2_column(self): continue if not isinstance(key, FeatureColumn): return False - if not key._is_v2_column: # pylint: disable=protected-access + if not key._is_v2_column: return False return True @@ -4863,7 +4384,7 @@ def name(self): return self.feature_name feature_names = [] for key in _collect_leaf_level_keys(self): - if isinstance(key, (FeatureColumn, fc_old._FeatureColumn)): # pylint: disable=protected-access + if isinstance(key, (FeatureColumn, fc_old._FeatureColumn)): feature_names.append(key.name) else: # key must be a string feature_names.append(key) @@ -4876,16 +4397,14 @@ def parse_example_spec(self): for key in self.keys: if isinstance(key, FeatureColumn): config.update(key.parse_example_spec) - elif isinstance(key, fc_old._FeatureColumn): # pylint: disable=protected-access - config.update(key._parse_example_spec) # pylint: disable=protected-access + elif isinstance(key, fc_old._FeatureColumn): + config.update(key._parse_example_spec) else: # key must be a string config.update({key: parsing_ops.VarLenFeature(dtypes.string)}) return config @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec @@ -4895,10 +4414,8 @@ def transform_feature(self, transformation_cache, state_manager): for key in _collect_leaf_level_keys(self): if isinstance(key, six.string_types): feature_tensors.append(transformation_cache.get(key, state_manager)) - elif isinstance(key, (fc_old._CategoricalColumn, CategoricalColumn)): # pylint: disable=protected-access - ids_and_weights = key.get_sparse_tensors( - transformation_cache, state_manager - ) + elif isinstance(key, (fc_old._CategoricalColumn, CategoricalColumn)): + ids_and_weights = key.get_sparse_tensors(transformation_cache, state_manager) if ids_and_weights.weight_tensor is not None: raise ValueError( 'crossed_column does not support weight_tensor, but the given ' @@ -4911,20 +4428,18 @@ def transform_feature(self, transformation_cache, state_manager): return sparse_ops.sparse_cross_hashed( inputs=feature_tensors, num_buckets=self.hash_bucket_size, - hash_key=self.hash_key + hash_key=self.hash_key, ) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Generates a hashed sparse cross from the input tensors.""" feature_tensors = [] for key in _collect_leaf_level_keys(self): if isinstance(key, six.string_types): feature_tensors.append(inputs.get(key)) - elif isinstance(key, (CategoricalColumn, fc_old._CategoricalColumn)): # pylint: disable=protected-access - ids_and_weights = key._get_sparse_tensors(inputs) # pylint: disable=protected-access + elif isinstance(key, (CategoricalColumn, fc_old._CategoricalColumn)): + ids_and_weights = key._get_sparse_tensors(inputs) if ids_and_weights.weight_tensor is not None: raise ValueError( 'crossed_column does not support weight_tensor, but the given ' @@ -4937,7 +4452,7 @@ def _transform_feature(self, inputs): return sparse_ops.sparse_cross_hashed( inputs=feature_tensors, num_buckets=self.hash_bucket_size, - hash_key=self.hash_key + hash_key=self.hash_key, ) @property @@ -4946,24 +4461,16 @@ def num_buckets(self): return self.hash_bucket_size @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" - return CategoricalColumn.IdWeightPair( - transformation_cache.get(self, state_manager), None - ) + return CategoricalColumn.IdWeightPair(transformation_cache.get(self, state_manager), None) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): """See `CategoricalColumn` base class.""" del weight_collections del trainable @@ -4985,12 +4492,7 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): """See 'FeatureColumn` base class.""" _check_config_keys(config, cls._fields) kwargs = config.copy() - kwargs['keys'] = tuple( - [ - deserialize_feature_column(c, custom_objects, columns_by_name) - for c in config['keys'] - ] - ) + kwargs['keys'] = tuple([deserialize_feature_column(c, custom_objects, columns_by_name) for c in config['keys']]) return cls(**kwargs) @@ -5016,10 +4518,7 @@ def _prune_invalid_ids(sparse_ids, sparse_weights): """Prune invalid IDs (< 0) from the input ids and weights.""" is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) if sparse_weights is not None: - is_id_valid = math_ops.logical_and( - is_id_valid, - array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool) - ) + is_id_valid = math_ops.logical_and(is_id_valid, array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) if sparse_weights is not None: sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) @@ -5038,9 +4537,9 @@ def _prune_invalid_weights(sparse_ids, sparse_weights): class IndicatorColumn( DenseColumn, SequenceDenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._SequenceDenseColumn, # pylint: disable=protected-access - collections.namedtuple('IndicatorColumn', ('categorical_column')) + fc_old._DenseColumn, + fc_old._SequenceDenseColumn, + collections.namedtuple('IndicatorColumn', ('categorical_column')), ): """Represents a one-hot column for use in deep networks. @@ -5051,10 +4550,7 @@ class IndicatorColumn( @property def _is_v2_column(self): - return ( - isinstance(self.categorical_column, FeatureColumn) - and self.categorical_column._is_v2_column - ) # pylint: disable=protected-access + return isinstance(self.categorical_column, FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): @@ -5075,31 +4571,23 @@ def _transform_id_weight_pair(self, id_weight_pair): weighted_column = sparse_ops.sparse_merge( sp_ids=id_tensor, sp_values=weight_tensor, - vocab_size=int(self._variable_shape[-1]) + vocab_size=int(self._variable_shape[-1]), ) # Remove (?, -1) index. - weighted_column = sparse_ops.sparse_slice( - weighted_column, [0, 0], weighted_column.dense_shape - ) + weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0], weighted_column.dense_shape) # Use scatter_nd to merge duplicated indices if existed, # instead of sparse_tensor_to_dense. return array_ops.scatter_nd( - weighted_column.indices, weighted_column.values, - weighted_column.dense_shape + weighted_column.indices, + weighted_column.values, + weighted_column.dense_shape, ) - dense_id_tensor = sparse_ops.sparse_tensor_to_dense( - id_tensor, default_value=-1 - ) + dense_id_tensor = sparse_ops.sparse_tensor_to_dense(id_tensor, default_value=-1) # One hot must be float for tf.concat reasons since all other inputs to # input_layer are float32. - one_hot_id_tensor = array_ops.one_hot( - dense_id_tensor, - depth=self._variable_shape[-1], - on_value=1.0, - off_value=0.0 - ) + one_hot_id_tensor = array_ops.one_hot(dense_id_tensor, depth=self._variable_shape[-1], on_value=1.0, off_value=0.0) # Reduce to get a multi-hot per example. return math_ops.reduce_sum(one_hot_id_tensor, axis=[-2]) @@ -5119,16 +4607,12 @@ def transform_feature(self, transformation_cache, state_manager): Raises: ValueError: if input rank is not known at graph building time. """ - id_weight_pair = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager - ) + id_weight_pair = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) return self._transform_id_weight_pair(id_weight_pair) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): - id_weight_pair = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + id_weight_pair = self.categorical_column._get_sparse_tensors(inputs) return self._transform_id_weight_pair(id_weight_pair) @property @@ -5137,11 +4621,9 @@ def parse_example_spec(self): return self.categorical_column.parse_example_spec @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec @property def variable_shape(self): @@ -5149,16 +4631,12 @@ def variable_shape(self): if isinstance(self.categorical_column, FeatureColumn): return tensor_shape.TensorShape([1, self.categorical_column.num_buckets]) else: - return tensor_shape.TensorShape( - [1, self.categorical_column._num_buckets] - ) # pylint: disable=protected-access + return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): - return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) # pylint: disable=protected-access + return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) def get_dense_tensor(self, transformation_cache, state_manager): """Returns dense `Tensor` representing feature. @@ -5183,24 +4661,20 @@ def get_dense_tensor(self, transformation_cache, state_manager): 'non-sequence categorical_column_with_*. ' 'Suggested fix B: If you wish to create sequence input, use ' 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) # Feature has been already transformed. Return the intermediate # representation created by transform_feature. return transformation_cache.get(self, state_manager) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable if isinstance( self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn) - ): # pylint: disable=protected-access + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), + ): raise ValueError( 'In indicator_column: {}. ' 'categorical_column must not be of type _SequenceCategoricalColumn. ' @@ -5208,9 +4682,7 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): 'non-sequence categorical_column_with_*. ' 'Suggested fix B: If you wish to create sequence input, use ' 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) # Feature has been already transformed. Return the intermediate # representation created by transform_feature. @@ -5224,56 +4696,38 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): 'categorical_column must be of type SequenceCategoricalColumn ' 'to use SequenceFeatures. ' 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) # Feature has been already transformed. Return the intermediate # representation created by transform_feature. dense_tensor = transformation_cache.get(self, state_manager) - sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager - ) - sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor - ) - return SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length - ) + sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) + sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) + return SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sequence_dense_tensor( - self, inputs, weight_collections=None, trainable=None - ): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): # Do nothing with weight_collections and trainable since no variables are # created in this function. del weight_collections del trainable if not isinstance( self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn) - ): # pylint: disable=protected-access + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), + ): raise ValueError( 'In indicator_column: {}. ' 'categorical_column must be of type _SequenceCategoricalColumn ' 'to use SequenceFeatures. ' 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format( - self.name, type(self.categorical_column), self.categorical_column - ) + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) ) # Feature has been already transformed. Return the intermediate # representation created by _transform_feature. dense_tensor = inputs.get(self) - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access - sequence_length = fc_utils.sequence_length_from_sparse_tensor( - sparse_tensors.id_tensor - ) - return SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=sequence_length - ) + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) + sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) + return SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) @property def parents(self): @@ -5283,9 +4737,7 @@ def parents(self): def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) - config['categorical_column'] = serialize_feature_column( - self.categorical_column - ) + config['categorical_column'] = serialize_feature_column(self.categorical_column) return config @classmethod @@ -5312,9 +4764,7 @@ def _verify_static_batch_size_equality(tensors, columns): # bath_size is a tf.compat.v1.Dimension object. expected_batch_size = None for i in range(0, len(tensors)): - batch_size = tensor_shape.Dimension( - tensor_shape.dimension_value(tensors[i].shape[0]) - ) + batch_size = tensor_shape.Dimension(tensor_shape.dimension_value(tensors[i].shape[0])) if batch_size.value is not None: if expected_batch_size is None: bath_size_column_index = i @@ -5323,25 +4773,24 @@ def _verify_static_batch_size_equality(tensors, columns): raise ValueError( 'Batch size (first dimension) of each feature must be same. ' 'Batch size of columns ({}, {}): ({}, {})'.format( - columns[bath_size_column_index].name, columns[i].name, - expected_batch_size, batch_size + columns[bath_size_column_index].name, + columns[i].name, + expected_batch_size, + batch_size, ) ) class SequenceCategoricalColumn( CategoricalColumn, - fc_old._SequenceCategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('SequenceCategoricalColumn', ('categorical_column')) + fc_old._SequenceCategoricalColumn, + collections.namedtuple('SequenceCategoricalColumn', ('categorical_column')), ): """Represents sequences of categorical data.""" @property def _is_v2_column(self): - return ( - isinstance(self.categorical_column, FeatureColumn) - and self.categorical_column._is_v2_column - ) # pylint: disable=protected-access + return isinstance(self.categorical_column, FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): @@ -5359,25 +4808,19 @@ def parse_example_spec(self): return self.categorical_column.parse_example_spec @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec def transform_feature(self, transformation_cache, state_manager): """See `FeatureColumn` base class.""" - ret_tensor = self.categorical_column.transform_feature( - transformation_cache, state_manager - ) + ret_tensor = self.categorical_column.transform_feature(transformation_cache, state_manager) shape = array_ops.shape(ret_tensor) target_shape = [shape[0], shape[1], math_ops.reduce_prod(shape[2:])] ret_tensor = sparse_ops.sparse_reshape(ret_tensor, target_shape) return ret_tensor - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): ret_tensor = self.categorical_column._transform_feature(inputs) shape = array_ops.shape(ret_tensor) @@ -5391,11 +4834,9 @@ def num_buckets(self): return self.categorical_column.num_buckets @property - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): - return self.categorical_column._num_buckets # pylint: disable=protected-access + return self.categorical_column._num_buckets def _get_sparse_tensors_helper(self, sparse_tensors): id_tensor = sparse_tensors.id_tensor @@ -5432,18 +4873,12 @@ def get_sparse_tensors(self, transformation_cache, state_manager): state_manager: A `StateManager` to create / access resources such as lookup tables. """ - sparse_tensors = self.categorical_column.get_sparse_tensors( - transformation_cache, state_manager - ) + sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) return self._get_sparse_tensors_helper(sparse_tensors) - @deprecation.deprecated( - _FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION - ) - def _get_sparse_tensors( - self, inputs, weight_collections=None, trainable=None - ): - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) return self._get_sparse_tensors_helper(sparse_tensors) @property @@ -5454,9 +4889,7 @@ def parents(self): def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) - config['categorical_column'] = serialize_feature_column( - self.categorical_column - ) + config['categorical_column'] = serialize_feature_column(self.categorical_column) return config @classmethod @@ -5476,9 +4909,7 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): def _check_config_keys(config, expected_keys): """Checks that a config has all expected_keys.""" if set(config.keys()) != set(expected_keys): - raise ValueError( - 'Invalid config: {}, expected keys: {}'.format(config, expected_keys) - ) + raise ValueError('Invalid config: {}, expected keys: {}'.format(config, expected_keys)) def serialize_feature_column(fc): @@ -5524,16 +4955,12 @@ def serialize_feature_column(fc): if isinstance(fc, six.string_types): return fc elif isinstance(fc, FeatureColumn): - return utils.serialize_keras_class_and_config( - fc.__class__.__name__, fc._get_config() - ) + return utils.serialize_keras_class_and_config(fc.__class__.__name__, fc._get_config()) else: raise ValueError('Instance: {} is not a FeatureColumn'.format(fc)) -def deserialize_feature_column( - config, custom_objects=None, columns_by_name=None -): +def deserialize_feature_column(config, custom_objects=None, columns_by_name=None): """Deserializes a `config` generated with `serialize_feature_column`. This method should only be used to deserialize parent FeatureColumns when @@ -5563,11 +4990,19 @@ def deserialize_feature_column( module_feature_column_classes = { cls.__name__: cls for cls in [ - BucketizedColumn, EmbeddingColumn, HashedCategoricalColumn, - IdentityCategoricalColumn, IndicatorColumn, NumericColumn, - SequenceCategoricalColumn, SequenceDenseColumn, SharedEmbeddingColumn, - VocabularyFileCategoricalColumn, VocabularyListCategoricalColumn, - WeightedCategoricalColumn, init_ops.TruncatedNormal + BucketizedColumn, + EmbeddingColumn, + HashedCategoricalColumn, + IdentityCategoricalColumn, + IndicatorColumn, + NumericColumn, + SequenceCategoricalColumn, + SequenceDenseColumn, + SharedEmbeddingColumn, + VocabularyFileCategoricalColumn, + VocabularyListCategoricalColumn, + WeightedCategoricalColumn, + init_ops.TruncatedNormal, ] } if columns_by_name is None: @@ -5577,19 +5012,14 @@ def deserialize_feature_column( config, module_objects=module_feature_column_classes, custom_objects=custom_objects, - printable_module_name='feature_column_v2' + printable_module_name='feature_column_v2', ) if not issubclass(cls, FeatureColumn): - raise ValueError( - 'Expected FeatureColumn class, instead found: {}'.format(cls) - ) + raise ValueError('Expected FeatureColumn class, instead found: {}'.format(cls)) # Always deserialize the FeatureColumn, in order to get the name. - new_instance = cls._from_config( # pylint: disable=protected-access - cls_config, - custom_objects=custom_objects, - columns_by_name=columns_by_name) + new_instance = cls._from_config(cls_config, custom_objects=custom_objects, columns_by_name=columns_by_name) # If the name already exists, re-use the column from columns_by_name, # (new_instance remains unused). @@ -5634,10 +5064,7 @@ def deserialize_feature_columns(configs, custom_objects=None): ValueError if called with input that is not a list of FeatureColumns. """ columns_by_name = {} - return [ - deserialize_feature_column(c, custom_objects, columns_by_name) - for c in configs - ] + return [deserialize_feature_column(c, custom_objects, columns_by_name) for c in configs] def is_embedding_column(fc): diff --git a/easy_rec/python/compat/feature_column/sequence_feature_column.py b/easy_rec/python/compat/feature_column/sequence_feature_column.py index 9c7f3b9b2..862f84068 100644 --- a/easy_rec/python/compat/feature_column/sequence_feature_column.py +++ b/easy_rec/python/compat/feature_column/sequence_feature_column.py @@ -23,13 +23,18 @@ import collections from tensorflow.python.framework import dtypes, ops, tensor_shape +from tensorflow.python.ops import ( # NOQA + array_ops, + check_ops, + math_ops, + parsing_ops, + sparse_ops, +) from easy_rec.python.compat.feature_column import feature_column as fc_v1 from easy_rec.python.compat.feature_column import feature_column_v2 as fc from easy_rec.python.compat.feature_column import utils as fc_utils -from tensorflow.python.ops import array_ops, check_ops, math_ops, parsing_ops, sparse_ops # NOQA - # pylint: disable=protected-access @@ -89,7 +94,7 @@ def __init__(self, feature_columns, trainable=True, name=None, **kwargs): trainable=trainable, name=name, expected_column_type=fc.SequenceDenseColumn, - **kwargs + **kwargs, ) def _target_shape(self, input_shape, total_elements): @@ -114,26 +119,20 @@ def call(self, features): ValueError: If features are not a dictionary. """ if not isinstance(features, dict): - raise ValueError( - 'We expected a dictionary here. Instead we got: ', features - ) + raise ValueError('We expected a dictionary here. Instead we got: ', features) transformation_cache = fc.FeatureTransformationCache(features) output_tensors = [] sequence_lengths = [] for column in self._feature_columns: with ops.name_scope(column.name): - dense_tensor, sequence_length = column.get_sequence_dense_tensor( - transformation_cache, self._state_manager - ) + dense_tensor, sequence_length = column.get_sequence_dense_tensor(transformation_cache, self._state_manager) # Flattens the final dimension to produce a 3D Tensor. output_tensors.append(self._process_dense_tensor(column, dense_tensor)) sequence_lengths.append(sequence_length) # Check and process sequence lengths. - fc._verify_static_batch_size_equality( - sequence_lengths, self._feature_columns - ) + fc._verify_static_batch_size_equality(sequence_lengths, self._feature_columns) sequence_length = _assert_all_equal_and_return(sequence_lengths) return self._verify_and_concat_tensors(output_tensors), sequence_length @@ -163,42 +162,34 @@ def concatenate_context_input(context_input, sequence_input): sequence_input, 3, message='sequence_input must have rank 3', - data=[array_ops.shape(sequence_input)] + data=[array_ops.shape(sequence_input)], ) seq_type_check = check_ops.assert_type( sequence_input, dtypes.float32, - message='sequence_input must have dtype float32; got {}.'.format( - sequence_input.dtype - ) + message='sequence_input must have dtype float32; got {}.'.format(sequence_input.dtype), ) ctx_rank_check = check_ops.assert_rank( context_input, 2, message='context_input must have rank 2', - data=[array_ops.shape(context_input)] + data=[array_ops.shape(context_input)], ) ctx_type_check = check_ops.assert_type( context_input, dtypes.float32, - message='context_input must have dtype float32; got {}.'.format( - context_input.dtype - ) + message='context_input must have dtype float32; got {}.'.format(context_input.dtype), ) - with ops.control_dependencies( - [seq_rank_check, seq_type_check, ctx_rank_check, ctx_type_check] - ): + with ops.control_dependencies([seq_rank_check, seq_type_check, ctx_rank_check, ctx_type_check]): padded_length = array_ops.shape(sequence_input)[1] tiled_context_input = array_ops.tile( array_ops.expand_dims(context_input, 1), - array_ops.concat([[1], [padded_length], [1]], 0) + array_ops.concat([[1], [padded_length], [1]], 0), ) return array_ops.concat([sequence_input, tiled_context_input], 2) -def sequence_categorical_column_with_identity( - key, num_buckets, default_value=None, feature_name=None -): +def sequence_categorical_column_with_identity(key, num_buckets, default_value=None, feature_name=None): """Returns a feature column that represents sequences of integers. Pass this to `embedding_column` or `indicator_column` to convert sequence @@ -243,22 +234,18 @@ def sequence_categorical_column_with_identity( feature_name=feature_name, key=key, num_buckets=num_buckets, - default_value=default_value + default_value=default_value, ) ) def sequence_numeric_column_with_bucketized_column(source_column, boundaries): - if not isinstance(source_column, (SequenceNumericColumn, )): # pylint: disable=protected-access + if not isinstance(source_column, (SequenceNumericColumn,)): # pylint: disable=protected-access raise ValueError( - 'source_column must be a column generated with sequence_numeric_column(). ' - 'Given: {}'.format(source_column) + 'source_column must be a column generated with sequence_numeric_column(). ' 'Given: {}'.format(source_column) ) if len(source_column.shape) > 1: - raise ValueError( - 'source_column must be one-dimensional column. ' - 'Given: {}'.format(source_column) - ) + raise ValueError('source_column must be one-dimensional column. ' 'Given: {}'.format(source_column)) if not boundaries: raise ValueError('boundaries must not be empty.') if not (isinstance(boundaries, list) or isinstance(boundaries, tuple)): @@ -270,35 +257,27 @@ def sequence_numeric_column_with_bucketized_column(source_column, boundaries): def sequence_numeric_column_with_raw_column(source_column, sequence_length): - if not isinstance(source_column, (SequenceNumericColumn, )): # pylint: disable=protected-access + if not isinstance(source_column, (SequenceNumericColumn,)): # pylint: disable=protected-access raise ValueError( - 'source_column must be a column generated with sequence_numeric_column(). ' - 'Given: {}'.format(source_column) + 'source_column must be a column generated with sequence_numeric_column(). ' 'Given: {}'.format(source_column) ) if len(source_column.shape) > 1: - raise ValueError( - 'source_column must be one-dimensional column. ' - 'Given: {}'.format(source_column) - ) + raise ValueError('source_column must be one-dimensional column. ' 'Given: {}'.format(source_column)) return fc.SequenceNumericColumn(source_column, sequence_length) -def sequence_weighted_categorical_column( - categorical_column, weight_feature_key, dtype=dtypes.float32 -): +def sequence_weighted_categorical_column(categorical_column, weight_feature_key, dtype=dtypes.float32): if (dtype is None) or not (dtype.is_integer or dtype.is_floating): raise ValueError('dtype {} is not convertible to float.'.format(dtype)) return fc.SequenceWeightedCategoricalColumn( categorical_column=categorical_column, weight_feature_key=weight_feature_key, - dtype=dtype + dtype=dtype, ) -def sequence_categorical_column_with_hash_bucket( - key, hash_bucket_size, dtype=dtypes.string, feature_name=None -): +def sequence_categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dtypes.string, feature_name=None): """A sequence of categorical terms where ids are set by hashing. Pass this to `embedding_column` or `indicator_column` to convert sequence @@ -340,7 +319,7 @@ def sequence_categorical_column_with_hash_bucket( feature_name=feature_name, key=key, hash_bucket_size=hash_bucket_size, - dtype=dtype + dtype=dtype, ) ) @@ -352,7 +331,7 @@ def sequence_categorical_column_with_vocabulary_file( num_oov_buckets=0, default_value=None, dtype=dtypes.string, - feature_name=None + feature_name=None, ): """A sequence of categorical terms where ids use a vocabulary file. @@ -413,7 +392,7 @@ def sequence_categorical_column_with_vocabulary_file( vocabulary_size=vocabulary_size, num_oov_buckets=num_oov_buckets, default_value=default_value, - dtype=dtype + dtype=dtype, ) ) @@ -424,7 +403,7 @@ def sequence_categorical_column_with_vocabulary_list( dtype=None, default_value=-1, num_oov_buckets=0, - feature_name=None + feature_name=None, ): """A sequence of categorical terms where ids use an in-memory list. @@ -483,18 +462,18 @@ def sequence_categorical_column_with_vocabulary_list( vocabulary_list=vocabulary_list, dtype=dtype, default_value=default_value, - num_oov_buckets=num_oov_buckets + num_oov_buckets=num_oov_buckets, ) ) def sequence_numeric_column( key, - shape=(1, ), - default_value=0., + shape=(1,), + default_value=0.0, dtype=dtypes.float32, normalizer_fn=None, - feature_name=None + feature_name=None, ): """Returns a feature column that represents sequences of numeric data. @@ -538,14 +517,9 @@ def sequence_numeric_column( """ shape = fc._check_shape(shape=shape, key=key) if not (dtype.is_integer or dtype.is_floating): - raise ValueError( - 'dtype must be convertible to float. ' - 'dtype: {}, key: {}'.format(dtype, key) - ) + raise ValueError('dtype must be convertible to float. ' 'dtype: {}, key: {}'.format(dtype, key)) if normalizer_fn is not None and not callable(normalizer_fn): - raise TypeError( - 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn) - ) + raise TypeError('normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) return SequenceNumericColumn( feature_name=feature_name, @@ -553,7 +527,7 @@ def sequence_numeric_column( shape=shape, default_value=default_value, dtype=dtype, - normalizer_fn=normalizer_fn + normalizer_fn=normalizer_fn, ) @@ -570,12 +544,12 @@ def _assert_all_equal_and_return(tensors, name=None): class SequenceNumericColumn( - fc.SequenceDenseColumn, fc_v1._FeatureColumn, + fc.SequenceDenseColumn, + fc_v1._FeatureColumn, collections.namedtuple( - 'SequenceNumericColumn', ( - 'feature_name', 'key', 'shape', 'default_value', 'dtype', 'normalizer_fn' - ) - ) + 'SequenceNumericColumn', + ('feature_name', 'key', 'shape', 'default_value', 'dtype', 'normalizer_fn'), + ), ): """Represents sequences of numeric data.""" @@ -639,13 +613,9 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): lookup tables. """ sp_tensor = transformation_cache.get(self, state_manager) - dense_tensor = sparse_ops.sparse_tensor_to_dense( - sp_tensor, default_value=self.default_value - ) + dense_tensor = sparse_ops.sparse_tensor_to_dense(sp_tensor, default_value=self.default_value) # Reshape into [batch_size, T, variable_shape]. - dense_shape = array_ops.concat( - [array_ops.shape(dense_tensor)[:1], [-1], self.variable_shape], axis=0 - ) + dense_shape = array_ops.concat([array_ops.shape(dense_tensor)[:1], [-1], self.variable_shape], axis=0) dense_tensor = array_ops.reshape(dense_tensor, shape=dense_shape) # Get the number of timesteps per example @@ -656,13 +626,9 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): num_elements = self.variable_shape.num_elements() else: num_elements = 1 - seq_length = fc_utils.sequence_length_from_sparse_tensor( - sp_tensor, num_elements=num_elements - ) + seq_length = fc_utils.sequence_length_from_sparse_tensor(sp_tensor, num_elements=num_elements) - return fc.SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=seq_length - ) + return fc.SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=seq_length) # TODO(b/119409767): Implement parents, _{get,from}_config. @property diff --git a/easy_rec/python/compat/feature_column/utils.py b/easy_rec/python/compat/feature_column/utils.py index 7418d1e66..da303c7d3 100644 --- a/easy_rec/python/compat/feature_column/utils.py +++ b/easy_rec/python/compat/feature_column/utils.py @@ -38,9 +38,7 @@ def sequence_length_from_sparse_tensor(sp_tensor, num_elements=1): # Example: orig tensor [[1, 2], [3]], col_ids = (0, 1, 1), # row_ids = (0, 0, 1), seq_length = [2, 1]. If num_elements = 2, # these will get grouped, and the final seq_length is [1, 1] - seq_length = math_ops.cast( - math_ops.ceil(seq_length / num_elements), dtypes.int64 - ) + seq_length = math_ops.cast(math_ops.ceil(seq_length / num_elements), dtypes.int64) # If the last n rows do not have ids, seq_length will have shape # [batch_size - n]. Pad the remaining values with zeros. @@ -51,18 +49,12 @@ def sequence_length_from_sparse_tensor(sp_tensor, num_elements=1): def assert_string_or_int(dtype, prefix): if (dtype != dtypes.string) and (not dtype.is_integer): - raise ValueError( - '{} dtype must be string or integer. dtype: {}.'.format(prefix, dtype) - ) + raise ValueError('{} dtype must be string or integer. dtype: {}.'.format(prefix, dtype)) def assert_key_is_string(key): if not isinstance(key, six.string_types): - raise ValueError( - 'key must be a string. Got: type {}. Given key: {}.'.format( - type(key), key - ) - ) + raise ValueError('key must be a string. Got: type {}. Given key: {}.'.format(type(key), key)) def check_default_value(shape, default_value, dtype, key): @@ -106,26 +98,22 @@ def check_default_value(shape, default_value, dtype, key): if nest.is_sequence(default_value): if not _is_shape_and_default_value_compatible(default_value, shape): raise ValueError( - 'The shape of default_value must be equal to given shape. ' - 'default_value: {}, shape: {}, key: {}'.format( + 'The shape of default_value must be equal to given shape. ' 'default_value: {}, shape: {}, key: {}'.format( default_value, shape, key ) ) # Check if the values in the list are all integers or are convertible to # floats. - is_list_all_int = all( - isinstance(v, int) for v in nest.flatten(default_value) - ) - is_list_has_float = any( - isinstance(v, float) for v in nest.flatten(default_value) - ) + is_list_all_int = all(isinstance(v, int) for v in nest.flatten(default_value)) + is_list_has_float = any(isinstance(v, float) for v in nest.flatten(default_value)) if is_list_all_int: return _as_tuple(default_value) if is_list_has_float and dtype.is_floating: return _as_tuple(default_value) raise TypeError( - 'default_value must be compatible with dtype. ' - 'default_value: {}, dtype: {}, key: {}'.format(default_value, dtype, key) + 'default_value must be compatible with dtype. ' 'default_value: {}, dtype: {}, key: {}'.format( + default_value, dtype, key + ) ) diff --git a/easy_rec/python/compat/layers.py b/easy_rec/python/compat/layers.py index 75b170ce4..f94c15320 100644 --- a/easy_rec/python/compat/layers.py +++ b/easy_rec/python/compat/layers.py @@ -34,7 +34,7 @@ def layer_norm( trainable=True, begin_norm_axis=1, begin_params_axis=-1, - scope=None + scope=None, ): """Adds a Layer Normalization layer. @@ -94,9 +94,7 @@ def layer_norm( or if `inputs.shape[begin_params_axis:]` is not fully defined at graph build time. """ - with variable_scope.variable_scope( - scope, 'LayerNorm', [inputs], reuse=reuse - ) as sc: + with variable_scope.variable_scope(scope, 'LayerNorm', [inputs], reuse=reuse) as sc: inputs = ops.convert_to_tensor(inputs) inputs_shape = inputs.shape inputs_rank = inputs_shape.ndims @@ -108,40 +106,34 @@ def layer_norm( if begin_params_axis >= inputs_rank or begin_norm_axis >= inputs_rank: raise ValueError( 'begin_params_axis (%d) and begin_norm_axis (%d) ' - 'must be < rank(inputs) (%d)' % - (begin_params_axis, begin_norm_axis, inputs_rank) + 'must be < rank(inputs) (%d)' % (begin_params_axis, begin_norm_axis, inputs_rank) ) params_shape = inputs_shape[begin_params_axis:] if not params_shape.is_fully_defined(): raise ValueError( - 'Inputs %s: shape(inputs)[%s:] is not fully defined: %s' % - (inputs.name, begin_params_axis, inputs_shape) + 'Inputs %s: shape(inputs)[%s:] is not fully defined: %s' % (inputs.name, begin_params_axis, inputs_shape) ) # Allocate parameters for the beta and gamma of the normalization. beta, gamma = None, None if center: - beta_collections = get_variable_collections( - variables_collections, 'beta' - ) + beta_collections = get_variable_collections(variables_collections, 'beta') beta = model_variable( 'beta', shape=params_shape, dtype=dtype, initializer=init_ops.zeros_initializer(), collections=beta_collections, - trainable=trainable + trainable=trainable, ) if scale: - gamma_collections = get_variable_collections( - variables_collections, 'gamma' - ) + gamma_collections = get_variable_collections(variables_collections, 'gamma') gamma = model_variable( 'gamma', shape=params_shape, dtype=dtype, initializer=init_ops.ones_initializer(), collections=gamma_collections, - trainable=trainable + trainable=trainable, ) # Calculate the moments on the last axis (layer activations). norm_axes = list(range(begin_norm_axis, inputs_rank)) @@ -154,7 +146,7 @@ def layer_norm( variance, offset=beta, scale=gamma, - variance_epsilon=variance_epsilon + variance_epsilon=variance_epsilon, ) outputs.set_shape(inputs_shape) if activation_fn is not None: @@ -224,7 +216,7 @@ def variable( device=None, partitioner=None, custom_getter=None, - use_resource=None + use_resource=None, ): """Gets an existing variable with these parameters or creates a new one. @@ -255,18 +247,13 @@ def variable( Returns: The created or existing variable. """ - collections = list( - collections - if collections is not None else [ops.GraphKeys.GLOBAL_VARIABLES] - ) + collections = list(collections if collections is not None else [ops.GraphKeys.GLOBAL_VARIABLES]) # Remove duplicates collections = list(set(collections)) getter = variable_scope.get_variable if custom_getter is not None: - getter = functools.partial( - custom_getter, reuse=variable_scope.get_variable_scope().reuse - ) + getter = functools.partial(custom_getter, reuse=variable_scope.get_variable_scope().reuse) with ops.device(device or ''): return getter( name, @@ -278,7 +265,7 @@ def variable( collections=collections, caching_device=caching_device, partitioner=partitioner, - use_resource=use_resource + use_resource=use_resource, ) @@ -294,7 +281,7 @@ def model_variable( device=None, partitioner=None, custom_getter=None, - use_resource=None + use_resource=None, ): """Gets an existing model variable with these parameters or creates a new one. @@ -327,9 +314,7 @@ def model_variable( The created or existing variable. """ collections = list(collections or []) - collections += [ - ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.MODEL_VARIABLES - ] + collections += [ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.MODEL_VARIABLES] var = variable( name, shape=shape, @@ -342,6 +327,6 @@ def model_variable( device=device, partitioner=partitioner, custom_getter=custom_getter, - use_resource=use_resource + use_resource=use_resource, ) return var diff --git a/easy_rec/python/compat/optimizers.py b/easy_rec/python/compat/optimizers.py index eaf45975b..2ef07ed68 100644 --- a/easy_rec/python/compat/optimizers.py +++ b/easy_rec/python/compat/optimizers.py @@ -21,8 +21,20 @@ import six import tensorflow as tf + # from tensorflow.contrib import framework as contrib_framework from tensorflow.python.framework import dtypes, ops + +# from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import ( # NOQA + array_ops, + clip_ops, + control_flow_ops, + gen_nn_ops, + init_ops, + math_ops, + random_ops, +) from tensorflow.python.ops import variable_scope as vs from tensorflow.python.ops import variables as vars_ from tensorflow.python.summary import summary @@ -33,9 +45,6 @@ from easy_rec.python.ops.incr_record import set_sparse_indices from easy_rec.python.utils import constant, estimator_utils -# from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops, clip_ops, control_flow_ops, gen_nn_ops, init_ops, math_ops, random_ops # NOQA - try: from tensorflow.python.framework import indexed_slices except Exception: @@ -54,18 +63,12 @@ sok = None OPTIMIZER_CLS_NAMES = { - 'Adagrad': - train.AdagradOptimizer, - 'Adam': - train.AdamOptimizer, - 'Ftrl': - train.FtrlOptimizer, - 'Momentum': - lambda learning_rate: train.MomentumOptimizer(learning_rate, momentum=0.9), # pylint: disable=line-too-long - 'RMSProp': - train.RMSPropOptimizer, - 'SGD': - train.GradientDescentOptimizer, + 'Adagrad': train.AdagradOptimizer, + 'Adam': train.AdamOptimizer, + 'Ftrl': train.FtrlOptimizer, + 'Momentum': lambda learning_rate: train.MomentumOptimizer(learning_rate, momentum=0.9), + 'RMSProp': train.RMSPropOptimizer, + 'SGD': train.GradientDescentOptimizer, } OPTIMIZER_SUMMARIES = [ @@ -94,7 +97,7 @@ def optimize_loss( not_apply_grad_after_first_step=False, increment_global_step=True, incr_save=False, - embedding_parallel=False + embedding_parallel=False, ): """Given loss and parameters for optimizer, returns a training op. @@ -201,18 +204,16 @@ def optimize_loss( # Learning rate variable, with possible decay. lr = None if learning_rate is not None: - if ( - isinstance(learning_rate, ops.Tensor) - and learning_rate.get_shape().ndims == 0 - ): + if isinstance(learning_rate, ops.Tensor) and learning_rate.get_shape().ndims == 0: lr = learning_rate elif isinstance(learning_rate, float): if learning_rate < 0.0: raise ValueError('Invalid learning_rate %s.', learning_rate) lr = vs.get_variable( - 'learning_rate', [], + 'learning_rate', + [], trainable=False, - initializer=init_ops.constant_initializer(learning_rate) + initializer=init_ops.constant_initializer(learning_rate), ) else: raise ValueError( @@ -224,10 +225,7 @@ def optimize_loss( else: for summ in summaries: if summ not in OPTIMIZER_SUMMARIES: - raise ValueError( - 'Summaries should be one of [%s], you provided %s.' % - (', '.join(OPTIMIZER_SUMMARIES), summ) - ) + raise ValueError('Summaries should be one of [%s], you provided %s.' % (', '.join(OPTIMIZER_SUMMARIES), summ)) if learning_rate is not None and learning_rate_decay_fn is not None: if global_step is None: raise ValueError('global_step is required for learning_rate_decay_fn.') @@ -238,25 +236,15 @@ def optimize_loss( # Create optimizer, given specified parameters. if isinstance(optimizer, six.string_types): if lr is None: - raise ValueError( - 'Learning rate is None, but should be specified if ' - 'optimizer is string (%s).' % optimizer - ) + raise ValueError('Learning rate is None, but should be specified if ' 'optimizer is string (%s).' % optimizer) if optimizer not in OPTIMIZER_CLS_NAMES: raise ValueError( - 'Optimizer name should be one of [%s], you provided %s.' % - (', '.join(OPTIMIZER_CLS_NAMES), optimizer) + 'Optimizer name should be one of [%s], you provided %s.' % (', '.join(OPTIMIZER_CLS_NAMES), optimizer) ) opt = OPTIMIZER_CLS_NAMES[optimizer](learning_rate=lr) - elif ( - isinstance(optimizer, type) - and issubclass(optimizer, optimizer_.Optimizer) - ): + elif isinstance(optimizer, type) and issubclass(optimizer, optimizer_.Optimizer): if lr is None: - raise ValueError( - 'Learning rate is None, but should be specified if ' - 'optimizer is class (%s).' % optimizer - ) + raise ValueError('Learning rate is None, but should be specified if ' 'optimizer is class (%s).' % optimizer) opt = optimizer(learning_rate=lr) elif isinstance(optimizer, optimizer_.Optimizer): opt = optimizer @@ -266,12 +254,10 @@ def optimize_loss( else: opt = optimizer() if not isinstance(opt, optimizer_.Optimizer): - raise ValueError( - 'Unrecognized optimizer: function should return ' - 'subclass of Optimizer. Got %s.' % str(opt) - ) - elif isinstance(optimizer, sok_optimizer.OptimizerWrapperV1) or \ - isinstance(optimizer, sok_optimizer.OptimizerWrapperV2): + raise ValueError('Unrecognized optimizer: function should return ' 'subclass of Optimizer. Got %s.' % str(opt)) + elif isinstance(optimizer, sok_optimizer.OptimizerWrapperV1) or isinstance( + optimizer, sok_optimizer.OptimizerWrapperV2 + ): opt = optimizer else: raise ValueError( @@ -286,9 +272,7 @@ def optimize_loss( variables = vars_.trainable_variables() # Compute gradients. - gradients = opt.compute_gradients( - loss, variables, colocate_gradients_with_ops=colocate_gradients_with_ops - ) + gradients = opt.compute_gradients(loss, variables, colocate_gradients_with_ops=colocate_gradients_with_ops) if estimator_utils.has_hvd() and hvd.size() > 1: if not embedding_parallel: @@ -298,8 +282,11 @@ def optimize_loss( reduced_grads.append( ( hvd.allreduce( - g, op=hvd.Average, compression=hvd.compression.NoneCompressor - ), v + g, + op=hvd.Average, + compression=hvd.compression.NoneCompressor, + ), + v, ) ) gradients = reduced_grads @@ -326,9 +313,8 @@ def optimize_loss( else: reduced_grads.append( ( - indexed_slices.IndexedSlices( - indices=g.indices, values=g.values / hvd.size() - ), v + indexed_slices.IndexedSlices(indices=g.indices, values=g.values / hvd.size()), + v, ) ) @@ -338,14 +324,16 @@ def optimize_loss( reduced_part_grads = hvd.grouped_allreduce( part_grads, op=hvd.Average, - compression=hvd.compression.NoneCompressor + compression=hvd.compression.NoneCompressor, ) for g, v in zip(reduced_part_grads, part_vars): reduced_grads.append((g, v)) else: for g, v in zip(part_grads, part_vars): g = hvd.allreduce( - g, op=hvd.Average, compression=hvd.compression.NoneCompressor + g, + op=hvd.Average, + compression=hvd.compression.NoneCompressor, ) reduced_grads.append((g, v)) if len(part_sparse_grads) > 0: @@ -353,23 +341,23 @@ def optimize_loss( reduced_part_grads = hvd.grouped_allreduce( part_sparse_grads, op=hvd.Average, - compression=hvd.compression.NoneCompressor + compression=hvd.compression.NoneCompressor, ) for g, v in zip(reduced_part_grads, part_sparse_vars): reduced_grads.append((g, v)) else: for g, v in zip(part_sparse_grads, part_sparse_vars): g = hvd.allreduce( - g, op=hvd.Average, compression=hvd.compression.NoneCompressor + g, + op=hvd.Average, + compression=hvd.compression.NoneCompressor, ) reduced_grads.append((g, v)) gradients = reduced_grads # Optionally add gradient noise. if gradient_noise_scale is not None: - gradients = _add_scaled_noise_to_gradients( - gradients, gradient_noise_scale - ) + gradients = _add_scaled_noise_to_gradients(gradients, gradient_noise_scale) # Multiply some gradients. if gradient_multipliers is not None: @@ -387,24 +375,18 @@ def optimize_loss( # Optionally clip gradients by global norm. if isinstance(clip_gradients, float): # gradients = _clip_gradients_by_norm(gradients, clip_gradients) - sparse_norm, dense_norm, grad_norm = _get_grad_norm( - gradients, embedding_parallel - ) + sparse_norm, dense_norm, grad_norm = _get_grad_norm(gradients, embedding_parallel) summary.scalar('global_norm/sparse_grad', sparse_norm) summary.scalar('global_norm/dense_grad', dense_norm) summary.scalar('global_norm/gradient_norm', grad_norm) grads = [x[0] for x in gradients] vars = [x[1] for x in gradients] - clipped_grads, _ = clip_ops.clip_by_global_norm( - grads, clip_gradients, use_norm=grad_norm - ) + clipped_grads, _ = clip_ops.clip_by_global_norm(grads, clip_gradients, use_norm=grad_norm) gradients = list(zip(clipped_grads, vars)) elif callable(clip_gradients): gradients = clip_gradients(gradients) elif clip_gradients is not None: - raise ValueError( - 'Unknown type %s for clip_gradients' % type(clip_gradients) - ) + raise ValueError('Unknown type %s for clip_gradients' % type(clip_gradients)) # Add scalar summary for loss. if 'loss' in summaries: @@ -425,15 +407,11 @@ def optimize_loss( if 'gradient_norm' in summaries: summary.scalar( 'gradient_norm/%s' % var_name, - clip_ops.global_norm([grad_values]) + clip_ops.global_norm([grad_values]), ) - if clip_gradients is not None and ( - 'global_gradient_norm' in summaries or 'gradient_norm' in summaries - ): - sparse_norm, dense_norm, grad_norm = _get_grad_norm( - gradients, embedding_parallel - ) + if clip_gradients is not None and ('global_gradient_norm' in summaries or 'gradient_norm' in summaries): + sparse_norm, dense_norm, grad_norm = _get_grad_norm(gradients, embedding_parallel) summary.scalar('global_norm/clipped_sparse_grad', sparse_norm) summary.scalar('global_norm/clipped_dense_grad', dense_norm) summary.scalar('global_norm/clipped_gradient_norm', grad_norm) @@ -443,7 +421,7 @@ def _apply_grad(): grad_updates = opt.apply_gradients( gradients, global_step=global_step if increment_global_step else None, - name='train' + name='train', ) embed_para_vars = ops.get_collection(constant.EmbeddingParallel) @@ -452,9 +430,7 @@ def _apply_grad(): if var.name in embed_para_vars: for slot_name in slot_names: tmp_var = opt.get_slot(var, slot_name) - logging.info( - 'add shard embedding optimizer var: %s' % tmp_var.name - ) + logging.info('add shard embedding optimizer var: %s' % tmp_var.name) ops.add_to_collection(constant.EmbeddingParallel, tmp_var.name) incr_save_ops = [] @@ -462,14 +438,10 @@ def _apply_grad(): for grad, var in gradients: if isinstance(grad, indexed_slices.IndexedSlices): indices = grad.indices - with ops.colocate_with(var), ops.control_dependencies( - [grad_updates] - ): + with ops.colocate_with(var), ops.control_dependencies([grad_updates]): incr_save_op = set_sparse_indices(indices, var_name=var.op.name) incr_save_ops.append(incr_save_op) - ops.add_to_collection( - 'SPARSE_UPDATE_VARIABLES', (var, grad.indices.dtype) - ) + ops.add_to_collection('SPARSE_UPDATE_VARIABLES', (var, grad.indices.dtype)) else: ops.add_to_collection('DENSE_UPDATE_VARIABLES', var) return tf.group(incr_save_ops) @@ -500,20 +472,16 @@ def _get_grad_norm(grads_and_vars, embedding_parallel=False): else: dense_norms.append(gen_nn_ops.l2_loss(grad)) if hvd is not None and part_norms: - reduced_norms = hvd.grouped_allreduce( - part_norms, op=hvd.Sum, compression=hvd.compression.NoneCompressor - ) + reduced_norms = hvd.grouped_allreduce(part_norms, op=hvd.Sum, compression=hvd.compression.NoneCompressor) sparse_norms = sparse_norms + reduced_norms all_norms = sparse_norms + dense_norms - sparse_norm = math_ops.sqrt( - math_ops.reduce_sum(array_ops.stack(sparse_norms) * 2.0) - ) if sparse_norms else tf.constant(0.0) - dense_norm = math_ops.sqrt( - math_ops.reduce_sum(array_ops.stack(dense_norms) * 2.0) - ) if dense_norms else tf.constant(0.0) - grad_norm = math_ops.sqrt( - math_ops.reduce_sum(array_ops.stack(all_norms)) * 2.0 - ) if all_norms else tf.constant(0.0) + sparse_norm = ( + math_ops.sqrt(math_ops.reduce_sum(array_ops.stack(sparse_norms) * 2.0)) if sparse_norms else tf.constant(0.0) + ) + dense_norm = ( + math_ops.sqrt(math_ops.reduce_sum(array_ops.stack(dense_norms) * 2.0)) if dense_norms else tf.constant(0.0) + ) + grad_norm = math_ops.sqrt(math_ops.reduce_sum(array_ops.stack(all_norms)) * 2.0) if all_norms else tf.constant(0.0) return sparse_norm, dense_norm, grad_norm @@ -521,9 +489,7 @@ def _clip_gradients_by_norm(grads_and_vars, clip_gradients): """Clips gradients by global norm.""" gradients, variables = zip(*grads_and_vars) - clipped_gradients, _ = clip_ops.clip_by_global_norm( - gradients, clip_gradients - ) + clipped_gradients, _ = clip_ops.clip_by_global_norm(gradients, clip_gradients) return list(zip(clipped_gradients, variables)) @@ -538,16 +504,14 @@ def moving_average(name, value, decay): shape=value.get_shape(), dtype=value.dtype, initializer=init_ops.zeros_initializer(), - trainable=False - ) - return moving_averages.assign_moving_average( - moving_average_variable, value, decay, zero_debias=False + trainable=False, ) + return moving_averages.assign_moving_average(moving_average_variable, value, decay, zero_debias=False) # quicker adaptation at the beginning if global_step is not None: n = math_ops.cast(global_step, dtypes.float32) - decay = math_ops.minimum(decay, n / (n + 1.)) + decay = math_ops.minimum(decay, n / (n + 1.0)) # update averages mean = moving_average('mean', log_norm, decay) @@ -560,13 +524,13 @@ def moving_average(name, value, decay): def adaptive_clipping_fn( - std_factor=2., + std_factor=2.0, decay=0.95, static_max_norm=None, global_step=None, report_summary=False, epsilon=1e-8, - name=None + name=None, ): """Adapt the clipping value using statistics on the norms. @@ -599,19 +563,14 @@ def gradient_clipping(grads_and_vars): norm = clip_ops.global_norm(grads) - max_norm, log_mean = _adaptive_max_norm( - norm, std_factor, decay, global_step, epsilon, name - ) + max_norm, log_mean = _adaptive_max_norm(norm, std_factor, decay, global_step, epsilon, name) # reports the max gradient norm for debugging if report_summary: summary.scalar('global_norm/adaptive_max_gradient_norm', max_norm) # factor will be 1. if norm is smaller than max_norm - factor = array_ops.where( - norm < max_norm, array_ops.ones_like(norm), - math_ops.exp(log_mean) / norm - ) + factor = array_ops.where(norm < max_norm, array_ops.ones_like(norm), math_ops.exp(log_mean) / norm) if static_max_norm is not None: factor = math_ops.minimum(static_max_norm / norm, factor) @@ -622,11 +581,7 @@ def gradient_clipping(grads_and_vars): if grad is None: clipped_grads.append(None) elif isinstance(grad, indexed_slices.IndexedSlices): - clipped_grads.append( - indexed_slices.IndexedSlices( - grad.values * factor, grad.indices, grad.dense_shape - ) - ) + clipped_grads.append(indexed_slices.IndexedSlices(grad.values * factor, grad.indices, grad.dense_shape)) else: clipped_grads.append(grad * factor) @@ -656,17 +611,12 @@ def _multiply_gradients(grads_and_vars, gradient_multipliers): """Multiply specified gradients.""" multiplied_grads_and_vars = [] for grad, var in grads_and_vars: - if ( - grad is not None - and (var in gradient_multipliers or var.name in gradient_multipliers) - ): + if grad is not None and (var in gradient_multipliers or var.name in gradient_multipliers): key = var if var in gradient_multipliers else var.name multiplier = gradient_multipliers[key] if isinstance(grad, indexed_slices.IndexedSlices): grad_values = grad.values * multiplier - grad = indexed_slices.IndexedSlices( - grad_values, grad.indices, grad.dense_shape - ) + grad = indexed_slices.IndexedSlices(grad_values, grad.indices, grad.dense_shape) else: grad *= math_ops.cast(multiplier, grad.dtype) multiplied_grads_and_vars.append((grad, var)) diff --git a/easy_rec/python/compat/queues.py b/easy_rec/python/compat/queues.py index b28f73135..ac2c14cf8 100644 --- a/easy_rec/python/compat/queues.py +++ b/easy_rec/python/compat/queues.py @@ -25,7 +25,6 @@ from multiprocessing import context except ImportError: context = None - pass if context is not None: _ForkingPickler = context.reduction.ForkingPickler @@ -38,7 +37,6 @@ class Queue(object): - _sentinel = object() def __init__(self, ctx, maxsize=0, name=''): @@ -67,14 +65,30 @@ def __init__(self, ctx, maxsize=0, name=''): def __getstate__(self): context.assert_spawning(self) return ( - self._ignore_epipe, self._maxsize, self._reader, self._writer, - self._rlock, self._wlock, self._sem, self._opid, self._name, self._run + self._ignore_epipe, + self._maxsize, + self._reader, + self._writer, + self._rlock, + self._wlock, + self._sem, + self._opid, + self._name, + self._run, ) def __setstate__(self, state): ( - self._ignore_epipe, self._maxsize, self._reader, self._writer, - self._rlock, self._wlock, self._sem, self._opid, self._name, self._run + self._ignore_epipe, + self._maxsize, + self._reader, + self._writer, + self._rlock, + self._wlock, + self._sem, + self._opid, + self._name, + self._run, ) = state self._reset() @@ -154,8 +168,7 @@ def put_nowait(self, obj): def close(self, wait_send_finish=True): self._closed = True close = self._close - if not wait_send_finish and self._thread is not None and self._thread.is_alive( - ): + if not wait_send_finish and self._thread is not None and self._thread.is_alive(): try: if self._reader is not None: self._reader.close() @@ -194,11 +207,17 @@ def _start_thread(self): self._thread = threading.Thread( target=self._feed, args=( - self._buffer, self._notempty, self._send_bytes, self._wlock, - self._reader.close, self._writer.close, self._ignore_epipe, - self._on_queue_feeder_error, self._sem + self._buffer, + self._notempty, + self._send_bytes, + self._wlock, + self._reader.close, + self._writer.close, + self._ignore_epipe, + self._on_queue_feeder_error, + self._sem, ), - name='QueueFeederThread' + name='QueueFeederThread', ) self._thread.daemon = True @@ -209,16 +228,13 @@ def _start_thread(self): if not self._joincancelled: self._jointhread = Finalize( self._thread, - Queue._finalize_join, [weakref.ref(self._thread)], - exitpriority=-5 + Queue._finalize_join, + [weakref.ref(self._thread)], + exitpriority=-5, ) # Send sentinel to the thread queue object when garbage collected - self._close = Finalize( - self, - Queue._finalize_close, [self._buffer, self._notempty], - exitpriority=10 - ) + self._close = Finalize(self, Queue._finalize_close, [self._buffer, self._notempty], exitpriority=10) @staticmethod def _finalize_join(twr): @@ -238,8 +254,16 @@ def _finalize_close(buffer, notempty): notempty.notify() def _feed( - self, buffer, notempty, send_bytes, writelock, reader_close, writer_close, - ignore_epipe, onerror, queue_sem + self, + buffer, + notempty, + send_bytes, + writelock, + reader_close, + writer_close, + ignore_epipe, + onerror, + queue_sem, ): logging.debug('starting thread to feed data to pipe') nacquire = notempty.acquire @@ -267,7 +291,6 @@ def _feed( while self._run: obj = bpopleft() if obj is sentinel: - # logging.info('Queue[' + self._name + '] feeder thread got sentinel -- exiting: ' + str(self._run)) reader_close() writer_close() return @@ -286,10 +309,7 @@ def _feed( pass except Exception as e: if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: - logging.warning( - 'Queue[' + name + '] exception: pid=' + str(pid) + ' run=' + - str(self._run) + ' e=' + str(e) - ) + logging.warning('Queue[' + name + '] exception: pid=' + str(pid) + ' run=' + str(self._run) + ' e=' + str(e)) return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. @@ -297,8 +317,7 @@ def _feed( # started to cleanup. if is_exiting(): logging.warning( - 'Queue[' + name + '] thread error in exiting: pid=' + str(pid) + - ' run=' + str(self._run) + ' e=' + str(e) + 'Queue[' + name + '] thread error in exiting: pid=' + str(pid) + ' run=' + str(self._run) + ' e=' + str(e) ) return else: @@ -319,4 +338,5 @@ def _on_queue_feeder_error(e, obj): For overriding by concurrent.futures. """ import traceback + traceback.print_exc() diff --git a/easy_rec/python/compat/regularizers.py b/easy_rec/python/compat/regularizers.py index f49cb87dc..a7a448170 100644 --- a/easy_rec/python/compat/regularizers.py +++ b/easy_rec/python/compat/regularizers.py @@ -25,8 +25,11 @@ from tensorflow.python.platform import tf_logging as logging __all__ = [ - 'l1_regularizer', 'l2_regularizer', 'l1_l2_regularizer', 'sum_regularizer', - 'apply_regularization' + 'l1_regularizer', + 'l2_regularizer', + 'l1_l2_regularizer', + 'sum_regularizer', + 'apply_regularization', ] @@ -48,25 +51,17 @@ def l1_regularizer(scale, scope=None): if isinstance(scale, numbers.Integral): raise ValueError('scale cannot be an integer: %s' % scale) if isinstance(scale, numbers.Real): - if scale < 0.: - raise ValueError( - 'Setting a scale less than 0 on a regularizer: %g' % scale - ) - if scale == 0.: + if scale < 0.0: + raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale) + if scale == 0.0: logging.info('Scale of 0 disables regularizer.') return lambda _: None def l1(weights, name=None): """Applies L1 regularization to weights.""" with ops.name_scope(scope, 'l1_regularizer', [weights]) as name: - my_scale = ops.convert_to_tensor( - scale, dtype=weights.dtype.base_dtype, name='scale' - ) - return standard_ops.multiply( - my_scale, - standard_ops.reduce_sum(standard_ops.abs(weights)), - name=name - ) + my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') + return standard_ops.multiply(my_scale, standard_ops.reduce_sum(standard_ops.abs(weights)), name=name) return l1 @@ -87,22 +82,18 @@ def l2_regularizer(scale, scope=None): ValueError: If scale is negative or if scale is not a float. """ if isinstance(scale, numbers.Integral): - raise ValueError('scale cannot be an integer: %s' % (scale, )) + raise ValueError('scale cannot be an integer: %s' % (scale,)) if isinstance(scale, numbers.Real): - if scale < 0.: - raise ValueError( - 'Setting a scale less than 0 on a regularizer: %g.' % scale - ) - if scale == 0.: + if scale < 0.0: + raise ValueError('Setting a scale less than 0 on a regularizer: %g.' % scale) + if scale == 0.0: logging.info('Scale of 0 disables regularizer.') return lambda _: None def l2(weights): """Applies l2 regularization to weights.""" with ops.name_scope(scope, 'l2_regularizer', [weights]) as name: - my_scale = ops.convert_to_tensor( - scale, dtype=weights.dtype.base_dtype, name='scale' - ) + my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.multiply(my_scale, nn.l2_loss(weights), name=name) return l2 @@ -124,18 +115,15 @@ def l1_l2_regularizer(scale_l1=1.0, scale_l2=1.0, scope=None): ValueError: If scale is negative or if scale is not a float. """ if isinstance(scale_l1, numbers.Integral): - raise ValueError('scale_l1 cannot be an integer: %s' % (scale_l1, )) + raise ValueError('scale_l1 cannot be an integer: %s' % (scale_l1,)) if isinstance(scale_l2, numbers.Integral): - raise ValueError('scale_l2 cannot be an integer: %s' % (scale_l2, )) + raise ValueError('scale_l2 cannot be an integer: %s' % (scale_l2,)) scope = scope or 'l1_l2_regularizer' - if scale_l1 == 0.: + if scale_l1 == 0.0: return l2_regularizer(scale_l2, scope) - if scale_l2 == 0.: + if scale_l2 == 0.0: return l1_regularizer(scale_l1, scope) - return sum_regularizer( - [l1_regularizer(scale_l1), - l2_regularizer(scale_l2)], scope=scope - ) + return sum_regularizer([l1_regularizer(scale_l1), l2_regularizer(scale_l2)], scope=scope) def sum_regularizer(regularizer_list, scope=None): @@ -161,9 +149,7 @@ def sum_reg(weights): tensor = reg(weights) if tensor is not None: regularizer_tensors.append(tensor) - return math_ops.add_n( - regularizer_tensors, name=name - ) if regularizer_tensors else None + return math_ops.add_n(regularizer_tensors, name=name) if regularizer_tensors else None return sum_reg @@ -194,18 +180,13 @@ def apply_regularization(regularizer, weights_list=None): weights_list = ops.get_collection(ops.GraphKeys.WEIGHTS) if not weights_list: raise ValueError('No weights to regularize.') - with ops.name_scope( - 'get_regularization_penalty', values=weights_list - ) as scope: + with ops.name_scope('get_regularization_penalty', values=weights_list) as scope: penalties = [regularizer(w) for w in weights_list] - penalties = [ - p if p is not None else constant_op.constant(0.0) for p in penalties - ] + penalties = [p if p is not None else constant_op.constant(0.0) for p in penalties] for p in penalties: if p.get_shape().ndims != 0: raise ValueError( - 'regularizer must return a scalar Tensor instead of a ' - 'Tensor with rank %d.' % p.get_shape().ndims + 'regularizer must return a scalar Tensor instead of a ' 'Tensor with rank %d.' % p.get_shape().ndims ) summed_penalty = math_ops.add_n(penalties, name=scope) diff --git a/easy_rec/python/compat/sok_optimizer.py b/easy_rec/python/compat/sok_optimizer.py index e8f5a98c1..4c9e3cfb2 100644 --- a/easy_rec/python/compat/sok_optimizer.py +++ b/easy_rec/python/compat/sok_optimizer.py @@ -16,13 +16,19 @@ import tensorflow as tf from tensorflow.python.eager import context + # from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -from easy_rec.python.compat.dynamic_variable import DynamicVariable - # from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import array_ops, gradients, resource_variable_ops, state_ops # NOQA +from tensorflow.python.ops import ( # NOQA + array_ops, + gradients, + resource_variable_ops, + state_ops, +) + +from easy_rec.python.compat.dynamic_variable import DynamicVariable def OptimizerWrapper(optimizer): @@ -77,13 +83,10 @@ def OptimizerWrapper(optimizer): class OptimizerWrapperV1(object): - def __init__(self, optimizer): self._optimizer = optimizer # slots - unused = tf.Variable( - [0.0], dtype=tf.float32, name='unused', trainable=False - ) + unused = tf.Variable([0.0], dtype=tf.float32, name='unused', trainable=False) self._optimizer._create_slots([unused]) names, slots = [], [] for name in self._optimizer.get_slot_names(): @@ -104,7 +107,7 @@ def compute_gradients( var_list=None, aggregation_method=None, colocate_gradients_with_ops=False, - grad_loss=None + grad_loss=None, ): self._loss = loss tmp_grads = gradients.gradients(loss, var_list) @@ -139,7 +142,7 @@ def _create_slots_dynamic(self, var): dimension=var.dimension, initializer=self._initial_vals[slot_name], name='DynamicSlot', - trainable=False + trainable=False, ) else: tmp_config = var.config_dict @@ -151,7 +154,7 @@ def _create_slots_dynamic(self, var): var_type=var.backend_type, name='DynamicSlot', trainable=False, - **tmp_config + **tmp_config, ) self._optimizer._slots[slot_name][key] = slot @@ -169,12 +172,8 @@ def _slots(self): def apply_gradients(self, grads_and_vars, global_step=None, name=None): gradients = grads_and_vars - sparse_vars = [ - x for x in gradients if 'DynamicVariable' in str(type(x[1])) - ] - dense_vars = [ - x for x in gradients if 'DynamicVariable' not in str(type(x[1])) - ] + sparse_vars = [x for x in gradients if 'DynamicVariable' in str(type(x[1]))] + dense_vars = [x for x in gradients if 'DynamicVariable' not in str(type(x[1]))] def _dummy_finish(update_ops, name_scope): return update_ops @@ -184,9 +183,7 @@ def _dummy_finish(update_ops, name_scope): with ops.control_dependencies([array_ops.identity(self._loss)]): sparse_grad_updates = self.apply_sparse_gradients(sparse_vars, name=name) - dense_grad_updates = self._optimizer.apply_gradients( - dense_vars, global_step=None, name=name - ) + dense_grad_updates = self._optimizer.apply_gradients(dense_vars, global_step=None, name=name) if sparse_grad_updates is not None and dense_grad_updates is not None: grad_updates = sparse_grad_updates + dense_grad_updates elif sparse_grad_updates is not None: @@ -203,7 +200,7 @@ def _dummy_finish(update_ops, name_scope): apply_updates = resource_variable_ops.assign_add_variable_op( global_step.handle, ops.convert_to_tensor(1, dtype=global_step.dtype), - name=name + name=name, ) else: apply_updates = state_ops.assign_add(global_step, 1, name=name) @@ -217,9 +214,7 @@ def _dummy_finish(update_ops, name_scope): return apply_updates - def apply_sparse_gradients( - self, grads_and_vars, global_step=None, name=None - ): + def apply_sparse_gradients(self, grads_and_vars, global_step=None, name=None): # 1. Create slots and do sparse_read to_static_ops = [] grad_list, var_list = [], [] @@ -253,7 +248,7 @@ def apply_sparse_gradients( var_type=v.backend_type, name=tmp_slot_var_name, trainable=False, - **tmp_config + **tmp_config, ) self._optimizer._slots[slot_name][key] = slot @@ -266,9 +261,7 @@ def apply_sparse_gradients( # 3. Call tf-optimizer with ops.control_dependencies(to_static_ops): - train_op = self._optimizer.apply_gradients( - zip(grad_list, var_list), global_step=global_step, name=name - ) + train_op = self._optimizer.apply_gradients(zip(grad_list, var_list), global_step=global_step, name=name) # 5. Write buffer back to dynamic variables to_dynamic_ops = [] @@ -286,14 +279,11 @@ def apply_sparse_gradients( class OptimizerWrapperV2(object): - def __init__(self, optimizer): self._optimizer = optimizer # slots if tf.__version__[0] == '1': - unused = tf.Variable( - [0.0], name='unused', trainable=False, use_resource=True - ) + unused = tf.Variable([0.0], name='unused', trainable=False, use_resource=True) else: unused = tf.Variable([0.0], name='unused', trainable=False) self._optimizer._create_slots([unused]) @@ -342,7 +332,7 @@ def _create_slots_dynamic(self, var): var_type=var.backend_type, name='DynamicSlot', trainable=False, - **tmp_config + **tmp_config, ) self._optimizer._slots[key][slot_name] = slot @@ -396,7 +386,7 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): var_type=v.backend_type, name='DynamicSlot', trainable=False, - **tmp_config + **tmp_config, ) self._optimizer._slots[key][slot_name] = slot @@ -413,9 +403,7 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): # 3. Call tf-optimizer with tf.control_dependencies(to_static_ops): - train_op = self._optimizer.apply_gradients( - zip(grad_list, var_list), name=name - ) + train_op = self._optimizer.apply_gradients(zip(grad_list, var_list), name=name) # 4. Switch iterations self._optimizer._iterations = iterations @@ -433,7 +421,6 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): class SGD(object): - def __init__(self, lr): self._lr = tf.Variable(lr) @@ -445,8 +432,6 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): train_ops = [] for g, v in grads_and_vars: if g is not None: - scaled_g = ops.IndexedSlices( - g.values * self._lr, g.indices, g.dense_shape - ) + scaled_g = ops.IndexedSlices(g.values * self._lr, g.indices, g.dense_shape) train_ops.append(v.scatter_sub(scaled_g)) return tf.group(train_ops) diff --git a/easy_rec/python/compat/sync_replicas_optimizer.py b/easy_rec/python/compat/sync_replicas_optimizer.py index 634d589c2..d41ad7249 100644 --- a/easy_rec/python/compat/sync_replicas_optimizer.py +++ b/easy_rec/python/compat/sync_replicas_optimizer.py @@ -13,16 +13,28 @@ # limitations under the License. # ============================================================================== """Synchronize replicas for training.""" + from __future__ import absolute_import, division, print_function from tensorflow.core.framework import types_pb2 from tensorflow.python.framework import errors_impl, ops +from tensorflow.python.ops import ( # NOQA + array_ops, + control_flow_ops, + data_flow_ops, + state_ops, + variable_scope, + variables, +) from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.training import ( # NOQA + optimizer, + queue_runner, + session_manager, + session_run_hook, +) from tensorflow.python.util.tf_export import tf_export -from tensorflow.python.ops import array_ops, control_flow_ops, data_flow_ops, state_ops, variable_scope, variables # NOQA -from tensorflow.python.training import optimizer, queue_runner, session_manager, session_run_hook # NOQA - # Please note that the gradients from replicas are averaged instead of summed # (as in the old sync_replicas_optimizer) so you need to increase the learning @@ -136,7 +148,7 @@ def __init__( variables_to_average=None, use_locking=False, name='sync_replicas', - **extra_args + **extra_args, ): """Construct a sync_replicas optimizer. @@ -165,7 +177,8 @@ def __init__( super(SyncReplicasOptimizer, self).__init__(use_locking, name) logging.info( 'SyncReplicasV2: replicas_to_aggregate=%s; total_num_replicas=%s', - replicas_to_aggregate, total_num_replicas + replicas_to_aggregate, + total_num_replicas, ) self._opt = opt self._replicas_to_aggregate = replicas_to_aggregate @@ -249,14 +262,12 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): trainable=False, collections=[ops.GraphKeys.LOCAL_VARIABLES], dtype=global_step.dtype.base_dtype, - name='sync_rep_local_step' + name='sync_rep_local_step', ) self.local_step_init_op = state_ops.assign(self._local_step, global_step) chief_init_ops = [self.local_step_init_op] - self.ready_for_local_init_op = variables.report_uninitialized_variables( - variables.global_variables() - ) + self.ready_for_local_init_op = variables.report_uninitialized_variables(variables.global_variables()) with ops.name_scope(None, self._name): for grad, var in grads_and_vars: @@ -270,28 +281,18 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): grad_accum = data_flow_ops.ConditionalAccumulator( grad.dtype, shape=var.get_shape(), - shared_name=var.name + '/grad_accum' - ) - train_ops.append( - grad_accum.apply_grad(grad, local_step=self._local_step) - ) - aggregated_grad.append( - grad_accum.take_grad(self._replicas_to_aggregate) + shared_name=var.name + '/grad_accum', ) + train_ops.append(grad_accum.apply_grad(grad, local_step=self._local_step)) + aggregated_grad.append(grad_accum.take_grad(self._replicas_to_aggregate)) else: if not isinstance(grad, ops.IndexedSlices): raise ValueError('Unknown grad type!') grad_accum = data_flow_ops.SparseConditionalAccumulator( grad.dtype, shape=(), shared_name=var.name + '/grad_accum' ) - train_ops.append( - grad_accum.apply_indexed_slices_grad( - grad, local_step=self._local_step - ) - ) - aggregated_grad.append( - grad_accum.take_indexed_slices_grad(self._replicas_to_aggregate) - ) + train_ops.append(grad_accum.apply_indexed_slices_grad(grad, local_step=self._local_step)) + aggregated_grad.append(grad_accum.take_indexed_slices_grad(self._replicas_to_aggregate)) self._accumulator_list.append((grad_accum, var.device)) @@ -299,9 +300,7 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): # sync_op will be assigned to the same device as the global step. with ops.device(global_step.device), ops.name_scope(''): - update_op = self._opt.apply_gradients( - aggregated_grads_and_vars, global_step - ) + update_op = self._opt.apply_gradients(aggregated_grads_and_vars, global_step) def _get_token_qname(): SyncReplicasOptimizer.sync_que_id += 1 @@ -314,32 +313,26 @@ def _get_token_qname(): token_qname = _get_token_qname() logging.info('create sync_token_queue[%s]' % token_qname) with ops.device(global_step.device), ops.name_scope(''): - sync_token_queue = ( - data_flow_ops.FIFOQueue( - -1, - global_step.dtype.base_dtype, - shapes=(), - name=token_qname, - shared_name=token_qname - ) + sync_token_queue = data_flow_ops.FIFOQueue( + -1, + global_step.dtype.base_dtype, + shapes=(), + name=token_qname, + shared_name=token_qname, ) self._sync_token_queue = sync_token_queue self._is_sync_que_closed = sync_token_queue.is_closed() - self._close_sync_que = sync_token_queue.close( - cancel_pending_enqueues=True, name='close_sync_token_queue' - ) + self._close_sync_que = sync_token_queue.close(cancel_pending_enqueues=True, name='close_sync_token_queue') # dummy_queue is passed to the queue runner. Don't use the real queues # because the queue runner doesn't automatically reopen it once it # closed queues in PS devices. - dummy_queue = ( - data_flow_ops.FIFOQueue( - 1, - types_pb2.DT_INT32, - shapes=(), - name='dummy_queue', - shared_name='dummy_queue' - ) + dummy_queue = data_flow_ops.FIFOQueue( + 1, + types_pb2.DT_INT32, + shapes=(), + name='dummy_queue', + shared_name='dummy_queue', ) with ops.device(global_step.device), ops.name_scope(''): @@ -352,23 +345,17 @@ def _get_token_qname(): # Sync_op needs to insert tokens to the token queue at the end of the # step so the replicas can fetch them to start the next step. tokens = array_ops.fill([self._tokens_per_step], global_step) - sync_op = sync_token_queue.enqueue_many((tokens, )) + sync_op = sync_token_queue.enqueue_many((tokens,)) if self._variable_averages is not None: with ops.control_dependencies([sync_op]), ops.name_scope(''): sync_op = self._variable_averages.apply(self._variables_to_average) - self._chief_queue_runner = queue_runner.QueueRunner( - dummy_queue, [sync_op] - ) - ops.add_to_collection( - ops.GraphKeys.QUEUE_RUNNERS, self._chief_queue_runner - ) + self._chief_queue_runner = queue_runner.QueueRunner(dummy_queue, [sync_op]) + ops.add_to_collection(ops.GraphKeys.QUEUE_RUNNERS, self._chief_queue_runner) for accum, dev in self._accumulator_list: with ops.device(dev): - chief_init_ops.append( - accum.set_global_step(global_step, name='SetGlobalStep') - ) + chief_init_ops.append(accum.set_global_step(global_step, name='SetGlobalStep')) self.chief_init_op = control_flow_ops.group(*(chief_init_ops)) self._gradients_applied = True return train_op @@ -453,23 +440,20 @@ def get_init_tokens_op(self, num_tokens=-1): total_num_replicas. """ if self._gradients_applied is False: - raise ValueError( - 'get_init_tokens_op() should be called after apply_gradients().' - ) + raise ValueError('get_init_tokens_op() should be called after apply_gradients().') tokens_needed = self._replicas_to_aggregate - self._total_num_replicas if num_tokens == -1: num_tokens = self._replicas_to_aggregate elif num_tokens < tokens_needed: raise ValueError( - 'Too few tokens to finish the first step: %d (given) vs %d (needed)' % - (num_tokens, tokens_needed) + 'Too few tokens to finish the first step: %d (given) vs %d (needed)' % (num_tokens, tokens_needed) ) if num_tokens > 0: with ops.device(self._global_step.device), ops.name_scope(''): tokens = array_ops.fill([num_tokens], self._global_step) - init_tokens = self._sync_token_queue.enqueue_many((tokens, )) + init_tokens = self._sync_token_queue.enqueue_many((tokens,)) else: init_tokens = control_flow_ops.no_op(name='no_init_tokens') @@ -497,30 +481,26 @@ def __init__(self, sync_optimizer, is_chief, num_tokens): def begin(self): if self._sync_optimizer._gradients_applied is False: # pylint: disable=protected-access - raise ValueError( - 'SyncReplicasOptimizer.apply_gradient should be called before using ' - 'the hook.' - ) + raise ValueError('SyncReplicasOptimizer.apply_gradient should be called before using ' 'the hook.') if self._is_chief: self._local_init_op = self._sync_optimizer.chief_init_op - self._ready_for_local_init_op = ( - self._sync_optimizer.ready_for_local_init_op - ) - self._init_tokens_op = self._sync_optimizer.get_init_tokens_op( - self._num_tokens - ) + self._ready_for_local_init_op = self._sync_optimizer.ready_for_local_init_op + self._init_tokens_op = self._sync_optimizer.get_init_tokens_op(self._num_tokens) else: self._local_init_op = self._sync_optimizer.local_step_init_op - self._ready_for_local_init_op = ( - self._sync_optimizer.ready_for_local_init_op - ) + self._ready_for_local_init_op = self._sync_optimizer.ready_for_local_init_op self._init_tokens_op = None def after_create_session(self, session, coord): """Runs SyncReplicasOptimizer initialization ops.""" - local_init_success, msg = session_manager._ready( # pylint: disable=protected-access - self._ready_for_local_init_op, session, - 'Model is not ready for SyncReplicasOptimizer local init.') + ( + local_init_success, + msg, + ) = session_manager._ready( # pylint: disable=protected-access + self._ready_for_local_init_op, + session, + 'Model is not ready for SyncReplicasOptimizer local init.', + ) if not local_init_success: raise RuntimeError( 'Init operations did not make model ready for SyncReplicasOptimizer ' diff --git a/easy_rec/python/compat/weight_decay_optimizers.py b/easy_rec/python/compat/weight_decay_optimizers.py index 664328f99..b4e1b96f6 100755 --- a/easy_rec/python/compat/weight_decay_optimizers.py +++ b/easy_rec/python/compat/weight_decay_optimizers.py @@ -13,16 +13,21 @@ # limitations under the License. # ============================================================================== """Base class to make optimizers weight decay ready.""" + from __future__ import absolute_import, division, print_function from tensorflow.python.framework import ops +from tensorflow.python.ops import ( # NOQA + array_ops, + control_flow_ops, + resource_variable_ops, + state_ops, +) from tensorflow.python.training import adam from tensorflow.python.training import momentum as momentum_opt from tensorflow.python.training import optimizer from tensorflow.python.util.tf_export import tf_export -from tensorflow.python.ops import array_ops, control_flow_ops, resource_variable_ops, state_ops # NOQA - class DecoupledWeightDecayExtension(object): """This class allows to extend optimizers with decoupled weight decay. @@ -97,7 +102,7 @@ def minimize( colocate_gradients_with_ops=False, name=None, grad_loss=None, - decay_var_list=None + decay_var_list=None, ): """Add operations to minimize `loss` by updating `var_list` with decay. @@ -137,12 +142,10 @@ def minimize( aggregation_method=aggregation_method, colocate_gradients_with_ops=colocate_gradients_with_ops, name=name, - grad_loss=grad_loss + grad_loss=grad_loss, ) - def apply_gradients( - self, grads_and_vars, global_step=None, name=None, decay_var_list=None - ): + def apply_gradients(self, grads_and_vars, global_step=None, name=None, decay_var_list=None): """Apply gradients to variables and decay the variables. This function is the same as Optimizer.apply_gradients except that it @@ -174,9 +177,7 @@ def _prepare(self): weight_decay = self._weight_decay if callable(weight_decay): weight_decay = weight_decay() - self._weight_decay_tensor = ops.convert_to_tensor( - weight_decay, name='weight_decay' - ) + self._weight_decay_tensor = ops.convert_to_tensor(weight_decay, name='weight_decay') # Call the optimizers _prepare function. super(DecoupledWeightDecayExtension, self)._prepare() @@ -199,30 +200,25 @@ def _apply_dense(self, grad, var): def _resource_apply_dense(self, grad, var): with ops.control_dependencies([self._decay_weights_op(var)]): - return super(DecoupledWeightDecayExtension, - self)._resource_apply_dense(grad, var) + return super(DecoupledWeightDecayExtension, self)._resource_apply_dense(grad, var) def _apply_sparse(self, grad, var): scatter_add = state_ops.scatter_add decay_op = self._decay_weights_sparse_op(var, grad.indices, scatter_add) with ops.control_dependencies([decay_op]): - return super(DecoupledWeightDecayExtension, - self)._apply_sparse(grad, var) + return super(DecoupledWeightDecayExtension, self)._apply_sparse(grad, var) def _resource_scatter_add(self, x, i, v, _=None): # last argument allows for one overflow argument, to have the same function # signature as state_ops.scatter_add - with ops.control_dependencies( - [resource_variable_ops.resource_scatter_add(x.handle, i, v)] - ): + with ops.control_dependencies([resource_variable_ops.resource_scatter_add(x.handle, i, v)]): return x.value() def _resource_apply_sparse(self, grad, var, indices): scatter_add = self._resource_scatter_add decay_op = self._decay_weights_sparse_op(var, indices, scatter_add) with ops.control_dependencies([decay_op]): - return super(DecoupledWeightDecayExtension, - self)._resource_apply_sparse(grad, var, indices) + return super(DecoupledWeightDecayExtension, self)._resource_apply_sparse(grad, var, indices) def extend_with_decoupled_weight_decay(base_optimizer): @@ -263,9 +259,7 @@ def extend_with_decoupled_weight_decay(base_optimizer): and base_optimizer. """ - class OptimizerWithDecoupledWeightDecay( - DecoupledWeightDecayExtension, base_optimizer - ): + class OptimizerWithDecoupledWeightDecay(DecoupledWeightDecayExtension, base_optimizer): """Base_optimizer with decoupled weight decay. This class computes the update step of `base_optimizer` and @@ -282,17 +276,14 @@ class OptimizerWithDecoupledWeightDecay( def __init__(self, weight_decay, *args, **kwargs): # super delegation is necessary here # pylint: disable=useless-super-delegation - super(OptimizerWithDecoupledWeightDecay, - self).__init__(weight_decay, *args, **kwargs) + super(OptimizerWithDecoupledWeightDecay, self).__init__(weight_decay, *args, **kwargs) # pylint: enable=useless-super-delegation return OptimizerWithDecoupledWeightDecay @tf_export('contrib.opt.MomentumWOptimizer') -class MomentumWOptimizer( - DecoupledWeightDecayExtension, momentum_opt.MomentumOptimizer -): +class MomentumWOptimizer(DecoupledWeightDecayExtension, momentum_opt.MomentumOptimizer): """Optimizer that implements the Momentum algorithm with weight_decay. This is an implementation of the SGDW optimizer described in "Fixing @@ -321,7 +312,7 @@ def __init__( momentum, use_locking=False, name='MomentumW', - use_nesterov=False + use_nesterov=False, ): """Construct a new MomentumW optimizer. @@ -352,7 +343,7 @@ def __init__( momentum=momentum, use_locking=use_locking, name=name, - use_nesterov=use_nesterov + use_nesterov=use_nesterov, ) @@ -388,7 +379,7 @@ def __init__( beta2=0.999, epsilon=1e-8, use_locking=False, - name='AdamW' + name='AdamW', ): """Construct a new AdamW optimizer. @@ -415,7 +406,7 @@ def __init__( beta2=beta2, epsilon=epsilon, use_locking=use_locking, - name=name + name=name, ) @@ -454,7 +445,7 @@ def __init__( beta2=0.999, epsilon=1e-8, use_locking=False, - name='AdamAsyncW' + name='AdamAsyncW', ): """Construct a new AdamW optimizer. @@ -481,7 +472,8 @@ def __init__( beta2=beta2, epsilon=epsilon, use_locking=use_locking, - name=name + name=name, ) + except ImportError: pass diff --git a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py index 938f65c47..0b9ee1d91 100644 --- a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py +++ b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py @@ -18,13 +18,24 @@ from tensorflow.python.eager import context from tensorflow.python.framework import dtypes, ops, sparse_tensor +from tensorflow.python.ops import ( # NOQA + array_ops, + check_ops, + confusion_matrix, + control_flow_ops, + math_ops, + nn, + sets, + sparse_ops, + state_ops, + variable_scope, + weights_broadcast_ops, +) from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import distribution_strategy_context from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.tf_export import tf_export -from tensorflow.python.ops import array_ops, check_ops, confusion_matrix, control_flow_ops, math_ops, nn, sets, sparse_ops, state_ops, variable_scope, weights_broadcast_ops # NOQA - def metric_variable(shape, dtype, validate_shape=True, name=None): """Create variable in `GraphKeys.(LOCAL|METRIC_VARIABLES)` collections. @@ -64,13 +75,11 @@ def metric_variable(shape, dtype, validate_shape=True, name=None): # Note that synchronization "ON_READ" implies trainable=False. return variable_scope.variable( lambda: array_ops.zeros(shape, dtype), - collections=[ - ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES - ], + collections=[ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES], validate_shape=validate_shape, synchronization=variable_scope.VariableSynchronization.ON_READ, aggregation=variable_scope.VariableAggregation.SUM, - name=name + name=name, ) @@ -99,9 +108,7 @@ def _remove_squeezable_dimensions(predictions, labels, weights): """ predictions = ops.convert_to_tensor(predictions) if labels is not None: - labels, predictions = confusion_matrix.remove_squeezable_dimensions( - labels, predictions - ) + labels, predictions = confusion_matrix.remove_squeezable_dimensions(labels, predictions) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if weights is None: @@ -129,30 +136,29 @@ def _remove_squeezable_dimensions(predictions, labels, weights): def _maybe_expand_weights(): return control_flow_ops.cond( math_ops.equal(rank_diff, -1), - lambda: array_ops.expand_dims(weights, [-1]), lambda: weights + lambda: array_ops.expand_dims(weights, [-1]), + lambda: weights, ) # Don't attempt squeeze if it will fail based on static check. - if ( - (weights_rank is not None) - and (not weights_shape.dims[-1].is_compatible_with(1)) - ): + if (weights_rank is not None) and (not weights_shape.dims[-1].is_compatible_with(1)): maybe_squeeze_weights = lambda: weights # noqa: E731 else: - maybe_squeeze_weights = lambda: array_ops.squeeze( # noqa: E731 - weights, [-1]) # noqa: E126 + maybe_squeeze_weights = lambda: array_ops.squeeze(weights, [-1]) # noqa: E731 # noqa: E126 def _maybe_adjust_weights(): return control_flow_ops.cond( - math_ops.equal(rank_diff, 1), maybe_squeeze_weights, - _maybe_expand_weights + math_ops.equal(rank_diff, 1), + maybe_squeeze_weights, + _maybe_expand_weights, ) # If weights are scalar, do nothing. Otherwise, try to add or remove a # dimension to match predictions. weights = control_flow_ops.cond( - math_ops.equal(weights_rank_tensor, 0), lambda: weights, - _maybe_adjust_weights + math_ops.equal(weights_rank_tensor, 0), + lambda: weights, + _maybe_adjust_weights, ) return predictions, labels, weights @@ -179,13 +185,14 @@ def _maybe_expand_labels(labels, predictions): # If sparse, expand sparse shape. if isinstance(labels, sparse_tensor.SparseTensor): return control_flow_ops.cond( - math_ops.equal(array_ops.rank(predictions), - array_ops.size(labels.dense_shape) + 1), - lambda: sparse_ops.sparse_reshape( # pylint: disable=g-long-lambda - labels, - shape=array_ops.concat((labels.dense_shape, (1, )), 0), - name=scope), - lambda: labels) + math_ops.equal(array_ops.rank(predictions), array_ops.size(labels.dense_shape) + 1), + lambda: sparse_ops.sparse_reshape( # pylint: disable=g-long-lambda + labels, + shape=array_ops.concat((labels.dense_shape, (1,)), 0), + name=scope, + ), + lambda: labels, + ) # Otherwise, try to use static shape. labels_rank = labels.get_shape().ndims @@ -197,15 +204,14 @@ def _maybe_expand_labels(labels, predictions): if predictions_rank == labels_rank + 1: return array_ops.expand_dims(labels, -1, name=scope) raise ValueError( - 'Unexpected labels shape %s for predictions shape %s.' % - (labels.get_shape(), predictions.get_shape()) + 'Unexpected labels shape %s for predictions shape %s.' % (labels.get_shape(), predictions.get_shape()) ) # Otherwise, use dynamic shape. return control_flow_ops.cond( - math_ops.equal(array_ops.rank(predictions), - array_ops.rank(labels) + 1), - lambda: array_ops.expand_dims(labels, -1, name=scope), lambda: labels + math_ops.equal(array_ops.rank(predictions), array_ops.rank(labels) + 1), + lambda: array_ops.expand_dims(labels, -1, name=scope), + lambda: labels, ) @@ -244,13 +250,11 @@ def _safe_scalar_div(numerator, denominator, name): math_ops.equal(array_ops.constant(0.0, dtype=dtypes.float64), denominator), lambda: array_ops.constant(0.0, dtype=dtypes.float64), lambda: math_ops.div(numerator, denominator), - name=name + name=name, ) -def _streaming_confusion_matrix( - labels, predictions, num_classes, weights=None -): +def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): """Calculate a streaming confusion matrix. Calculates a confusion matrix. For estimation over a stream of data, @@ -274,9 +278,7 @@ def _streaming_confusion_matrix( update_op: An operation that increments the confusion matrix. """ # Local variable to accumulate the predictions in the confusion matrix. - total_cm = metric_variable( - [num_classes, num_classes], dtypes.float64, name='total_confusion_matrix' - ) + total_cm = metric_variable([num_classes, num_classes], dtypes.float64, name='total_confusion_matrix') # Cast the type to int64 required by confusion_matrix_ops. predictions = math_ops.to_int64(predictions) @@ -333,19 +335,11 @@ def fn(distribution, *a): ops.add_to_collections(metrics_collections, metric_value) return metric_value - return distribution_strategy_context.get_tower_context().merge_call( - fn, *args - ) + return distribution_strategy_context.get_tower_context().merge_call(fn, *args) @tf_export('metrics.mean') -def mean( - values, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None -): +def mean(values, weights=None, metrics_collections=None, updates_collections=None, name=None): """Computes the (weighted) mean of the given values. The `mean` function creates two local variables, `total` and `count` @@ -384,10 +378,7 @@ def mean( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean is not supported when eager execution ' - 'is enabled.' - ) + raise RuntimeError('tf.metrics.mean is not supported when eager execution ' 'is enabled.') with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.to_float(values) @@ -398,28 +389,18 @@ def mean( if weights is None: num_values = math_ops.to_float(array_ops.size(values)) else: - values, _, weights = _remove_squeezable_dimensions( - predictions=values, labels=None, weights=weights - ) - weights = weights_broadcast_ops.broadcast_weights( - math_ops.to_float(weights), values - ) + values, _, weights = _remove_squeezable_dimensions(predictions=values, labels=None, weights=weights) + weights = weights_broadcast_ops.broadcast_weights(math_ops.to_float(weights), values) values = math_ops.multiply(values, weights) num_values = math_ops.reduce_sum(weights) - update_total_op = state_ops.assign_add( - total, math_ops.reduce_sum(values), use_locking=True - ) + update_total_op = state_ops.assign_add(total, math_ops.reduce_sum(values), use_locking=True) with ops.control_dependencies([values]): - update_count_op = state_ops.assign_add( - count, num_values, use_locking=True - ) + update_count_op = state_ops.assign_add(count, num_values, use_locking=True) compute_mean = lambda _, t, c: _safe_div(t, c, 'value') # noqa: E731 - mean_t = _aggregate_across_towers( - metrics_collections, compute_mean, total, count - ) + mean_t = _aggregate_across_towers(metrics_collections, compute_mean, total, count) update_op = _safe_div(update_total_op, update_count_op, 'update_op') if updates_collections: @@ -435,7 +416,7 @@ def accuracy( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Calculates how often `predictions` matches `labels`. @@ -481,27 +462,23 @@ def accuracy( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.accuracy is not supported when eager ' - 'execution is enabled.' - ) + raise RuntimeError('tf.metrics.accuracy is not supported when eager ' 'execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if labels.dtype != predictions.dtype: predictions = math_ops.cast(predictions, labels.dtype) is_correct = math_ops.to_float(math_ops.equal(predictions, labels)) return mean( - is_correct, weights, metrics_collections, updates_collections, name - or 'accuracy' + is_correct, + weights, + metrics_collections, + updates_collections, + name or 'accuracy', ) -def _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=None, includes=None -): +def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=None, includes=None): """Computes true_positives, false_negatives, true_negatives, false_positives. This function creates up to four local variables, `true_positives`, @@ -557,28 +534,26 @@ def _confusion_matrix_at_thresholds( check_ops.assert_greater_equal( predictions, math_ops.cast(0.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]' + message='predictions must be in [0, 1]', ), check_ops.assert_less_equal( predictions, math_ops.cast(1.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]' - ) + message='predictions must be in [0, 1]', + ), ] ): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.to_float(predictions), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights + weights=weights, ) num_thresholds = len(thresholds) # Reshape predictions and labels. predictions_2d = array_ops.reshape(predictions, [-1, 1]) - labels_2d = array_ops.reshape( - math_ops.cast(labels, dtype=dtypes.bool), [1, -1] - ) + labels_2d = array_ops.reshape(math_ops.cast(labels, dtype=dtypes.bool), [1, -1]) # Use static shape if known. num_predictions = predictions_2d.get_shape().as_list()[0] @@ -588,13 +563,13 @@ def _confusion_matrix_at_thresholds( num_predictions = array_ops.shape(predictions_2d)[0] thresh_tiled = array_ops.tile( array_ops.expand_dims(array_ops.constant(thresholds), [1]), - array_ops.stack([1, num_predictions]) + array_ops.stack([1, num_predictions]), ) # Tile the predictions after thresholding them across different thresholds. pred_is_pos = math_ops.greater( array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]), - thresh_tiled + thresh_tiled, ) if ('fn' in includes) or ('tn' in includes): pred_is_neg = math_ops.logical_not(pred_is_pos) @@ -605,15 +580,9 @@ def _confusion_matrix_at_thresholds( label_is_neg = math_ops.logical_not(label_is_pos) if weights is not None: - weights = weights_broadcast_ops.broadcast_weights( - math_ops.to_float(weights), predictions - ) - weights_tiled = array_ops.tile( - array_ops.reshape(weights, [1, -1]), [num_thresholds, 1] - ) - thresh_tiled.get_shape().assert_is_compatible_with( - weights_tiled.get_shape() - ) + weights = weights_broadcast_ops.broadcast_weights(math_ops.to_float(weights), predictions) + weights_tiled = array_ops.tile(array_ops.reshape(weights, [1, -1]), [num_thresholds, 1]) + thresh_tiled.get_shape().assert_is_compatible_with(weights_tiled.get_shape()) else: weights_tiled = None @@ -621,59 +590,35 @@ def _confusion_matrix_at_thresholds( update_ops = {} if 'tp' in includes: - true_p = metric_variable( - [num_thresholds], dtypes.float32, name='true_positives' - ) - is_true_positive = math_ops.cast( - math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32 - ) + true_p = metric_variable([num_thresholds], dtypes.float32, name='true_positives') + is_true_positive = math_ops.cast(math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32) if weights_tiled is not None: is_true_positive *= weights_tiled - update_ops['tp'] = state_ops.assign_add( - true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True - ) + update_ops['tp'] = state_ops.assign_add(true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True) values['tp'] = true_p if 'fn' in includes: - false_n = metric_variable( - [num_thresholds], dtypes.float32, name='false_negatives' - ) - is_false_negative = math_ops.cast( - math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32 - ) + false_n = metric_variable([num_thresholds], dtypes.float32, name='false_negatives') + is_false_negative = math_ops.cast(math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32) if weights_tiled is not None: is_false_negative *= weights_tiled - update_ops['fn'] = state_ops.assign_add( - false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True - ) + update_ops['fn'] = state_ops.assign_add(false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True) values['fn'] = false_n if 'tn' in includes: - true_n = metric_variable( - [num_thresholds], dtypes.float32, name='true_negatives' - ) - is_true_negative = math_ops.cast( - math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32 - ) + true_n = metric_variable([num_thresholds], dtypes.float32, name='true_negatives') + is_true_negative = math_ops.cast(math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32) if weights_tiled is not None: is_true_negative *= weights_tiled - update_ops['tn'] = state_ops.assign_add( - true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True - ) + update_ops['tn'] = state_ops.assign_add(true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True) values['tn'] = true_n if 'fp' in includes: - false_p = metric_variable( - [num_thresholds], dtypes.float32, name='false_positives' - ) - is_false_positive = math_ops.cast( - math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32 - ) + false_p = metric_variable([num_thresholds], dtypes.float32, name='false_positives') + is_false_positive = math_ops.cast(math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32) if weights_tiled is not None: is_false_positive *= weights_tiled - update_ops['fp'] = state_ops.assign_add( - false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True - ) + update_ops['fp'] = state_ops.assign_add(false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True) values['fp'] = false_p return values, update_ops @@ -694,7 +639,7 @@ def auc( updates_collections=None, curve='ROC', name=None, - summation_method='trapezoidal' + summation_method='trapezoidal', ): """Computes the approximate AUC via a Riemann sum. @@ -768,25 +713,16 @@ def auc( print('use_distribute_pai_auc') logging.info('use_distribute_pai_auc') if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.auc is not supported when eager execution ' - 'is enabled.' - ) + raise RuntimeError('tf.metrics.auc is not supported when eager execution ' 'is enabled.') - with variable_scope.variable_scope( - name, 'auc', (labels, predictions, weights) - ): + with variable_scope.variable_scope(name, 'auc', (labels, predictions, weights)): if curve != 'ROC' and curve != 'PR': raise ValueError('curve must be either ROC or PR, %s unknown' % (curve)) kepsilon = 1e-7 # to account for floating point imprecisions - thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) - ] + thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] - values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights - ) + values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) # Add epsilons to avoid dividing by 0. epsilon = 1.0e-6 @@ -822,22 +758,22 @@ def interpolate_pr_auc(tp, fp, fn): Returns: pr_auc: an approximation of the area under the P-R curve. """ - dtp = tp[:num_thresholds - 1] - tp[1:] + dtp = tp[: num_thresholds - 1] - tp[1:] p = tp + fp - prec_slope = _safe_div(dtp, p[:num_thresholds - 1] - p[1:], 'prec_slope') + prec_slope = _safe_div(dtp, p[: num_thresholds - 1] - p[1:], 'prec_slope') intercept = tp[1:] - math_ops.multiply(prec_slope, p[1:]) safe_p_ratio = array_ops.where( - math_ops.logical_and(p[:num_thresholds - 1] > 0, p[1:] > 0), - _safe_div(p[:num_thresholds - 1], p[1:], 'recall_relative_ratio'), - array_ops.ones_like(p[1:]) + math_ops.logical_and(p[: num_thresholds - 1] > 0, p[1:] > 0), + _safe_div(p[: num_thresholds - 1], p[1:], 'recall_relative_ratio'), + array_ops.ones_like(p[1:]), ) return math_ops.reduce_sum( _safe_div( prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)), tp[1:] + fn[1:], - name='pr_auc_increment' + name='pr_auc_increment', ), - name='interpolate_pr_auc' + name='interpolate_pr_auc', ) def compute_auc(tp, fn, tn, fp, name): @@ -865,42 +801,41 @@ def compute_auc(tp, fn, tn, fp, name): # above. return math_ops.reduce_sum( math_ops.multiply( - x[:num_thresholds - 1] - x[1:], - (y[:num_thresholds - 1] + y[1:]) / 2. + x[: num_thresholds - 1] - x[1:], + (y[: num_thresholds - 1] + y[1:]) / 2.0, ), - name=name + name=name, ) elif summation_method == 'minoring': return math_ops.reduce_sum( math_ops.multiply( - x[:num_thresholds - 1] - x[1:], - math_ops.minimum(y[:num_thresholds - 1], y[1:]) + x[: num_thresholds - 1] - x[1:], + math_ops.minimum(y[: num_thresholds - 1], y[1:]), ), - name=name + name=name, ) elif summation_method == 'majoring': return math_ops.reduce_sum( math_ops.multiply( - x[:num_thresholds - 1] - x[1:], - math_ops.maximum(y[:num_thresholds - 1], y[1:]) + x[: num_thresholds - 1] - x[1:], + math_ops.maximum(y[: num_thresholds - 1], y[1:]), ), - name=name + name=name, ) else: raise ValueError('Invalid summation_method: %s' % summation_method) # sum up the areas of all the trapeziums def compute_auc_value(_, values): - return compute_auc( - values['tp'], values['fn'], values['tn'], values['fp'], 'value' - ) + return compute_auc(values['tp'], values['fn'], values['tn'], values['fp'], 'value') - auc_value = _aggregate_across_towers( - metrics_collections, compute_auc_value, values - ) + auc_value = _aggregate_across_towers(metrics_collections, compute_auc_value, values) update_op = compute_auc( - update_ops['tp'], update_ops['fn'], update_ops['tn'], update_ops['fp'], - 'update_op' + update_ops['tp'], + update_ops['fn'], + update_ops['tn'], + update_ops['fp'], + 'update_op', ) if updates_collections: @@ -916,7 +851,7 @@ def mean_absolute_error( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the mean absolute error between the labels and predictions. @@ -962,18 +897,16 @@ def mean_absolute_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_absolute_error is not supported ' - 'when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_absolute_error is not supported ' 'when eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) absolute_errors = math_ops.abs(predictions - labels) return mean( - absolute_errors, weights, metrics_collections, updates_collections, name - or 'mean_absolute_error' + absolute_errors, + weights, + metrics_collections, + updates_collections, + name or 'mean_absolute_error', ) @@ -985,7 +918,7 @@ def mean_cosine_distance( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the cosine distance between the labels and predictions. @@ -1029,23 +962,18 @@ def mean_cosine_distance( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_cosine_distance is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_cosine_distance is not supported when ' 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) radial_diffs = math_ops.multiply(predictions, labels) radial_diffs = math_ops.reduce_sum( - radial_diffs, reduction_indices=[ + radial_diffs, + reduction_indices=[ dim, - ], keepdims=True - ) - mean_distance, update_op = mean( - radial_diffs, weights, None, None, name or 'mean_cosine_distance' + ], + keepdims=True, ) + mean_distance, update_op = mean(radial_diffs, weights, None, None, name or 'mean_cosine_distance') mean_distance = math_ops.subtract(1.0, mean_distance) update_op = math_ops.subtract(1.0, update_op) @@ -1066,7 +994,7 @@ def mean_per_class_accuracy( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Calculates the mean of the per-class accuracies. @@ -1109,14 +1037,9 @@ def mean_per_class_accuracy( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_per_class_accuracy is not supported ' - 'when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_per_class_accuracy is not supported ' 'when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'mean_accuracy', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'mean_accuracy', (predictions, labels, weights)): labels = math_ops.to_int64(labels) # Flatten the input if its rank > 1. @@ -1146,23 +1069,15 @@ def mean_per_class_accuracy( is_correct *= weights ones *= weights - update_total_op = state_ops.scatter_add( - total, labels, ones, use_locking=True - ) - update_count_op = state_ops.scatter_add( - count, labels, is_correct, use_locking=True - ) + update_total_op = state_ops.scatter_add(total, labels, ones, use_locking=True) + update_count_op = state_ops.scatter_add(count, labels, is_correct, use_locking=True) def compute_mean_accuracy(_, count, total): per_class_accuracy = _safe_div(count, total, None) - mean_accuracy_v = math_ops.reduce_mean( - per_class_accuracy, name='mean_accuracy' - ) + mean_accuracy_v = math_ops.reduce_mean(per_class_accuracy, name='mean_accuracy') return mean_accuracy_v - mean_accuracy_v = _aggregate_across_towers( - metrics_collections, compute_mean_accuracy, count, total - ) + mean_accuracy_v = _aggregate_across_towers(metrics_collections, compute_mean_accuracy, count, total) update_op = _safe_div(update_count_op, update_total_op, name='update_op') if updates_collections: @@ -1179,7 +1094,7 @@ def mean_iou( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Calculate per-step mean Intersection-Over-Union (mIOU). @@ -1226,20 +1141,13 @@ def mean_iou( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_iou is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_iou is not supported when ' 'eager execution is enabled.') - with variable_scope.variable_scope( - name, 'mean_iou', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'mean_iou', (predictions, labels, weights)): # Check if shape is compatible. predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - total_cm, update_op = _streaming_confusion_matrix( - labels, predictions, num_classes, weights - ) + total_cm, update_op = _streaming_confusion_matrix(labels, predictions, num_classes, weights) def compute_mean_iou(_, total_cm): """Compute the mean intersection-over-union via the confusion matrix.""" @@ -1251,31 +1159,27 @@ def compute_mean_iou(_, total_cm): # The mean is only computed over classes that appear in the # label or prediction tensor. If the denominator is 0, we need to # ignore the class. - num_valid_entries = math_ops.reduce_sum( - math_ops.cast( - math_ops.not_equal(denominator, 0), dtype=dtypes.float32 - ) - ) + num_valid_entries = math_ops.reduce_sum(math_ops.cast(math_ops.not_equal(denominator, 0), dtype=dtypes.float32)) # If the value of the denominator is 0, set it to 1 to avoid # zero division. denominator = array_ops.where( - math_ops.greater(denominator, 0), denominator, - array_ops.ones_like(denominator) + math_ops.greater(denominator, 0), + denominator, + array_ops.ones_like(denominator), ) iou = math_ops.div(cm_diag, denominator) # If the number of valid entries is 0 (no classes) we return 0. result = array_ops.where( math_ops.greater(num_valid_entries, 0), - math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, 0 + math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, + 0, ) return result # TODO(priyag): Use outside_compilation if in TPU context. - mean_iou_v = _aggregate_across_towers( - metrics_collections, compute_mean_iou, total_cm - ) + mean_iou_v = _aggregate_across_towers(metrics_collections, compute_mean_iou, total_cm) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1291,7 +1195,7 @@ def mean_relative_error( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the mean relative error by normalizing with the given values. @@ -1338,26 +1242,23 @@ def mean_relative_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_relative_error is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_relative_error is not supported when ' 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) - predictions, normalizer = confusion_matrix.remove_squeezable_dimensions( - predictions, normalizer - ) + predictions, normalizer = confusion_matrix.remove_squeezable_dimensions(predictions, normalizer) predictions.get_shape().assert_is_compatible_with(normalizer.get_shape()) relative_errors = array_ops.where( - math_ops.equal(normalizer, 0.0), array_ops.zeros_like(labels), - math_ops.div(math_ops.abs(labels - predictions), normalizer) + math_ops.equal(normalizer, 0.0), + array_ops.zeros_like(labels), + math_ops.div(math_ops.abs(labels - predictions), normalizer), ) return mean( - relative_errors, weights, metrics_collections, updates_collections, name - or 'mean_relative_error' + relative_errors, + weights, + metrics_collections, + updates_collections, + name or 'mean_relative_error', ) @@ -1368,7 +1269,7 @@ def mean_squared_error( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the mean squared error between the labels and predictions. @@ -1414,29 +1315,21 @@ def mean_squared_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_squared_error is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_squared_error is not supported when ' 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) squared_error = math_ops.square(labels - predictions) return mean( - squared_error, weights, metrics_collections, updates_collections, name - or 'mean_squared_error' + squared_error, + weights, + metrics_collections, + updates_collections, + name or 'mean_squared_error', ) @tf_export('metrics.mean_tensor') -def mean_tensor( - values, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None -): +def mean_tensor(values, weights=None, metrics_collections=None, updates_collections=None, name=None): """Computes the element-wise (weighted) mean of the given tensors. In contrast to the `mean` function which returns a scalar with the @@ -1479,42 +1372,27 @@ def mean_tensor( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_tensor is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_tensor is not supported when ' 'eager execution is enabled.') with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.to_float(values) - total = metric_variable( - values.get_shape(), dtypes.float32, name='total_tensor' - ) - count = metric_variable( - values.get_shape(), dtypes.float32, name='count_tensor' - ) + total = metric_variable(values.get_shape(), dtypes.float32, name='total_tensor') + count = metric_variable(values.get_shape(), dtypes.float32, name='count_tensor') num_values = array_ops.ones_like(values) if weights is not None: - values, _, weights = _remove_squeezable_dimensions( - predictions=values, labels=None, weights=weights - ) - weights = weights_broadcast_ops.broadcast_weights( - math_ops.to_float(weights), values - ) + values, _, weights = _remove_squeezable_dimensions(predictions=values, labels=None, weights=weights) + weights = weights_broadcast_ops.broadcast_weights(math_ops.to_float(weights), values) values = math_ops.multiply(values, weights) num_values = math_ops.multiply(num_values, weights) update_total_op = state_ops.assign_add(total, values, use_locking=True) with ops.control_dependencies([values]): - update_count_op = state_ops.assign_add( - count, num_values, use_locking=True - ) + update_count_op = state_ops.assign_add(count, num_values, use_locking=True) compute_mean = lambda _, t, c: _safe_div(t, c, 'value') # noqa: E731 - mean_t = _aggregate_across_towers( - metrics_collections, compute_mean, total, count - ) + mean_t = _aggregate_across_towers(metrics_collections, compute_mean, total, count) update_op = _safe_div(update_total_op, update_count_op, 'update_op') if updates_collections: @@ -1530,7 +1408,7 @@ def percentage_below( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the percentage of values less than the given threshold. @@ -1571,21 +1449,19 @@ def percentage_below( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.percentage_below is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.percentage_below is not supported when ' 'eager execution is enabled.') is_below_threshold = math_ops.to_float(math_ops.less(values, threshold)) return mean( - is_below_threshold, weights, metrics_collections, updates_collections, name - or 'percentage_below_threshold' + is_below_threshold, + weights, + metrics_collections, + updates_collections, + name or 'percentage_below_threshold', ) -def _count_condition( - values, weights=None, metrics_collections=None, updates_collections=None -): +def _count_condition(values, weights=None, metrics_collections=None, updates_collections=None): """Sums the weights of cases where the given values are True. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1614,17 +1490,13 @@ def _count_condition( values = math_ops.to_float(values) if weights is not None: - with ops.control_dependencies( - (check_ops.assert_rank_in(weights, (0, array_ops.rank(values))), ) - ): + with ops.control_dependencies((check_ops.assert_rank_in(weights, (0, array_ops.rank(values))),)): weights = math_ops.to_float(weights) values = math_ops.multiply(values, weights) value_tensor = _aggregate_variable(count, metrics_collections) - update_op = state_ops.assign_add( - count, math_ops.reduce_sum(values), use_locking=True - ) + update_op = state_ops.assign_add(count, math_ops.reduce_sum(values), use_locking=True) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1638,7 +1510,7 @@ def false_negatives( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the total number of false negatives. @@ -1669,26 +1541,16 @@ def false_negatives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.false_negatives is not supported when ' - 'eager execution is enabled.' - ) - - with variable_scope.variable_scope( - name, 'false_negatives', (predictions, labels, weights) - ): + raise RuntimeError('tf.metrics.false_negatives is not supported when ' 'eager execution is enabled.') + with variable_scope.variable_scope(name, 'false_negatives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights - ) - is_false_negative = math_ops.logical_and( - math_ops.equal(labels, True), math_ops.equal(predictions, False) - ) - return _count_condition( - is_false_negative, weights, metrics_collections, updates_collections + weights=weights, ) + is_false_negative = math_ops.logical_and(math_ops.equal(labels, True), math_ops.equal(predictions, False)) + return _count_condition(is_false_negative, weights, metrics_collections, updates_collections) @tf_export('metrics.false_negatives_at_thresholds') @@ -1699,7 +1561,7 @@ def false_negatives_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes false negatives at provided threshold values. @@ -1733,16 +1595,11 @@ def false_negatives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.false_negatives_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.false_negatives_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'false_negatives', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'false_negatives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fn', ) + labels, predictions, thresholds, weights=weights, includes=('fn',) ) fn_value = _aggregate_variable(values['fn'], metrics_collections) @@ -1760,7 +1617,7 @@ def false_positives( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Sum the weights of false positives. @@ -1792,26 +1649,16 @@ def false_positives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.false_positives is not supported when ' - 'eager execution is enabled.' - ) - - with variable_scope.variable_scope( - name, 'false_positives', (predictions, labels, weights) - ): + raise RuntimeError('tf.metrics.false_positives is not supported when ' 'eager execution is enabled.') + with variable_scope.variable_scope(name, 'false_positives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights - ) - is_false_positive = math_ops.logical_and( - math_ops.equal(labels, False), math_ops.equal(predictions, True) - ) - return _count_condition( - is_false_positive, weights, metrics_collections, updates_collections + weights=weights, ) + is_false_positive = math_ops.logical_and(math_ops.equal(labels, False), math_ops.equal(predictions, True)) + return _count_condition(is_false_positive, weights, metrics_collections, updates_collections) @tf_export('metrics.false_positives_at_thresholds') @@ -1822,7 +1669,7 @@ def false_positives_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes false positives at provided threshold values. @@ -1856,16 +1703,11 @@ def false_positives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.false_positives_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.false_positives_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'false_positives', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'false_positives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fp', ) + labels, predictions, thresholds, weights=weights, includes=('fp',) ) fp_value = _aggregate_variable(values['fp'], metrics_collections) @@ -1883,7 +1725,7 @@ def true_negatives( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Sum the weights of true_negatives. @@ -1915,26 +1757,16 @@ def true_negatives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.true_negatives is not ' - 'supported when eager execution is enabled.' - ) - - with variable_scope.variable_scope( - name, 'true_negatives', (predictions, labels, weights) - ): + raise RuntimeError('tf.metrics.true_negatives is not ' 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights - ) - is_true_negative = math_ops.logical_and( - math_ops.equal(labels, False), math_ops.equal(predictions, False) - ) - return _count_condition( - is_true_negative, weights, metrics_collections, updates_collections + weights=weights, ) + is_true_negative = math_ops.logical_and(math_ops.equal(labels, False), math_ops.equal(predictions, False)) + return _count_condition(is_true_negative, weights, metrics_collections, updates_collections) @tf_export('metrics.true_negatives_at_thresholds') @@ -1945,7 +1777,7 @@ def true_negatives_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes true negatives at provided threshold values. @@ -1979,16 +1811,11 @@ def true_negatives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.true_negatives_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.true_negatives_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'true_negatives', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tn', ) + labels, predictions, thresholds, weights=weights, includes=('tn',) ) tn_value = _aggregate_variable(values['tn'], metrics_collections) @@ -2006,7 +1833,7 @@ def true_positives( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Sum the weights of true_positives. @@ -2038,26 +1865,16 @@ def true_positives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.true_positives is not ' - 'supported when eager execution is enabled.' - ) - - with variable_scope.variable_scope( - name, 'true_positives', (predictions, labels, weights) - ): + raise RuntimeError('tf.metrics.true_positives is not ' 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'true_positives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights - ) - is_true_positive = math_ops.logical_and( - math_ops.equal(labels, True), math_ops.equal(predictions, True) - ) - return _count_condition( - is_true_positive, weights, metrics_collections, updates_collections + weights=weights, ) + is_true_positive = math_ops.logical_and(math_ops.equal(labels, True), math_ops.equal(predictions, True)) + return _count_condition(is_true_positive, weights, metrics_collections, updates_collections) @tf_export('metrics.true_positives_at_thresholds') @@ -2068,7 +1885,7 @@ def true_positives_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes true positives at provided threshold values. @@ -2102,16 +1919,11 @@ def true_positives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.true_positives_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.true_positives_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'true_positives', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'true_positives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tp', ) + labels, predictions, thresholds, weights=weights, includes=('tp',) ) tp_value = _aggregate_variable(values['tp'], metrics_collections) @@ -2129,7 +1941,7 @@ def precision( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the precision of the predictions with respect to the labels. @@ -2175,19 +1987,13 @@ def precision( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.precision is not ' - 'supported when eager execution is enabled.' - ) - - with variable_scope.variable_scope( - name, 'precision', (predictions, labels, weights) - ): + raise RuntimeError('tf.metrics.precision is not ' 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'precision', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights + weights=weights, ) true_p, true_positives_update_op = true_positives( @@ -2196,7 +2002,7 @@ def precision( weights, metrics_collections=None, updates_collections=None, - name=None + name=None, ) false_p, false_positives_update_op = false_positives( labels, @@ -2204,24 +2010,18 @@ def precision( weights, metrics_collections=None, updates_collections=None, - name=None + name=None, ) def compute_precision(tp, fp, name): - return array_ops.where( - math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name - ) + return array_ops.where(math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name) def once_across_towers(_, true_p, false_p): return compute_precision(true_p, false_p, 'value') - p = _aggregate_across_towers( - metrics_collections, once_across_towers, true_p, false_p - ) + p = _aggregate_across_towers(metrics_collections, once_across_towers, true_p, false_p) - update_op = compute_precision( - true_positives_update_op, false_positives_update_op, 'update_op' - ) + update_op = compute_precision(true_positives_update_op, false_positives_update_op, 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2236,7 +2036,7 @@ def precision_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes precision values for different `thresholds` on `predictions`. @@ -2283,14 +2083,9 @@ def precision_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.precision_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.precision_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'precision_at_thresholds', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'precision_at_thresholds', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( labels, predictions, thresholds, weights, includes=('tp', 'fp') ) @@ -2304,13 +2099,9 @@ def compute_precision(tp, fp, name): def precision_across_towers(_, values): return compute_precision(values['tp'], values['fp'], 'value') - prec = _aggregate_across_towers( - metrics_collections, precision_across_towers, values - ) + prec = _aggregate_across_towers(metrics_collections, precision_across_towers, values) - update_op = compute_precision( - update_ops['tp'], update_ops['fp'], 'update_op' - ) + update_op = compute_precision(update_ops['tp'], update_ops['fp'], 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2324,7 +2115,7 @@ def recall( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the recall of the predictions with respect to the labels. @@ -2368,18 +2159,13 @@ def recall( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.recall is not supported is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.recall is not supported is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'recall', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'recall', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights + weights=weights, ) true_p, true_positives_update_op = true_positives( @@ -2388,7 +2174,7 @@ def recall( weights, metrics_collections=None, updates_collections=None, - name=None + name=None, ) false_n, false_negatives_update_op = false_negatives( labels, @@ -2396,25 +2182,23 @@ def recall( weights, metrics_collections=None, updates_collections=None, - name=None + name=None, ) def compute_recall(true_p, false_n, name): return array_ops.where( math_ops.greater(true_p + false_n, 0), - math_ops.div(true_p, true_p + false_n), 0, name + math_ops.div(true_p, true_p + false_n), + 0, + name, ) def once_across_towers(_, true_p, false_n): return compute_recall(true_p, false_n, 'value') - rec = _aggregate_across_towers( - metrics_collections, once_across_towers, true_p, false_n - ) + rec = _aggregate_across_towers(metrics_collections, once_across_towers, true_p, false_n) - update_op = compute_recall( - true_positives_update_op, false_negatives_update_op, 'update_op' - ) + update_op = compute_recall(true_positives_update_op, false_negatives_update_op, 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2444,9 +2228,7 @@ def _select_class_id(ids, selected_id): """ ids = sparse_tensor.convert_to_tensor_or_sparse_tensor(ids) if isinstance(ids, sparse_tensor.SparseTensor): - return sparse_ops.sparse_retain( - ids, math_ops.equal(ids.values, selected_id) - ) + return sparse_ops.sparse_retain(ids, math_ops.equal(ids.values, selected_id)) # TODO(ptucker): Make this more efficient, maybe add a sparse version of # tf.equal and tf.reduce_any? @@ -2454,18 +2236,12 @@ def _select_class_id(ids, selected_id): # Shape of filled IDs is the same as `ids` with the last dim collapsed to 1. ids_shape = array_ops.shape(ids, out_type=dtypes.int64) ids_last_dim = array_ops.size(ids_shape) - 1 - filled_selected_id_shape = math_ops.reduced_shape( - ids_shape, array_ops.reshape(ids_last_dim, [1]) - ) + filled_selected_id_shape = math_ops.reduced_shape(ids_shape, array_ops.reshape(ids_last_dim, [1])) # Intersect `ids` with the selected ID. - filled_selected_id = array_ops.fill( - filled_selected_id_shape, math_ops.to_int64(selected_id) - ) + filled_selected_id = array_ops.fill(filled_selected_id_shape, math_ops.to_int64(selected_id)) result = sets.set_intersection(filled_selected_id, ids) - return sparse_tensor.SparseTensor( - indices=result.indices, values=result.values, dense_shape=ids_shape - ) + return sparse_tensor.SparseTensor(indices=result.indices, values=result.values, dense_shape=ids_shape) def _maybe_select_class_id(labels, predictions_idx, selected_id=None): @@ -2489,13 +2265,11 @@ def _maybe_select_class_id(labels, predictions_idx, selected_id=None): return labels, predictions_idx return ( _select_class_id(labels, selected_id), - _select_class_id(predictions_idx, selected_id) + _select_class_id(predictions_idx, selected_id), ) -def _sparse_true_positive_at_k( - labels, predictions_idx, class_id=None, weights=None, name=None -): +def _sparse_true_positive_at_k(labels, predictions_idx, class_id=None, weights=None, name=None): """Calculates true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2522,26 +2296,18 @@ def _sparse_true_positive_at_k( Returns: A [D1, ... DN] `Tensor` of true positive counts. """ - with ops.name_scope( - name, 'true_positives', (predictions_idx, labels, weights) - ): - labels, predictions_idx = _maybe_select_class_id( - labels, predictions_idx, class_id - ) + with ops.name_scope(name, 'true_positives', (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) tp = sets.set_size(sets.set_intersection(predictions_idx, labels)) tp = math_ops.to_double(tp) if weights is not None: - with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, tp), ) - ): + with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, tp),)): weights = math_ops.to_double(weights) tp = math_ops.multiply(tp, weights) return tp -def _streaming_sparse_true_positive_at_k( - labels, predictions_idx, k=None, class_id=None, weights=None, name=None -): +def _streaming_sparse_true_positive_at_k(labels, predictions_idx, k=None, class_id=None, weights=None, name=None): """Calculates weighted per step true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2575,26 +2341,23 @@ def _streaming_sparse_true_positive_at_k( ValueError: If `weights` is not `None` and has an incompatible shape. """ with ops.name_scope( - name, _at_k_name('true_positive', k, class_id=class_id), - (predictions_idx, labels, weights) + name, + _at_k_name('true_positive', k, class_id=class_id), + (predictions_idx, labels, weights), ) as scope: tp = _sparse_true_positive_at_k( predictions_idx=predictions_idx, labels=labels, class_id=class_id, - weights=weights + weights=weights, ) batch_total_tp = math_ops.to_double(math_ops.reduce_sum(tp)) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add( - var, batch_total_tp, name='update', use_locking=True - ) + return var, state_ops.assign_add(var, batch_total_tp, name='update', use_locking=True) -def _sparse_false_negative_at_k( - labels, predictions_idx, class_id=None, weights=None -): +def _sparse_false_negative_at_k(labels, predictions_idx, class_id=None, weights=None): """Calculates false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2620,28 +2383,18 @@ def _sparse_false_negative_at_k( Returns: A [D1, ... DN] `Tensor` of false negative counts. """ - with ops.name_scope( - None, 'false_negatives', (predictions_idx, labels, weights) - ): - labels, predictions_idx = _maybe_select_class_id( - labels, predictions_idx, class_id - ) - fn = sets.set_size( - sets.set_difference(predictions_idx, labels, aminusb=False) - ) + with ops.name_scope(None, 'false_negatives', (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) + fn = sets.set_size(sets.set_difference(predictions_idx, labels, aminusb=False)) fn = math_ops.to_double(fn) if weights is not None: - with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, fn), ) - ): + with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, fn),)): weights = math_ops.to_double(weights) fn = math_ops.multiply(fn, weights) return fn -def _streaming_sparse_false_negative_at_k( - labels, predictions_idx, k, class_id=None, weights=None, name=None -): +def _streaming_sparse_false_negative_at_k(labels, predictions_idx, k, class_id=None, weights=None, name=None): """Calculates weighted per step false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2675,21 +2428,20 @@ def _streaming_sparse_false_negative_at_k( ValueError: If `weights` is not `None` and has an incompatible shape. """ with ops.name_scope( - name, _at_k_name('false_negative', k, class_id=class_id), - (predictions_idx, labels, weights) + name, + _at_k_name('false_negative', k, class_id=class_id), + (predictions_idx, labels, weights), ) as scope: fn = _sparse_false_negative_at_k( predictions_idx=predictions_idx, labels=labels, class_id=class_id, - weights=weights + weights=weights, ) batch_total_fn = math_ops.to_double(math_ops.reduce_sum(fn)) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add( - var, batch_total_fn, name='update', use_locking=True - ) + return var, state_ops.assign_add(var, batch_total_fn, name='update', use_locking=True) @tf_export('metrics.recall_at_k') @@ -2701,7 +2453,7 @@ def recall_at_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes recall@k of the predictions with respect to sparse labels. @@ -2770,15 +2522,9 @@ def recall_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.recall_at_k is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.recall_at_k is not ' 'supported when eager execution is enabled.') - with ops.name_scope( - name, _at_k_name('recall', k, class_id=class_id), - (predictions, labels, weights) - ) as scope: + with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), (predictions, labels, weights)) as scope: _, top_k_idx = nn.top_k(predictions, k) return recall_at_top_k( labels=labels, @@ -2788,7 +2534,7 @@ def recall_at_k( weights=weights, metrics_collections=metrics_collections, updates_collections=updates_collections, - name=scope + name=scope, ) @@ -2801,7 +2547,7 @@ def recall_at_top_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes recall@k of top-k predictions with respect to sparse labels. @@ -2849,8 +2595,9 @@ class indices, whereas `recall_at_k` expects logits. Refer to `recall_at_k` are not a list or tuple. """ with ops.name_scope( - name, _at_k_name('recall', k, class_id=class_id), - (predictions_idx, labels, weights) + name, + _at_k_name('recall', k, class_id=class_id), + (predictions_idx, labels, weights), ) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.to_int64(predictions_idx) @@ -2859,26 +2606,22 @@ class indices, whereas `recall_at_k` expects logits. Refer to `recall_at_k` labels=labels, k=k, class_id=class_id, - weights=weights + weights=weights, ) fn, fn_update = _streaming_sparse_false_negative_at_k( predictions_idx=top_k_idx, labels=labels, k=k, class_id=class_id, - weights=weights + weights=weights, ) def compute_recall(_, tp, fn): return math_ops.div(tp, math_ops.add(tp, fn), name=scope) - metric = _aggregate_across_towers( - metrics_collections, compute_recall, tp, fn - ) + metric = _aggregate_across_towers(metrics_collections, compute_recall, tp, fn) - update = math_ops.div( - tp_update, math_ops.add(tp_update, fn_update), name='update' - ) + update = math_ops.div(tp_update, math_ops.add(tp_update, fn_update), name='update') if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @@ -2892,7 +2635,7 @@ def recall_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes various recall values for different `thresholds` on `predictions`. @@ -2937,14 +2680,9 @@ def recall_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.recall_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.recall_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'recall_at_thresholds', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'recall_at_thresholds', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( labels, predictions, thresholds, weights, includes=('tp', 'fn') ) @@ -2958,9 +2696,7 @@ def compute_recall(tp, fn, name): def recall_across_towers(_, values): return compute_recall(values['tp'], values['fn'], 'value') - rec = _aggregate_across_towers( - metrics_collections, recall_across_towers, values - ) + rec = _aggregate_across_towers(metrics_collections, recall_across_towers, values) update_op = compute_recall(update_ops['tp'], update_ops['fn'], 'update_op') if updates_collections: @@ -2976,7 +2712,7 @@ def root_mean_squared_error( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the root mean squared error between the labels and predictions. @@ -3022,17 +2758,10 @@ def root_mean_squared_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.root_mean_squared_error is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.root_mean_squared_error is not ' 'supported when eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) - mse, update_mse_op = mean_squared_error( - labels, predictions, weights, None, None, name or 'root_mean_squared_error' - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + mse, update_mse_op = mean_squared_error(labels, predictions, weights, None, None, name or 'root_mean_squared_error') once_across_towers = lambda _, mse: math_ops.sqrt(mse) # noqa: E731 rmse = _aggregate_across_towers(metrics_collections, once_across_towers, mse) @@ -3053,7 +2782,7 @@ def sensitivity_at_specificity( num_thresholds=200, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the specificity at a given sensitivity. @@ -3106,26 +2835,17 @@ def sensitivity_at_specificity( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.sensitivity_at_specificity is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.sensitivity_at_specificity is not ' 'supported when eager execution is enabled.') if specificity < 0 or specificity > 1: raise ValueError('`specificity` must be in the range [0, 1].') - with variable_scope.variable_scope( - name, 'sensitivity_at_specificity', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'sensitivity_at_specificity', (predictions, labels, weights)): kepsilon = 1e-7 # to account for floating point imprecisions - thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) - ] + thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] - values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights - ) + values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): specificities = math_ops.div(tn, tn + fp + kepsilon) @@ -3133,22 +2853,19 @@ def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the sensitivity: - return math_ops.div( - tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, name - ) + return math_ops.div(tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, name) def sensitivity_across_towers(_, values): - return compute_sensitivity_at_specificity( - values['tp'], values['tn'], values['fp'], values['fn'], 'value' - ) + return compute_sensitivity_at_specificity(values['tp'], values['tn'], values['fp'], values['fn'], 'value') - sensitivity = _aggregate_across_towers( - metrics_collections, sensitivity_across_towers, values - ) + sensitivity = _aggregate_across_towers(metrics_collections, sensitivity_across_towers, values) update_op = compute_sensitivity_at_specificity( - update_ops['tp'], update_ops['tn'], update_ops['fp'], update_ops['fn'], - 'update_op' + update_ops['tp'], + update_ops['tn'], + update_ops['fp'], + update_ops['fn'], + 'update_op', ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -3177,45 +2894,34 @@ def _expand_and_tile(tensor, multiple, dim=0, name=None): """ if multiple < 1: raise ValueError('Invalid multiple %s, must be > 0.' % multiple) - with ops.name_scope( - name, 'expand_and_tile', (tensor, multiple, dim) - ) as scope: + with ops.name_scope(name, 'expand_and_tile', (tensor, multiple, dim)) as scope: # Sparse. tensor = sparse_tensor.convert_to_tensor_or_sparse_tensor(tensor) if isinstance(tensor, sparse_tensor.SparseTensor): if dim < 0: - expand_dims = array_ops.reshape( - array_ops.size(tensor.dense_shape) + dim, [1] - ) + expand_dims = array_ops.reshape(array_ops.size(tensor.dense_shape) + dim, [1]) else: expand_dims = [dim] expanded_shape = array_ops.concat( ( - array_ops.slice(tensor.dense_shape, [0], expand_dims), [1], - array_ops.slice(tensor.dense_shape, expand_dims, [-1]) + array_ops.slice(tensor.dense_shape, [0], expand_dims), + [1], + array_ops.slice(tensor.dense_shape, expand_dims, [-1]), ), 0, - name='expanded_shape' - ) - expanded = sparse_ops.sparse_reshape( - tensor, shape=expanded_shape, name='expand' + name='expanded_shape', ) + expanded = sparse_ops.sparse_reshape(tensor, shape=expanded_shape, name='expand') if multiple == 1: return expanded - return sparse_ops.sparse_concat( - dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope - ) + return sparse_ops.sparse_concat(dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope) # Dense. - expanded = array_ops.expand_dims( - tensor, dim if (dim >= 0) else (dim - 1), name='expand' - ) + expanded = array_ops.expand_dims(tensor, dim if (dim >= 0) else (dim - 1), name='expand') if multiple == 1: return expanded ones = array_ops.ones_like(array_ops.shape(tensor)) - tile_multiples = array_ops.concat( - (ones[:dim], (multiple, ), ones[dim:]), 0, name='multiples' - ) + tile_multiples = array_ops.concat((ones[:dim], (multiple,), ones[dim:]), 0, name='multiples') return array_ops.tile(expanded, tile_multiples, name=scope) @@ -3241,7 +2947,7 @@ def _num_relevant(labels, k): """ if k < 1: raise ValueError('Invalid k=%s.' % k) - with ops.name_scope(None, 'num_relevant', (labels, )) as scope: + with ops.name_scope(None, 'num_relevant', (labels,)) as scope: # For SparseTensor, calculate separate count for each row. labels = sparse_tensor.convert_to_tensor_or_sparse_tensor(labels) if isinstance(labels, sparse_tensor.SparseTensor): @@ -3287,12 +2993,8 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): Raises: ValueError: if the last dimension of predictions_idx is not set. """ - with ops.name_scope( - None, 'average_precision', (predictions_idx, labels) - ) as scope: - predictions_idx = math_ops.to_int64( - predictions_idx, name='predictions_idx' - ) + with ops.name_scope(None, 'average_precision', (predictions_idx, labels)) as scope: + predictions_idx = math_ops.to_int64(predictions_idx, name='predictions_idx') if predictions_idx.get_shape().ndims == 0: raise ValueError('The rank of predictions_idx must be at least 1.') k = predictions_idx.get_shape().as_list()[-1] @@ -3303,14 +3005,10 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # Expand dims to produce [D1, ... DN, k, 1] tensor. This gives us a separate # prediction for each k, so we can calculate separate true positive values # for each k. - predictions_idx_per_k = array_ops.expand_dims( - predictions_idx, -1, name='predictions_idx_per_k' - ) + predictions_idx_per_k = array_ops.expand_dims(predictions_idx, -1, name='predictions_idx_per_k') # Replicate labels k times to produce [D1, ... DN, k, num_labels] tensor. - labels_per_k = _expand_and_tile( - labels, multiple=k, dim=-1, name='labels_per_k' - ) + labels_per_k = _expand_and_tile(labels, multiple=k, dim=-1, name='labels_per_k') # The following tensors are all of shape [D1, ... DN, k], containing values # per row, per k value. @@ -3324,28 +3022,22 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # term from the formula above. # `relevant_precision_per_k` (float64) - Relevant precisions; i.e., # precisions at all k for which relevance indicator is true. - relevant_per_k = _sparse_true_positive_at_k( - labels_per_k, predictions_idx_per_k, name='relevant_per_k' - ) + relevant_per_k = _sparse_true_positive_at_k(labels_per_k, predictions_idx_per_k, name='relevant_per_k') tp_per_k = math_ops.cumsum(relevant_per_k, axis=-1, name='tp_per_k') - retrieved_per_k = math_ops.cumsum( - array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k' - ) + retrieved_per_k = math_ops.cumsum(array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k') precision_per_k = math_ops.div( math_ops.to_double(tp_per_k), math_ops.to_double(retrieved_per_k), - name='precision_per_k' + name='precision_per_k', ) relevant_precision_per_k = math_ops.multiply( precision_per_k, math_ops.to_double(relevant_per_k), - name='relevant_precision_per_k' + name='relevant_precision_per_k', ) # Reduce along k dimension to get the sum, yielding a [D1, ... DN] tensor. - precision_sum = math_ops.reduce_sum( - relevant_precision_per_k, reduction_indices=(-1, ), name='precision_sum' - ) + precision_sum = math_ops.reduce_sum(relevant_precision_per_k, reduction_indices=(-1,), name='precision_sum') # Divide by number of relevant items to get average precision. These are # the "num_relevant_items" and "AveP" terms from the formula above. @@ -3359,7 +3051,7 @@ def _streaming_sparse_average_precision_at_top_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes average precision@k of predictions with respect to sparse labels. @@ -3405,50 +3097,36 @@ def _streaming_sparse_average_precision_at_top_k( update: `Operation` that increments variables appropriately, and whose value matches `metric`. """ - with ops.name_scope( - name, 'average_precision_at_top_k', (predictions_idx, labels, weights) - ) as scope: + with ops.name_scope(name, 'average_precision_at_top_k', (predictions_idx, labels, weights)) as scope: # Calculate per-example average precision, and apply weights. - average_precision = _sparse_average_precision_at_top_k( - predictions_idx=predictions_idx, labels=labels - ) + average_precision = _sparse_average_precision_at_top_k(predictions_idx=predictions_idx, labels=labels) if weights is not None: - weights = weights_broadcast_ops.broadcast_weights( - math_ops.to_double(weights), average_precision - ) + weights = weights_broadcast_ops.broadcast_weights(math_ops.to_double(weights), average_precision) average_precision = math_ops.multiply(average_precision, weights) # Create accumulation variables and update ops for max average precision and # total average precision. - with ops.name_scope(None, 'max', (average_precision, )) as max_scope: + with ops.name_scope(None, 'max', (average_precision,)) as max_scope: # `max` is the max possible precision. Since max for any row is 1.0: # - For the unweighted case, this is just the number of rows. # - For the weighted case, it's the sum of the weights broadcast across # `average_precision` rows. max_var = metric_variable([], dtypes.float64, name=max_scope) if weights is None: - batch_max = math_ops.to_double( - array_ops.size(average_precision, name='batch_max') - ) + batch_max = math_ops.to_double(array_ops.size(average_precision, name='batch_max')) else: batch_max = math_ops.reduce_sum(weights, name='batch_max') - max_update = state_ops.assign_add( - max_var, batch_max, name='update', use_locking=True - ) - with ops.name_scope(None, 'total', (average_precision, )) as total_scope: + max_update = state_ops.assign_add(max_var, batch_max, name='update', use_locking=True) + with ops.name_scope(None, 'total', (average_precision,)) as total_scope: total_var = metric_variable([], dtypes.float64, name=total_scope) batch_total = math_ops.reduce_sum(average_precision, name='batch_total') - total_update = state_ops.assign_add( - total_var, batch_total, name='update', use_locking=True - ) + total_update = state_ops.assign_add(total_var, batch_total, name='update', use_locking=True) # Divide total by max to get mean, for both vars and the update ops. def precision_across_towers(_, total_var, max_var): return _safe_scalar_div(total_var, max_var, name='mean') - mean_average_precision = _aggregate_across_towers( - metrics_collections, precision_across_towers, total_var, max_var - ) + mean_average_precision = _aggregate_across_towers(metrics_collections, precision_across_towers, total_var, max_var) update = _safe_scalar_div(total_update, max_update, name=scope) if updates_collections: @@ -3466,7 +3144,7 @@ def sparse_average_precision_at_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Renamed to `average_precision_at_k`, please use that method instead.""" return average_precision_at_k( @@ -3476,7 +3154,7 @@ def sparse_average_precision_at_k( weights=weights, metrics_collections=metrics_collections, updates_collections=updates_collections, - name=name + name=name, ) @@ -3488,7 +3166,7 @@ def average_precision_at_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes average precision@k of predictions with respect to sparse labels. @@ -3543,16 +3221,11 @@ def average_precision_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.sparse_average_precision_at_k is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.sparse_average_precision_at_k is not ' 'supported when eager execution is enabled.') if k < 1: raise ValueError('Invalid k=%s.' % k) - with ops.name_scope( - name, _at_k_name('average_precision', k), (predictions, labels, weights) - ) as scope: + with ops.name_scope(name, _at_k_name('average_precision', k), (predictions, labels, weights)) as scope: # Calculate top k indices to produce [D1, ... DN, k] tensor. _, predictions_idx = nn.top_k(predictions, k) return _streaming_sparse_average_precision_at_top_k( @@ -3561,13 +3234,11 @@ def average_precision_at_k( weights=weights, metrics_collections=metrics_collections, updates_collections=updates_collections, - name=scope + name=scope, ) -def _sparse_false_positive_at_k( - labels, predictions_idx, class_id=None, weights=None -): +def _sparse_false_positive_at_k(labels, predictions_idx, class_id=None, weights=None): """Calculates false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3593,28 +3264,18 @@ def _sparse_false_positive_at_k( Returns: A [D1, ... DN] `Tensor` of false positive counts. """ - with ops.name_scope( - None, 'false_positives', (predictions_idx, labels, weights) - ): - labels, predictions_idx = _maybe_select_class_id( - labels, predictions_idx, class_id - ) - fp = sets.set_size( - sets.set_difference(predictions_idx, labels, aminusb=True) - ) + with ops.name_scope(None, 'false_positives', (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) + fp = sets.set_size(sets.set_difference(predictions_idx, labels, aminusb=True)) fp = math_ops.to_double(fp) if weights is not None: - with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, fp), ) - ): + with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, fp),)): weights = math_ops.to_double(weights) fp = math_ops.multiply(fp, weights) return fp -def _streaming_sparse_false_positive_at_k( - labels, predictions_idx, k=None, class_id=None, weights=None, name=None -): +def _streaming_sparse_false_positive_at_k(labels, predictions_idx, k=None, class_id=None, weights=None, name=None): """Calculates weighted per step false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3648,21 +3309,20 @@ def _streaming_sparse_false_positive_at_k( ValueError: If `weights` is not `None` and has an incompatible shape. """ with ops.name_scope( - name, _at_k_name('false_positive', k, class_id=class_id), - (predictions_idx, labels, weights) + name, + _at_k_name('false_positive', k, class_id=class_id), + (predictions_idx, labels, weights), ) as scope: fp = _sparse_false_positive_at_k( predictions_idx=predictions_idx, labels=labels, class_id=class_id, - weights=weights + weights=weights, ) batch_total_fp = math_ops.to_double(math_ops.reduce_sum(fp)) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add( - var, batch_total_fp, name='update', use_locking=True - ) + return var, state_ops.assign_add(var, batch_total_fp, name='update', use_locking=True) @tf_export('metrics.precision_at_top_k') @@ -3674,7 +3334,7 @@ def precision_at_top_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes precision@k of the predictions with respect to sparse labels. @@ -3723,14 +3383,12 @@ def precision_at_top_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.precision_at_top_k is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.precision_at_top_k is not ' 'supported when eager execution is enabled.') with ops.name_scope( - name, _at_k_name('precision', k, class_id=class_id), - (predictions_idx, labels, weights) + name, + _at_k_name('precision', k, class_id=class_id), + (predictions_idx, labels, weights), ) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.to_int64(predictions_idx) @@ -3739,26 +3397,22 @@ def precision_at_top_k( labels=labels, k=k, class_id=class_id, - weights=weights + weights=weights, ) fp, fp_update = _streaming_sparse_false_positive_at_k( predictions_idx=top_k_idx, labels=labels, k=k, class_id=class_id, - weights=weights + weights=weights, ) def precision_across_towers(_, tp, fp): return math_ops.div(tp, math_ops.add(tp, fp), name=scope) - metric = _aggregate_across_towers( - metrics_collections, precision_across_towers, tp, fp - ) + metric = _aggregate_across_towers(metrics_collections, precision_across_towers, tp, fp) - update = math_ops.div( - tp_update, math_ops.add(tp_update, fp_update), name='update' - ) + update = math_ops.div(tp_update, math_ops.add(tp_update, fp_update), name='update') if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @@ -3774,7 +3428,7 @@ def sparse_precision_at_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Renamed to `precision_at_k`, please use that method instead.""" return precision_at_k( @@ -3785,7 +3439,7 @@ def sparse_precision_at_k( weights=weights, metrics_collections=metrics_collections, updates_collections=updates_collections, - name=name + name=name, ) @@ -3798,7 +3452,7 @@ def precision_at_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes precision@k of the predictions with respect to sparse labels. @@ -3868,14 +3522,12 @@ def precision_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.sparse_precision_at_k is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.sparse_precision_at_k is not ' 'supported when eager execution is enabled.') with ops.name_scope( - name, _at_k_name('precision', k, class_id=class_id), - (predictions, labels, weights) + name, + _at_k_name('precision', k, class_id=class_id), + (predictions, labels, weights), ) as scope: _, top_k_idx = nn.top_k(predictions, k) return precision_at_top_k( @@ -3886,7 +3538,7 @@ def precision_at_k( weights=weights, metrics_collections=metrics_collections, updates_collections=updates_collections, - name=scope + name=scope, ) @@ -3899,7 +3551,7 @@ def specificity_at_sensitivity( num_thresholds=200, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the specificity at a given sensitivity. @@ -3952,26 +3604,17 @@ def specificity_at_sensitivity( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.specificity_at_sensitivity is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.specificity_at_sensitivity is not ' 'supported when eager execution is enabled.') if sensitivity < 0 or sensitivity > 1: raise ValueError('`sensitivity` must be in the range [0, 1].') - with variable_scope.variable_scope( - name, 'specificity_at_sensitivity', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'specificity_at_sensitivity', (predictions, labels, weights)): kepsilon = 1e-7 # to account for floating point imprecisions - thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) - ] + thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] thresholds = [0.0 - kepsilon] + thresholds + [1.0 - kepsilon] - values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights - ) + values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): """Computes the specificity at the given sensitivity. @@ -3991,31 +3634,26 @@ def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): # We'll need to use this trick until tf.argmax allows us to specify # whether we should use the first or last index in case of ties. min_val = math_ops.reduce_min(math_ops.abs(sensitivities - sensitivity)) - indices_at_minval = math_ops.equal( - math_ops.abs(sensitivities - sensitivity), min_val - ) + indices_at_minval = math_ops.equal(math_ops.abs(sensitivities - sensitivity), min_val) indices_at_minval = math_ops.to_int64(indices_at_minval) indices_at_minval = math_ops.cumsum(indices_at_minval) tf_index = math_ops.argmax(indices_at_minval, 0) tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the specificity: - return math_ops.div( - tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, name - ) + return math_ops.div(tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, name) def specificity_across_towers(_, values): - return compute_specificity_at_sensitivity( - values['tp'], values['tn'], values['fp'], values['fn'], 'value' - ) + return compute_specificity_at_sensitivity(values['tp'], values['tn'], values['fp'], values['fn'], 'value') - specificity = _aggregate_across_towers( - metrics_collections, specificity_across_towers, values - ) + specificity = _aggregate_across_towers(metrics_collections, specificity_across_towers, values) update_op = compute_specificity_at_sensitivity( - update_ops['tp'], update_ops['tn'], update_ops['fp'], update_ops['fn'], - 'update_op' + update_ops['tp'], + update_ops['tn'], + update_ops['fp'], + update_ops['fn'], + 'update_op', ) if updates_collections: ops.add_to_collections(updates_collections, update_op) diff --git a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py index 641fbe362..417d5a481 100644 --- a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py +++ b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py @@ -18,12 +18,23 @@ from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.eager import context from tensorflow.python.framework import dtypes, ops, sparse_tensor +from tensorflow.python.ops import ( # NOQA + array_ops, + check_ops, + confusion_matrix, + control_flow_ops, + math_ops, + nn, + sets, + sparse_ops, + state_ops, + variable_scope, + weights_broadcast_ops, +) from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.tf_export import tf_export -from tensorflow.python.ops import array_ops, check_ops, confusion_matrix, control_flow_ops, math_ops, nn, sets, sparse_ops, state_ops, variable_scope, weights_broadcast_ops # NOQA - # from tensorflow.python.training import distribution_strategy_context @@ -65,13 +76,11 @@ def metric_variable(shape, dtype, validate_shape=True, name=None): return variable_scope.variable( lambda: array_ops.zeros(shape, dtype), trainable=False, - collections=[ - ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES - ], + collections=[ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES], validate_shape=validate_shape, synchronization=variable_scope.VariableSynchronization.ON_READ, aggregation=variable_scope.VariableAggregation.SUM, - name=name + name=name, ) @@ -100,9 +109,7 @@ def _remove_squeezable_dimensions(predictions, labels, weights): """ predictions = ops.convert_to_tensor(predictions) if labels is not None: - labels, predictions = confusion_matrix.remove_squeezable_dimensions( - labels, predictions - ) + labels, predictions = confusion_matrix.remove_squeezable_dimensions(labels, predictions) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if weights is None: @@ -130,30 +137,29 @@ def _remove_squeezable_dimensions(predictions, labels, weights): def _maybe_expand_weights(): return control_flow_ops.cond( math_ops.equal(rank_diff, -1), - lambda: array_ops.expand_dims(weights, [-1]), lambda: weights + lambda: array_ops.expand_dims(weights, [-1]), + lambda: weights, ) # Don't attempt squeeze if it will fail based on static check. - if ( - (weights_rank is not None) - and (not weights_shape.dims[-1].is_compatible_with(1)) - ): + if (weights_rank is not None) and (not weights_shape.dims[-1].is_compatible_with(1)): maybe_squeeze_weights = lambda: weights # noqa: E731 else: - maybe_squeeze_weights = lambda: array_ops.squeeze( # noqa: E731 - weights, [-1]) + maybe_squeeze_weights = lambda: array_ops.squeeze(weights, [-1]) # noqa: E731 def _maybe_adjust_weights(): return control_flow_ops.cond( - math_ops.equal(rank_diff, 1), maybe_squeeze_weights, - _maybe_expand_weights + math_ops.equal(rank_diff, 1), + maybe_squeeze_weights, + _maybe_expand_weights, ) # If weights are scalar, do nothing. Otherwise, try to add or remove a # dimension to match predictions. weights = control_flow_ops.cond( - math_ops.equal(weights_rank_tensor, 0), lambda: weights, - _maybe_adjust_weights + math_ops.equal(weights_rank_tensor, 0), + lambda: weights, + _maybe_adjust_weights, ) return predictions, labels, weights @@ -180,13 +186,14 @@ def _maybe_expand_labels(labels, predictions): # If sparse, expand sparse shape. if isinstance(labels, sparse_tensor.SparseTensor): return control_flow_ops.cond( - math_ops.equal(array_ops.rank(predictions), - array_ops.size(labels.dense_shape) + 1), - lambda: sparse_ops.sparse_reshape( # pylint: disable=g-long-lambda - labels, - shape=array_ops.concat((labels.dense_shape, (1, )), 0), - name=scope), - lambda: labels) + math_ops.equal(array_ops.rank(predictions), array_ops.size(labels.dense_shape) + 1), + lambda: sparse_ops.sparse_reshape( # pylint: disable=g-long-lambda + labels, + shape=array_ops.concat((labels.dense_shape, (1,)), 0), + name=scope, + ), + lambda: labels, + ) # Otherwise, try to use static shape. labels_rank = labels.get_shape().ndims @@ -198,15 +205,14 @@ def _maybe_expand_labels(labels, predictions): if predictions_rank == labels_rank + 1: return array_ops.expand_dims(labels, -1, name=scope) raise ValueError( - 'Unexpected labels shape %s for predictions shape %s.' % - (labels.get_shape(), predictions.get_shape()) + 'Unexpected labels shape %s for predictions shape %s.' % (labels.get_shape(), predictions.get_shape()) ) # Otherwise, use dynamic shape. return control_flow_ops.cond( - math_ops.equal(array_ops.rank(predictions), - array_ops.rank(labels) + 1), - lambda: array_ops.expand_dims(labels, -1, name=scope), lambda: labels + math_ops.equal(array_ops.rank(predictions), array_ops.rank(labels) + 1), + lambda: array_ops.expand_dims(labels, -1, name=scope), + lambda: labels, ) @@ -226,9 +232,7 @@ def _safe_scalar_div(numerator, denominator, name): return math_ops.div_no_nan(numerator, denominator, name=name) -def _streaming_confusion_matrix( - labels, predictions, num_classes, weights=None -): +def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): """Calculate a streaming confusion matrix. Calculates a confusion matrix. For estimation over a stream of data, @@ -252,9 +256,7 @@ def _streaming_confusion_matrix( update_op: An operation that increments the confusion matrix. """ # Local variable to accumulate the predictions in the confusion matrix. - total_cm = metric_variable( - [num_classes, num_classes], dtypes.float64, name='total_confusion_matrix' - ) + total_cm = metric_variable([num_classes, num_classes], dtypes.float64, name='total_confusion_matrix') # Cast the type to int64 required by confusion_matrix_ops. predictions = math_ops.cast(predictions, dtypes.int64) @@ -311,19 +313,11 @@ def fn(distribution, *a): ops.add_to_collections(metrics_collections, metric_value) return metric_value - return distribution_strategy_context.get_replica_context().merge_call( - fn, args=args - ) + return distribution_strategy_context.get_replica_context().merge_call(fn, args=args) @tf_export(v1=['metrics.mean']) -def mean( - values, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None -): +def mean(values, weights=None, metrics_collections=None, updates_collections=None, name=None): """Computes the (weighted) mean of the given values. The `mean` function creates two local variables, `total` and `count` @@ -362,10 +356,7 @@ def mean( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean is not supported when eager execution ' - 'is enabled.' - ) + raise RuntimeError('tf.metrics.mean is not supported when eager execution ' 'is enabled.') with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.cast(values, dtypes.float32) @@ -376,32 +367,20 @@ def mean( if weights is None: num_values = math_ops.cast(array_ops.size(values), dtypes.float32) else: - values, _, weights = _remove_squeezable_dimensions( - predictions=values, labels=None, weights=weights - ) - weights = weights_broadcast_ops.broadcast_weights( - math_ops.cast(weights, dtypes.float32), values - ) + values, _, weights = _remove_squeezable_dimensions(predictions=values, labels=None, weights=weights) + weights = weights_broadcast_ops.broadcast_weights(math_ops.cast(weights, dtypes.float32), values) values = math_ops.multiply(values, weights) num_values = math_ops.reduce_sum(weights) - update_total_op = state_ops.assign_add( - total, math_ops.reduce_sum(values), use_locking=True - ) + update_total_op = state_ops.assign_add(total, math_ops.reduce_sum(values), use_locking=True) with ops.control_dependencies([values]): - update_count_op = state_ops.assign_add( - count, num_values, use_locking=True - ) + update_count_op = state_ops.assign_add(count, num_values, use_locking=True) def compute_mean(_, t, c): return math_ops.div_no_nan(t, math_ops.maximum(c, 0), name='value') - mean_t = _aggregate_across_replicas( - metrics_collections, compute_mean, total, count - ) - update_op = math_ops.div_no_nan( - update_total_op, math_ops.maximum(update_count_op, 0), name='update_op' - ) + mean_t = _aggregate_across_replicas(metrics_collections, compute_mean, total, count) + update_op = math_ops.div_no_nan(update_total_op, math_ops.maximum(update_count_op, 0), name='update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -416,7 +395,7 @@ def accuracy( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Calculates how often `predictions` matches `labels`. @@ -462,29 +441,23 @@ def accuracy( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.accuracy is not supported when eager ' - 'execution is enabled.' - ) + raise RuntimeError('tf.metrics.accuracy is not supported when eager ' 'execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if labels.dtype != predictions.dtype: predictions = math_ops.cast(predictions, labels.dtype) - is_correct = math_ops.cast( - math_ops.equal(predictions, labels), dtypes.float32 - ) + is_correct = math_ops.cast(math_ops.equal(predictions, labels), dtypes.float32) return mean( - is_correct, weights, metrics_collections, updates_collections, name - or 'accuracy' + is_correct, + weights, + metrics_collections, + updates_collections, + name or 'accuracy', ) -def _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=None, includes=None -): +def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=None, includes=None): """Computes true_positives, false_negatives, true_negatives, false_positives. This function creates up to four local variables, `true_positives`, @@ -540,28 +513,26 @@ def _confusion_matrix_at_thresholds( check_ops.assert_greater_equal( predictions, math_ops.cast(0.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]' + message='predictions must be in [0, 1]', ), check_ops.assert_less_equal( predictions, math_ops.cast(1.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]' - ) + message='predictions must be in [0, 1]', + ), ] ): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtypes.float32), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights + weights=weights, ) num_thresholds = len(thresholds) # Reshape predictions and labels. predictions_2d = array_ops.reshape(predictions, [-1, 1]) - labels_2d = array_ops.reshape( - math_ops.cast(labels, dtype=dtypes.bool), [1, -1] - ) + labels_2d = array_ops.reshape(math_ops.cast(labels, dtype=dtypes.bool), [1, -1]) # Use static shape if known. num_predictions = predictions_2d.get_shape().as_list()[0] @@ -571,13 +542,13 @@ def _confusion_matrix_at_thresholds( num_predictions = array_ops.shape(predictions_2d)[0] thresh_tiled = array_ops.tile( array_ops.expand_dims(array_ops.constant(thresholds), [1]), - array_ops.stack([1, num_predictions]) + array_ops.stack([1, num_predictions]), ) # Tile the predictions after thresholding them across different thresholds. pred_is_pos = math_ops.greater( array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]), - thresh_tiled + thresh_tiled, ) if ('fn' in includes) or ('tn' in includes): pred_is_neg = math_ops.logical_not(pred_is_pos) @@ -588,15 +559,9 @@ def _confusion_matrix_at_thresholds( label_is_neg = math_ops.logical_not(label_is_pos) if weights is not None: - weights = weights_broadcast_ops.broadcast_weights( - math_ops.cast(weights, dtypes.float32), predictions - ) - weights_tiled = array_ops.tile( - array_ops.reshape(weights, [1, -1]), [num_thresholds, 1] - ) - thresh_tiled.get_shape().assert_is_compatible_with( - weights_tiled.get_shape() - ) + weights = weights_broadcast_ops.broadcast_weights(math_ops.cast(weights, dtypes.float32), predictions) + weights_tiled = array_ops.tile(array_ops.reshape(weights, [1, -1]), [num_thresholds, 1]) + thresh_tiled.get_shape().assert_is_compatible_with(weights_tiled.get_shape()) else: weights_tiled = None @@ -604,67 +569,42 @@ def _confusion_matrix_at_thresholds( update_ops = {} if 'tp' in includes: - true_p = metric_variable( - [num_thresholds], dtypes.float32, name='true_positives' - ) - is_true_positive = math_ops.cast( - math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32 - ) + true_p = metric_variable([num_thresholds], dtypes.float32, name='true_positives') + is_true_positive = math_ops.cast(math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32) if weights_tiled is not None: is_true_positive *= weights_tiled - update_ops['tp'] = state_ops.assign_add( - true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True - ) + update_ops['tp'] = state_ops.assign_add(true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True) values['tp'] = true_p if 'fn' in includes: - false_n = metric_variable( - [num_thresholds], dtypes.float32, name='false_negatives' - ) - is_false_negative = math_ops.cast( - math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32 - ) + false_n = metric_variable([num_thresholds], dtypes.float32, name='false_negatives') + is_false_negative = math_ops.cast(math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32) if weights_tiled is not None: is_false_negative *= weights_tiled - update_ops['fn'] = state_ops.assign_add( - false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True - ) + update_ops['fn'] = state_ops.assign_add(false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True) values['fn'] = false_n if 'tn' in includes: - true_n = metric_variable( - [num_thresholds], dtypes.float32, name='true_negatives' - ) - is_true_negative = math_ops.cast( - math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32 - ) + true_n = metric_variable([num_thresholds], dtypes.float32, name='true_negatives') + is_true_negative = math_ops.cast(math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32) if weights_tiled is not None: is_true_negative *= weights_tiled - update_ops['tn'] = state_ops.assign_add( - true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True - ) + update_ops['tn'] = state_ops.assign_add(true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True) values['tn'] = true_n if 'fp' in includes: - false_p = metric_variable( - [num_thresholds], dtypes.float32, name='false_positives' - ) - is_false_positive = math_ops.cast( - math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32 - ) + false_p = metric_variable([num_thresholds], dtypes.float32, name='false_positives') + is_false_positive = math_ops.cast(math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32) if weights_tiled is not None: is_false_positive *= weights_tiled - update_ops['fp'] = state_ops.assign_add( - false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True - ) + update_ops['fp'] = state_ops.assign_add(false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True) values['fp'] = false_p return values, update_ops def _aggregate_variable(v, collections): - f = lambda distribution, value: distribution.extended.read_var( # noqa: E731 - value) + f = lambda distribution, value: distribution.extended.read_var(value) # noqa: E731 return _aggregate_across_replicas(collections, f, v) @@ -679,7 +619,7 @@ def auc( curve='ROC', name=None, summation_method='trapezoidal', - thresholds=None + thresholds=None, ): """Computes the approximate AUC via a Riemann sum. @@ -760,14 +700,9 @@ def auc( """ print('use_tf_auc') if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.auc is not supported when eager execution ' - 'is enabled.' - ) + raise RuntimeError('tf.metrics.auc is not supported when eager execution ' 'is enabled.') - with variable_scope.variable_scope( - name, 'auc', (labels, predictions, weights) - ): + with variable_scope.variable_scope(name, 'auc', (labels, predictions, weights)): if curve != 'ROC' and curve != 'PR': raise ValueError('curve must be either ROC or PR, %s unknown' % (curve)) @@ -779,18 +714,13 @@ def auc( else: # Otherwise, linearly interpolate (num_thresholds - 2) thresholds in # (0, 1). - thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) - for i in range(num_thresholds - 2) - ] + thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] # Add an endpoint "threshold" below zero and above one for either threshold # method. thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] - values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights - ) + values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) # Add epsilons to avoid dividing by 0. epsilon = 1.0e-6 @@ -826,29 +756,30 @@ def interpolate_pr_auc(tp, fp, fn): Returns: pr_auc: an approximation of the area under the P-R curve. """ - dtp = tp[:num_thresholds - 1] - tp[1:] + dtp = tp[: num_thresholds - 1] - tp[1:] p = tp + fp prec_slope = math_ops.div_no_nan( dtp, - math_ops.maximum(p[:num_thresholds - 1] - p[1:], 0), - name='prec_slope' + math_ops.maximum(p[: num_thresholds - 1] - p[1:], 0), + name='prec_slope', ) intercept = tp[1:] - math_ops.multiply(prec_slope, p[1:]) safe_p_ratio = array_ops.where( - math_ops.logical_and(p[:num_thresholds - 1] > 0, p[1:] > 0), + math_ops.logical_and(p[: num_thresholds - 1] > 0, p[1:] > 0), math_ops.div_no_nan( - p[:num_thresholds - 1], + p[: num_thresholds - 1], math_ops.maximum(p[1:], 0), - name='recall_relative_ratio' - ), array_ops.ones_like(p[1:]) + name='recall_relative_ratio', + ), + array_ops.ones_like(p[1:]), ) return math_ops.reduce_sum( math_ops.div_no_nan( prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)), math_ops.maximum(tp[1:] + fn[1:], 0), - name='pr_auc_increment' + name='pr_auc_increment', ), - name='interpolate_pr_auc' + name='interpolate_pr_auc', ) def compute_auc(tp, fn, tn, fp, name): @@ -876,42 +807,41 @@ def compute_auc(tp, fn, tn, fp, name): # above. return math_ops.reduce_sum( math_ops.multiply( - x[:num_thresholds - 1] - x[1:], - (y[:num_thresholds - 1] + y[1:]) / 2. + x[: num_thresholds - 1] - x[1:], + (y[: num_thresholds - 1] + y[1:]) / 2.0, ), - name=name + name=name, ) elif summation_method == 'minoring': return math_ops.reduce_sum( math_ops.multiply( - x[:num_thresholds - 1] - x[1:], - math_ops.minimum(y[:num_thresholds - 1], y[1:]) + x[: num_thresholds - 1] - x[1:], + math_ops.minimum(y[: num_thresholds - 1], y[1:]), ), - name=name + name=name, ) elif summation_method == 'majoring': return math_ops.reduce_sum( math_ops.multiply( - x[:num_thresholds - 1] - x[1:], - math_ops.maximum(y[:num_thresholds - 1], y[1:]) + x[: num_thresholds - 1] - x[1:], + math_ops.maximum(y[: num_thresholds - 1], y[1:]), ), - name=name + name=name, ) else: raise ValueError('Invalid summation_method: %s' % summation_method) # sum up the areas of all the trapeziums def compute_auc_value(_, values): - return compute_auc( - values['tp'], values['fn'], values['tn'], values['fp'], 'value' - ) + return compute_auc(values['tp'], values['fn'], values['tn'], values['fp'], 'value') - auc_value = _aggregate_across_replicas( - metrics_collections, compute_auc_value, values - ) + auc_value = _aggregate_across_replicas(metrics_collections, compute_auc_value, values) update_op = compute_auc( - update_ops['tp'], update_ops['fn'], update_ops['tn'], update_ops['fp'], - 'update_op' + update_ops['tp'], + update_ops['fn'], + update_ops['tn'], + update_ops['fp'], + 'update_op', ) if updates_collections: @@ -927,7 +857,7 @@ def mean_absolute_error( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the mean absolute error between the labels and predictions. @@ -973,18 +903,16 @@ def mean_absolute_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_absolute_error is not supported ' - 'when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_absolute_error is not supported ' 'when eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) absolute_errors = math_ops.abs(predictions - labels) return mean( - absolute_errors, weights, metrics_collections, updates_collections, name - or 'mean_absolute_error' + absolute_errors, + weights, + metrics_collections, + updates_collections, + name or 'mean_absolute_error', ) @@ -996,7 +924,7 @@ def mean_cosine_distance( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the cosine distance between the labels and predictions. @@ -1040,23 +968,18 @@ def mean_cosine_distance( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_cosine_distance is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_cosine_distance is not supported when ' 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) radial_diffs = math_ops.multiply(predictions, labels) radial_diffs = math_ops.reduce_sum( - radial_diffs, axis=[ + radial_diffs, + axis=[ dim, - ], keepdims=True - ) - mean_distance, update_op = mean( - radial_diffs, weights, None, None, name or 'mean_cosine_distance' + ], + keepdims=True, ) + mean_distance, update_op = mean(radial_diffs, weights, None, None, name or 'mean_cosine_distance') mean_distance = math_ops.subtract(1.0, mean_distance) update_op = math_ops.subtract(1.0, update_op) @@ -1077,7 +1000,7 @@ def mean_per_class_accuracy( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Calculates the mean of the per-class accuracies. @@ -1120,14 +1043,9 @@ def mean_per_class_accuracy( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_per_class_accuracy is not supported ' - 'when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_per_class_accuracy is not supported ' 'when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'mean_accuracy', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'mean_accuracy', (predictions, labels, weights)): labels = math_ops.cast(labels, dtypes.int64) # Flatten the input if its rank > 1. @@ -1147,9 +1065,7 @@ def mean_per_class_accuracy( if labels.dtype != predictions.dtype: predictions = math_ops.cast(predictions, labels.dtype) - is_correct = math_ops.cast( - math_ops.equal(predictions, labels), dtypes.float32 - ) + is_correct = math_ops.cast(math_ops.equal(predictions, labels), dtypes.float32) if weights is not None: if weights.get_shape().ndims > 1: @@ -1163,21 +1079,13 @@ def mean_per_class_accuracy( update_count_op = state_ops.scatter_add(count, labels, is_correct) def compute_mean_accuracy(_, count, total): - per_class_accuracy = math_ops.div_no_nan( - count, math_ops.maximum(total, 0), name=None - ) - mean_accuracy_v = math_ops.reduce_mean( - per_class_accuracy, name='mean_accuracy' - ) + per_class_accuracy = math_ops.div_no_nan(count, math_ops.maximum(total, 0), name=None) + mean_accuracy_v = math_ops.reduce_mean(per_class_accuracy, name='mean_accuracy') return mean_accuracy_v - mean_accuracy_v = _aggregate_across_replicas( - metrics_collections, compute_mean_accuracy, count, total - ) + mean_accuracy_v = _aggregate_across_replicas(metrics_collections, compute_mean_accuracy, count, total) - update_op = math_ops.div_no_nan( - update_count_op, math_ops.maximum(update_total_op, 0), name='update_op' - ) + update_op = math_ops.div_no_nan(update_count_op, math_ops.maximum(update_total_op, 0), name='update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1192,7 +1100,7 @@ def mean_iou( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Calculate per-step mean Intersection-Over-Union (mIOU). @@ -1239,60 +1147,45 @@ def mean_iou( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_iou is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_iou is not supported when ' 'eager execution is enabled.') - with variable_scope.variable_scope( - name, 'mean_iou', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'mean_iou', (predictions, labels, weights)): # Check if shape is compatible. predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - total_cm, update_op = _streaming_confusion_matrix( - labels, predictions, num_classes, weights - ) + total_cm, update_op = _streaming_confusion_matrix(labels, predictions, num_classes, weights) def compute_mean_iou(_, total_cm): """Compute the mean intersection-over-union via the confusion matrix.""" - sum_over_row = math_ops.cast( - math_ops.reduce_sum(total_cm, 0), dtypes.float32 - ) - sum_over_col = math_ops.cast( - math_ops.reduce_sum(total_cm, 1), dtypes.float32 - ) + sum_over_row = math_ops.cast(math_ops.reduce_sum(total_cm, 0), dtypes.float32) + sum_over_col = math_ops.cast(math_ops.reduce_sum(total_cm, 1), dtypes.float32) cm_diag = math_ops.cast(array_ops.diag_part(total_cm), dtypes.float32) denominator = sum_over_row + sum_over_col - cm_diag # The mean is only computed over classes that appear in the # label or prediction tensor. If the denominator is 0, we need to # ignore the class. - num_valid_entries = math_ops.reduce_sum( - math_ops.cast( - math_ops.not_equal(denominator, 0), dtype=dtypes.float32 - ) - ) + num_valid_entries = math_ops.reduce_sum(math_ops.cast(math_ops.not_equal(denominator, 0), dtype=dtypes.float32)) # If the value of the denominator is 0, set it to 1 to avoid # zero division. denominator = array_ops.where( - math_ops.greater(denominator, 0), denominator, - array_ops.ones_like(denominator) + math_ops.greater(denominator, 0), + denominator, + array_ops.ones_like(denominator), ) iou = math_ops.div(cm_diag, denominator) # If the number of valid entries is 0 (no classes) we return 0. result = array_ops.where( math_ops.greater(num_valid_entries, 0), - math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, 0 + math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, + 0, ) return result # TODO(priyag): Use outside_compilation if in TPU context. - mean_iou_v = _aggregate_across_replicas( - metrics_collections, compute_mean_iou, total_cm - ) + mean_iou_v = _aggregate_across_replicas(metrics_collections, compute_mean_iou, total_cm) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1308,7 +1201,7 @@ def mean_relative_error( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the mean relative error by normalizing with the given values. @@ -1355,26 +1248,23 @@ def mean_relative_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_relative_error is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_relative_error is not supported when ' 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) - predictions, normalizer = confusion_matrix.remove_squeezable_dimensions( - predictions, normalizer - ) + predictions, normalizer = confusion_matrix.remove_squeezable_dimensions(predictions, normalizer) predictions.get_shape().assert_is_compatible_with(normalizer.get_shape()) relative_errors = array_ops.where( - math_ops.equal(normalizer, 0.0), array_ops.zeros_like(labels), - math_ops.div(math_ops.abs(labels - predictions), normalizer) + math_ops.equal(normalizer, 0.0), + array_ops.zeros_like(labels), + math_ops.div(math_ops.abs(labels - predictions), normalizer), ) return mean( - relative_errors, weights, metrics_collections, updates_collections, name - or 'mean_relative_error' + relative_errors, + weights, + metrics_collections, + updates_collections, + name or 'mean_relative_error', ) @@ -1385,7 +1275,7 @@ def mean_squared_error( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the mean squared error between the labels and predictions. @@ -1431,29 +1321,21 @@ def mean_squared_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_squared_error is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_squared_error is not supported when ' 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) squared_error = math_ops.squared_difference(labels, predictions) return mean( - squared_error, weights, metrics_collections, updates_collections, name - or 'mean_squared_error' + squared_error, + weights, + metrics_collections, + updates_collections, + name or 'mean_squared_error', ) @tf_export(v1=['metrics.mean_tensor']) -def mean_tensor( - values, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None -): +def mean_tensor(values, weights=None, metrics_collections=None, updates_collections=None, name=None): """Computes the element-wise (weighted) mean of the given tensors. In contrast to the `mean` function which returns a scalar with the @@ -1496,47 +1378,29 @@ def mean_tensor( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.mean_tensor is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.mean_tensor is not supported when ' 'eager execution is enabled.') with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.cast(values, dtypes.float32) - total = metric_variable( - values.get_shape(), dtypes.float32, name='total_tensor' - ) - count = metric_variable( - values.get_shape(), dtypes.float32, name='count_tensor' - ) + total = metric_variable(values.get_shape(), dtypes.float32, name='total_tensor') + count = metric_variable(values.get_shape(), dtypes.float32, name='count_tensor') num_values = array_ops.ones_like(values) if weights is not None: - values, _, weights = _remove_squeezable_dimensions( - predictions=values, labels=None, weights=weights - ) - weights = weights_broadcast_ops.broadcast_weights( - math_ops.cast(weights, dtypes.float32), values - ) + values, _, weights = _remove_squeezable_dimensions(predictions=values, labels=None, weights=weights) + weights = weights_broadcast_ops.broadcast_weights(math_ops.cast(weights, dtypes.float32), values) values = math_ops.multiply(values, weights) num_values = math_ops.multiply(num_values, weights) update_total_op = state_ops.assign_add(total, values, use_locking=True) with ops.control_dependencies([values]): - update_count_op = state_ops.assign_add( - count, num_values, use_locking=True - ) + update_count_op = state_ops.assign_add(count, num_values, use_locking=True) - compute_mean = lambda _, t, c: math_ops.div_no_nan( # noqa: E731 - t, math_ops.maximum(c, 0), name='value') + compute_mean = lambda _, t, c: math_ops.div_no_nan(t, math_ops.maximum(c, 0), name='value') # noqa: E731 - mean_t = _aggregate_across_replicas( - metrics_collections, compute_mean, total, count - ) + mean_t = _aggregate_across_replicas(metrics_collections, compute_mean, total, count) - update_op = math_ops.div_no_nan( - update_total_op, math_ops.maximum(update_count_op, 0), name='update_op' - ) + update_op = math_ops.div_no_nan(update_total_op, math_ops.maximum(update_count_op, 0), name='update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1550,7 +1414,7 @@ def percentage_below( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the percentage of values less than the given threshold. @@ -1591,23 +1455,19 @@ def percentage_below( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.percentage_below is not supported when ' - 'eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.percentage_below is not supported when ' 'eager execution is enabled.') - is_below_threshold = math_ops.cast( - math_ops.less(values, threshold), dtypes.float32 - ) + is_below_threshold = math_ops.cast(math_ops.less(values, threshold), dtypes.float32) return mean( - is_below_threshold, weights, metrics_collections, updates_collections, name - or 'percentage_below_threshold' + is_below_threshold, + weights, + metrics_collections, + updates_collections, + name or 'percentage_below_threshold', ) -def _count_condition( - values, weights=None, metrics_collections=None, updates_collections=None -): +def _count_condition(values, weights=None, metrics_collections=None, updates_collections=None): """Sums the weights of cases where the given values are True. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1636,17 +1496,13 @@ def _count_condition( values = math_ops.cast(values, dtypes.float32) if weights is not None: - with ops.control_dependencies( - (check_ops.assert_rank_in(weights, (0, array_ops.rank(values))), ) - ): + with ops.control_dependencies((check_ops.assert_rank_in(weights, (0, array_ops.rank(values))),)): weights = math_ops.cast(weights, dtypes.float32) values = math_ops.multiply(values, weights) value_tensor = _aggregate_variable(count, metrics_collections) - update_op = state_ops.assign_add( - count, math_ops.reduce_sum(values), use_locking=True - ) + update_op = state_ops.assign_add(count, math_ops.reduce_sum(values), use_locking=True) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1660,7 +1516,7 @@ def false_negatives( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the total number of false negatives. @@ -1691,26 +1547,16 @@ def false_negatives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.false_negatives is not supported when ' - 'eager execution is enabled.' - ) - - with variable_scope.variable_scope( - name, 'false_negatives', (predictions, labels, weights) - ): + raise RuntimeError('tf.metrics.false_negatives is not supported when ' 'eager execution is enabled.') + with variable_scope.variable_scope(name, 'false_negatives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights - ) - is_false_negative = math_ops.logical_and( - math_ops.equal(labels, True), math_ops.equal(predictions, False) - ) - return _count_condition( - is_false_negative, weights, metrics_collections, updates_collections + weights=weights, ) + is_false_negative = math_ops.logical_and(math_ops.equal(labels, True), math_ops.equal(predictions, False)) + return _count_condition(is_false_negative, weights, metrics_collections, updates_collections) @tf_export(v1=['metrics.false_negatives_at_thresholds']) @@ -1721,7 +1567,7 @@ def false_negatives_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes false negatives at provided threshold values. @@ -1755,16 +1601,11 @@ def false_negatives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.false_negatives_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.false_negatives_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'false_negatives', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'false_negatives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fn', ) + labels, predictions, thresholds, weights=weights, includes=('fn',) ) fn_value = _aggregate_variable(values['fn'], metrics_collections) @@ -1782,7 +1623,7 @@ def false_positives( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Sum the weights of false positives. @@ -1814,26 +1655,16 @@ def false_positives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.false_positives is not supported when ' - 'eager execution is enabled.' - ) - - with variable_scope.variable_scope( - name, 'false_positives', (predictions, labels, weights) - ): + raise RuntimeError('tf.metrics.false_positives is not supported when ' 'eager execution is enabled.') + with variable_scope.variable_scope(name, 'false_positives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights - ) - is_false_positive = math_ops.logical_and( - math_ops.equal(labels, False), math_ops.equal(predictions, True) - ) - return _count_condition( - is_false_positive, weights, metrics_collections, updates_collections + weights=weights, ) + is_false_positive = math_ops.logical_and(math_ops.equal(labels, False), math_ops.equal(predictions, True)) + return _count_condition(is_false_positive, weights, metrics_collections, updates_collections) @tf_export(v1=['metrics.false_positives_at_thresholds']) @@ -1844,7 +1675,7 @@ def false_positives_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes false positives at provided threshold values. @@ -1878,16 +1709,11 @@ def false_positives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.false_positives_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.false_positives_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'false_positives', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'false_positives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fp', ) + labels, predictions, thresholds, weights=weights, includes=('fp',) ) fp_value = _aggregate_variable(values['fp'], metrics_collections) @@ -1905,7 +1731,7 @@ def true_negatives( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Sum the weights of true_negatives. @@ -1937,26 +1763,16 @@ def true_negatives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.true_negatives is not ' - 'supported when eager execution is enabled.' - ) - - with variable_scope.variable_scope( - name, 'true_negatives', (predictions, labels, weights) - ): + raise RuntimeError('tf.metrics.true_negatives is not ' 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights - ) - is_true_negative = math_ops.logical_and( - math_ops.equal(labels, False), math_ops.equal(predictions, False) - ) - return _count_condition( - is_true_negative, weights, metrics_collections, updates_collections + weights=weights, ) + is_true_negative = math_ops.logical_and(math_ops.equal(labels, False), math_ops.equal(predictions, False)) + return _count_condition(is_true_negative, weights, metrics_collections, updates_collections) @tf_export(v1=['metrics.true_negatives_at_thresholds']) @@ -1967,7 +1783,7 @@ def true_negatives_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes true negatives at provided threshold values. @@ -2001,16 +1817,11 @@ def true_negatives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.true_negatives_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.true_negatives_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'true_negatives', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tn', ) + labels, predictions, thresholds, weights=weights, includes=('tn',) ) tn_value = _aggregate_variable(values['tn'], metrics_collections) @@ -2028,7 +1839,7 @@ def true_positives( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Sum the weights of true_positives. @@ -2060,26 +1871,16 @@ def true_positives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.true_positives is not ' - 'supported when eager execution is enabled.' - ) - - with variable_scope.variable_scope( - name, 'true_positives', (predictions, labels, weights) - ): + raise RuntimeError('tf.metrics.true_positives is not ' 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'true_positives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights - ) - is_true_positive = math_ops.logical_and( - math_ops.equal(labels, True), math_ops.equal(predictions, True) - ) - return _count_condition( - is_true_positive, weights, metrics_collections, updates_collections + weights=weights, ) + is_true_positive = math_ops.logical_and(math_ops.equal(labels, True), math_ops.equal(predictions, True)) + return _count_condition(is_true_positive, weights, metrics_collections, updates_collections) @tf_export(v1=['metrics.true_positives_at_thresholds']) @@ -2090,7 +1891,7 @@ def true_positives_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes true positives at provided threshold values. @@ -2124,16 +1925,11 @@ def true_positives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.true_positives_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.true_positives_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'true_positives', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'true_positives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tp', ) + labels, predictions, thresholds, weights=weights, includes=('tp',) ) tp_value = _aggregate_variable(values['tp'], metrics_collections) @@ -2151,7 +1947,7 @@ def precision( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the precision of the predictions with respect to the labels. @@ -2197,19 +1993,13 @@ def precision( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.precision is not ' - 'supported when eager execution is enabled.' - ) - - with variable_scope.variable_scope( - name, 'precision', (predictions, labels, weights) - ): + raise RuntimeError('tf.metrics.precision is not ' 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'precision', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights + weights=weights, ) true_p, true_positives_update_op = true_positives( @@ -2218,7 +2008,7 @@ def precision( weights, metrics_collections=None, updates_collections=None, - name=None + name=None, ) false_p, false_positives_update_op = false_positives( labels, @@ -2226,24 +2016,18 @@ def precision( weights, metrics_collections=None, updates_collections=None, - name=None + name=None, ) def compute_precision(tp, fp, name): - return array_ops.where( - math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name - ) + return array_ops.where(math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name) def once_across_replicas(_, true_p, false_p): return compute_precision(true_p, false_p, 'value') - p = _aggregate_across_replicas( - metrics_collections, once_across_replicas, true_p, false_p - ) + p = _aggregate_across_replicas(metrics_collections, once_across_replicas, true_p, false_p) - update_op = compute_precision( - true_positives_update_op, false_positives_update_op, 'update_op' - ) + update_op = compute_precision(true_positives_update_op, false_positives_update_op, 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2258,7 +2042,7 @@ def precision_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes precision values for different `thresholds` on `predictions`. @@ -2305,14 +2089,9 @@ def precision_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.precision_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.precision_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'precision_at_thresholds', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'precision_at_thresholds', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( labels, predictions, thresholds, weights, includes=('tp', 'fp') ) @@ -2326,13 +2105,9 @@ def compute_precision(tp, fp, name): def precision_across_replicas(_, values): return compute_precision(values['tp'], values['fp'], 'value') - prec = _aggregate_across_replicas( - metrics_collections, precision_across_replicas, values - ) + prec = _aggregate_across_replicas(metrics_collections, precision_across_replicas, values) - update_op = compute_precision( - update_ops['tp'], update_ops['fp'], 'update_op' - ) + update_op = compute_precision(update_ops['tp'], update_ops['fp'], 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2346,7 +2121,7 @@ def recall( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the recall of the predictions with respect to the labels. @@ -2390,18 +2165,13 @@ def recall( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.recall is not supported is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.recall is not supported is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'recall', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'recall', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights + weights=weights, ) true_p, true_positives_update_op = true_positives( @@ -2410,7 +2180,7 @@ def recall( weights, metrics_collections=None, updates_collections=None, - name=None + name=None, ) false_n, false_negatives_update_op = false_negatives( labels, @@ -2418,25 +2188,23 @@ def recall( weights, metrics_collections=None, updates_collections=None, - name=None + name=None, ) def compute_recall(true_p, false_n, name): return array_ops.where( math_ops.greater(true_p + false_n, 0), - math_ops.div(true_p, true_p + false_n), 0, name + math_ops.div(true_p, true_p + false_n), + 0, + name, ) def once_across_replicas(_, true_p, false_n): return compute_recall(true_p, false_n, 'value') - rec = _aggregate_across_replicas( - metrics_collections, once_across_replicas, true_p, false_n - ) + rec = _aggregate_across_replicas(metrics_collections, once_across_replicas, true_p, false_n) - update_op = compute_recall( - true_positives_update_op, false_negatives_update_op, 'update_op' - ) + update_op = compute_recall(true_positives_update_op, false_negatives_update_op, 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2466,9 +2234,7 @@ def _select_class_id(ids, selected_id): """ ids = sparse_tensor.convert_to_tensor_or_sparse_tensor(ids) if isinstance(ids, sparse_tensor.SparseTensor): - return sparse_ops.sparse_retain( - ids, math_ops.equal(ids.values, selected_id) - ) + return sparse_ops.sparse_retain(ids, math_ops.equal(ids.values, selected_id)) # TODO(ptucker): Make this more efficient, maybe add a sparse version of # tf.equal and tf.reduce_any? @@ -2476,18 +2242,12 @@ def _select_class_id(ids, selected_id): # Shape of filled IDs is the same as `ids` with the last dim collapsed to 1. ids_shape = array_ops.shape(ids, out_type=dtypes.int64) ids_last_dim = array_ops.size(ids_shape) - 1 - filled_selected_id_shape = math_ops.reduced_shape( - ids_shape, array_ops.reshape(ids_last_dim, [1]) - ) + filled_selected_id_shape = math_ops.reduced_shape(ids_shape, array_ops.reshape(ids_last_dim, [1])) # Intersect `ids` with the selected ID. - filled_selected_id = array_ops.fill( - filled_selected_id_shape, math_ops.cast(selected_id, dtypes.int64) - ) + filled_selected_id = array_ops.fill(filled_selected_id_shape, math_ops.cast(selected_id, dtypes.int64)) result = sets.set_intersection(filled_selected_id, ids) - return sparse_tensor.SparseTensor( - indices=result.indices, values=result.values, dense_shape=ids_shape - ) + return sparse_tensor.SparseTensor(indices=result.indices, values=result.values, dense_shape=ids_shape) def _maybe_select_class_id(labels, predictions_idx, selected_id=None): @@ -2511,13 +2271,11 @@ def _maybe_select_class_id(labels, predictions_idx, selected_id=None): return labels, predictions_idx return ( _select_class_id(labels, selected_id), - _select_class_id(predictions_idx, selected_id) + _select_class_id(predictions_idx, selected_id), ) -def _sparse_true_positive_at_k( - labels, predictions_idx, class_id=None, weights=None, name=None -): +def _sparse_true_positive_at_k(labels, predictions_idx, class_id=None, weights=None, name=None): """Calculates true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2544,26 +2302,18 @@ def _sparse_true_positive_at_k( Returns: A [D1, ... DN] `Tensor` of true positive counts. """ - with ops.name_scope( - name, 'true_positives', (predictions_idx, labels, weights) - ): - labels, predictions_idx = _maybe_select_class_id( - labels, predictions_idx, class_id - ) + with ops.name_scope(name, 'true_positives', (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) tp = sets.set_size(sets.set_intersection(predictions_idx, labels)) tp = math_ops.cast(tp, dtypes.float64) if weights is not None: - with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, tp), ) - ): + with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, tp),)): weights = math_ops.cast(weights, dtypes.float64) tp = math_ops.multiply(tp, weights) return tp -def _streaming_sparse_true_positive_at_k( - labels, predictions_idx, k=None, class_id=None, weights=None, name=None -): +def _streaming_sparse_true_positive_at_k(labels, predictions_idx, k=None, class_id=None, weights=None, name=None): """Calculates weighted per step true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2597,26 +2347,23 @@ def _streaming_sparse_true_positive_at_k( ValueError: If `weights` is not `None` and has an incompatible shape. """ with ops.name_scope( - name, _at_k_name('true_positive', k, class_id=class_id), - (predictions_idx, labels, weights) + name, + _at_k_name('true_positive', k, class_id=class_id), + (predictions_idx, labels, weights), ) as scope: tp = _sparse_true_positive_at_k( predictions_idx=predictions_idx, labels=labels, class_id=class_id, - weights=weights + weights=weights, ) batch_total_tp = math_ops.cast(math_ops.reduce_sum(tp), dtypes.float64) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add( - var, batch_total_tp, name='update', use_locking=True - ) + return var, state_ops.assign_add(var, batch_total_tp, name='update', use_locking=True) -def _sparse_false_negative_at_k( - labels, predictions_idx, class_id=None, weights=None -): +def _sparse_false_negative_at_k(labels, predictions_idx, class_id=None, weights=None): """Calculates false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2642,28 +2389,18 @@ def _sparse_false_negative_at_k( Returns: A [D1, ... DN] `Tensor` of false negative counts. """ - with ops.name_scope( - None, 'false_negatives', (predictions_idx, labels, weights) - ): - labels, predictions_idx = _maybe_select_class_id( - labels, predictions_idx, class_id - ) - fn = sets.set_size( - sets.set_difference(predictions_idx, labels, aminusb=False) - ) + with ops.name_scope(None, 'false_negatives', (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) + fn = sets.set_size(sets.set_difference(predictions_idx, labels, aminusb=False)) fn = math_ops.cast(fn, dtypes.float64) if weights is not None: - with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, fn), ) - ): + with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, fn),)): weights = math_ops.cast(weights, dtypes.float64) fn = math_ops.multiply(fn, weights) return fn -def _streaming_sparse_false_negative_at_k( - labels, predictions_idx, k, class_id=None, weights=None, name=None -): +def _streaming_sparse_false_negative_at_k(labels, predictions_idx, k, class_id=None, weights=None, name=None): """Calculates weighted per step false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2697,21 +2434,20 @@ def _streaming_sparse_false_negative_at_k( ValueError: If `weights` is not `None` and has an incompatible shape. """ with ops.name_scope( - name, _at_k_name('false_negative', k, class_id=class_id), - (predictions_idx, labels, weights) + name, + _at_k_name('false_negative', k, class_id=class_id), + (predictions_idx, labels, weights), ) as scope: fn = _sparse_false_negative_at_k( predictions_idx=predictions_idx, labels=labels, class_id=class_id, - weights=weights + weights=weights, ) batch_total_fn = math_ops.cast(math_ops.reduce_sum(fn), dtypes.float64) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add( - var, batch_total_fn, name='update', use_locking=True - ) + return var, state_ops.assign_add(var, batch_total_fn, name='update', use_locking=True) @tf_export(v1=['metrics.recall_at_k']) @@ -2723,7 +2459,7 @@ def recall_at_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes recall@k of the predictions with respect to sparse labels. @@ -2792,15 +2528,9 @@ def recall_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.recall_at_k is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.recall_at_k is not ' 'supported when eager execution is enabled.') - with ops.name_scope( - name, _at_k_name('recall', k, class_id=class_id), - (predictions, labels, weights) - ) as scope: + with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), (predictions, labels, weights)) as scope: _, top_k_idx = nn.top_k(predictions, k) return recall_at_top_k( labels=labels, @@ -2810,7 +2540,7 @@ def recall_at_k( weights=weights, metrics_collections=metrics_collections, updates_collections=updates_collections, - name=scope + name=scope, ) @@ -2823,7 +2553,7 @@ def recall_at_top_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes recall@k of top-k predictions with respect to sparse labels. @@ -2871,8 +2601,9 @@ class indices, whereas `recall_at_k` expects logits. Refer to `recall_at_k` are not a list or tuple. """ with ops.name_scope( - name, _at_k_name('recall', k, class_id=class_id), - (predictions_idx, labels, weights) + name, + _at_k_name('recall', k, class_id=class_id), + (predictions_idx, labels, weights), ) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.cast(predictions_idx, dtypes.int64) @@ -2881,26 +2612,22 @@ class indices, whereas `recall_at_k` expects logits. Refer to `recall_at_k` labels=labels, k=k, class_id=class_id, - weights=weights + weights=weights, ) fn, fn_update = _streaming_sparse_false_negative_at_k( predictions_idx=top_k_idx, labels=labels, k=k, class_id=class_id, - weights=weights + weights=weights, ) def compute_recall(_, tp, fn): return math_ops.div(tp, math_ops.add(tp, fn), name=scope) - metric = _aggregate_across_replicas( - metrics_collections, compute_recall, tp, fn - ) + metric = _aggregate_across_replicas(metrics_collections, compute_recall, tp, fn) - update = math_ops.div( - tp_update, math_ops.add(tp_update, fn_update), name='update' - ) + update = math_ops.div(tp_update, math_ops.add(tp_update, fn_update), name='update') if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @@ -2914,7 +2641,7 @@ def recall_at_thresholds( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes various recall values for different `thresholds` on `predictions`. @@ -2959,14 +2686,9 @@ def recall_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.recall_at_thresholds is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.recall_at_thresholds is not ' 'supported when eager execution is enabled.') - with variable_scope.variable_scope( - name, 'recall_at_thresholds', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'recall_at_thresholds', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( labels, predictions, thresholds, weights, includes=('tp', 'fn') ) @@ -2980,9 +2702,7 @@ def compute_recall(tp, fn, name): def recall_across_replicas(_, values): return compute_recall(values['tp'], values['fn'], 'value') - rec = _aggregate_across_replicas( - metrics_collections, recall_across_replicas, values - ) + rec = _aggregate_across_replicas(metrics_collections, recall_across_replicas, values) update_op = compute_recall(update_ops['tp'], update_ops['fn'], 'update_op') if updates_collections: @@ -2998,7 +2718,7 @@ def root_mean_squared_error( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the root mean squared error between the labels and predictions. @@ -3044,22 +2764,13 @@ def root_mean_squared_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.root_mean_squared_error is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.root_mean_squared_error is not ' 'supported when eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights - ) - mse, update_mse_op = mean_squared_error( - labels, predictions, weights, None, None, name or 'root_mean_squared_error' - ) + predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + mse, update_mse_op = mean_squared_error(labels, predictions, weights, None, None, name or 'root_mean_squared_error') once_across_replicas = lambda _, mse: math_ops.sqrt(mse) # noqa: E731 - rmse = _aggregate_across_replicas( - metrics_collections, once_across_replicas, mse - ) + rmse = _aggregate_across_replicas(metrics_collections, once_across_replicas, mse) update_rmse_op = math_ops.sqrt(update_mse_op) if updates_collections: @@ -3077,7 +2788,7 @@ def sensitivity_at_specificity( num_thresholds=200, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the specificity at a given sensitivity. @@ -3130,26 +2841,17 @@ def sensitivity_at_specificity( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.sensitivity_at_specificity is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.sensitivity_at_specificity is not ' 'supported when eager execution is enabled.') if specificity < 0 or specificity > 1: raise ValueError('`specificity` must be in the range [0, 1].') - with variable_scope.variable_scope( - name, 'sensitivity_at_specificity', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'sensitivity_at_specificity', (predictions, labels, weights)): kepsilon = 1e-7 # to account for floating point imprecisions - thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) - ] + thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] - values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights - ) + values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): specificities = math_ops.div(tn, tn + fp + kepsilon) @@ -3157,22 +2859,19 @@ def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the sensitivity: - return math_ops.div( - tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, name - ) + return math_ops.div(tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, name) def sensitivity_across_replicas(_, values): - return compute_sensitivity_at_specificity( - values['tp'], values['tn'], values['fp'], values['fn'], 'value' - ) + return compute_sensitivity_at_specificity(values['tp'], values['tn'], values['fp'], values['fn'], 'value') - sensitivity = _aggregate_across_replicas( - metrics_collections, sensitivity_across_replicas, values - ) + sensitivity = _aggregate_across_replicas(metrics_collections, sensitivity_across_replicas, values) update_op = compute_sensitivity_at_specificity( - update_ops['tp'], update_ops['tn'], update_ops['fp'], update_ops['fn'], - 'update_op' + update_ops['tp'], + update_ops['tn'], + update_ops['fp'], + update_ops['fn'], + 'update_op', ) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -3201,45 +2900,34 @@ def _expand_and_tile(tensor, multiple, dim=0, name=None): """ if multiple < 1: raise ValueError('Invalid multiple %s, must be > 0.' % multiple) - with ops.name_scope( - name, 'expand_and_tile', (tensor, multiple, dim) - ) as scope: + with ops.name_scope(name, 'expand_and_tile', (tensor, multiple, dim)) as scope: # Sparse. tensor = sparse_tensor.convert_to_tensor_or_sparse_tensor(tensor) if isinstance(tensor, sparse_tensor.SparseTensor): if dim < 0: - expand_dims = array_ops.reshape( - array_ops.size(tensor.dense_shape) + dim, [1] - ) + expand_dims = array_ops.reshape(array_ops.size(tensor.dense_shape) + dim, [1]) else: expand_dims = [dim] expanded_shape = array_ops.concat( ( - array_ops.slice(tensor.dense_shape, [0], expand_dims), [1], - array_ops.slice(tensor.dense_shape, expand_dims, [-1]) + array_ops.slice(tensor.dense_shape, [0], expand_dims), + [1], + array_ops.slice(tensor.dense_shape, expand_dims, [-1]), ), 0, - name='expanded_shape' - ) - expanded = sparse_ops.sparse_reshape( - tensor, shape=expanded_shape, name='expand' + name='expanded_shape', ) + expanded = sparse_ops.sparse_reshape(tensor, shape=expanded_shape, name='expand') if multiple == 1: return expanded - return sparse_ops.sparse_concat( - dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope - ) + return sparse_ops.sparse_concat(dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope) # Dense. - expanded = array_ops.expand_dims( - tensor, dim if (dim >= 0) else (dim - 1), name='expand' - ) + expanded = array_ops.expand_dims(tensor, dim if (dim >= 0) else (dim - 1), name='expand') if multiple == 1: return expanded ones = array_ops.ones_like(array_ops.shape(tensor)) - tile_multiples = array_ops.concat( - (ones[:dim], (multiple, ), ones[dim:]), 0, name='multiples' - ) + tile_multiples = array_ops.concat((ones[:dim], (multiple,), ones[dim:]), 0, name='multiples') return array_ops.tile(expanded, tile_multiples, name=scope) @@ -3265,7 +2953,7 @@ def _num_relevant(labels, k): """ if k < 1: raise ValueError('Invalid k=%s.' % k) - with ops.name_scope(None, 'num_relevant', (labels, )) as scope: + with ops.name_scope(None, 'num_relevant', (labels,)) as scope: # For SparseTensor, calculate separate count for each row. labels = sparse_tensor.convert_to_tensor_or_sparse_tensor(labels) if isinstance(labels, sparse_tensor.SparseTensor): @@ -3275,10 +2963,11 @@ def _num_relevant(labels, k): # number of labels along the last dimension that are non-negative. num_labels = math_ops.reduce_sum( array_ops.where_v2( - math_ops.greater_equal(labels, 0), array_ops.ones_like(labels), - array_ops.zeros_like(labels) + math_ops.greater_equal(labels, 0), + array_ops.ones_like(labels), + array_ops.zeros_like(labels), ), - axis=-1 + axis=-1, ) return math_ops.minimum(num_labels, k, name=scope) @@ -3315,12 +3004,8 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): Raises: ValueError: if the last dimension of predictions_idx is not set. """ - with ops.name_scope( - None, 'average_precision', (predictions_idx, labels) - ) as scope: - predictions_idx = math_ops.cast( - predictions_idx, dtypes.int64, name='predictions_idx' - ) + with ops.name_scope(None, 'average_precision', (predictions_idx, labels)) as scope: + predictions_idx = math_ops.cast(predictions_idx, dtypes.int64, name='predictions_idx') if predictions_idx.get_shape().ndims == 0: raise ValueError('The rank of predictions_idx must be at least 1.') k = predictions_idx.get_shape().as_list()[-1] @@ -3331,14 +3016,10 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # Expand dims to produce [D1, ... DN, k, 1] tensor. This gives us a separate # prediction for each k, so we can calculate separate true positive values # for each k. - predictions_idx_per_k = array_ops.expand_dims( - predictions_idx, -1, name='predictions_idx_per_k' - ) + predictions_idx_per_k = array_ops.expand_dims(predictions_idx, -1, name='predictions_idx_per_k') # Replicate labels k times to produce [D1, ... DN, k, num_labels] tensor. - labels_per_k = _expand_and_tile( - labels, multiple=k, dim=-1, name='labels_per_k' - ) + labels_per_k = _expand_and_tile(labels, multiple=k, dim=-1, name='labels_per_k') # The following tensors are all of shape [D1, ... DN, k], containing values # per row, per k value. @@ -3352,34 +3033,26 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # term from the formula above. # `relevant_precision_per_k` (float64) - Relevant precisions; i.e., # precisions at all k for which relevance indicator is true. - relevant_per_k = _sparse_true_positive_at_k( - labels_per_k, predictions_idx_per_k, name='relevant_per_k' - ) + relevant_per_k = _sparse_true_positive_at_k(labels_per_k, predictions_idx_per_k, name='relevant_per_k') tp_per_k = math_ops.cumsum(relevant_per_k, axis=-1, name='tp_per_k') - retrieved_per_k = math_ops.cumsum( - array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k' - ) + retrieved_per_k = math_ops.cumsum(array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k') precision_per_k = math_ops.div( math_ops.cast(tp_per_k, dtypes.float64), math_ops.cast(retrieved_per_k, dtypes.float64), - name='precision_per_k' + name='precision_per_k', ) relevant_precision_per_k = math_ops.multiply( precision_per_k, math_ops.cast(relevant_per_k, dtypes.float64), - name='relevant_precision_per_k' + name='relevant_precision_per_k', ) # Reduce along k dimension to get the sum, yielding a [D1, ... DN] tensor. - precision_sum = math_ops.reduce_sum( - relevant_precision_per_k, axis=(-1, ), name='precision_sum' - ) + precision_sum = math_ops.reduce_sum(relevant_precision_per_k, axis=(-1,), name='precision_sum') # Divide by number of relevant items to get average precision. These are # the "num_relevant_items" and "AveP" terms from the formula above. - num_relevant_items = math_ops.cast( - _num_relevant(labels, k), dtypes.float64 - ) + num_relevant_items = math_ops.cast(_num_relevant(labels, k), dtypes.float64) return math_ops.div(precision_sum, num_relevant_items, name=scope) @@ -3389,7 +3062,7 @@ def _streaming_sparse_average_precision_at_top_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes average precision@k of predictions with respect to sparse labels. @@ -3435,42 +3108,30 @@ def _streaming_sparse_average_precision_at_top_k( update: `Operation` that increments variables appropriately, and whose value matches `metric`. """ - with ops.name_scope( - name, 'average_precision_at_top_k', (predictions_idx, labels, weights) - ) as scope: + with ops.name_scope(name, 'average_precision_at_top_k', (predictions_idx, labels, weights)) as scope: # Calculate per-example average precision, and apply weights. - average_precision = _sparse_average_precision_at_top_k( - predictions_idx=predictions_idx, labels=labels - ) + average_precision = _sparse_average_precision_at_top_k(predictions_idx=predictions_idx, labels=labels) if weights is not None: - weights = weights_broadcast_ops.broadcast_weights( - math_ops.cast(weights, dtypes.float64), average_precision - ) + weights = weights_broadcast_ops.broadcast_weights(math_ops.cast(weights, dtypes.float64), average_precision) average_precision = math_ops.multiply(average_precision, weights) # Create accumulation variables and update ops for max average precision and # total average precision. - with ops.name_scope(None, 'max', (average_precision, )) as max_scope: + with ops.name_scope(None, 'max', (average_precision,)) as max_scope: # `max` is the max possible precision. Since max for any row is 1.0: # - For the unweighted case, this is just the number of rows. # - For the weighted case, it's the sum of the weights broadcast across # `average_precision` rows. max_var = metric_variable([], dtypes.float64, name=max_scope) if weights is None: - batch_max = math_ops.cast( - array_ops.size(average_precision, name='batch_max'), dtypes.float64 - ) + batch_max = math_ops.cast(array_ops.size(average_precision, name='batch_max'), dtypes.float64) else: batch_max = math_ops.reduce_sum(weights, name='batch_max') - max_update = state_ops.assign_add( - max_var, batch_max, name='update', use_locking=True - ) - with ops.name_scope(None, 'total', (average_precision, )) as total_scope: + max_update = state_ops.assign_add(max_var, batch_max, name='update', use_locking=True) + with ops.name_scope(None, 'total', (average_precision,)) as total_scope: total_var = metric_variable([], dtypes.float64, name=total_scope) batch_total = math_ops.reduce_sum(average_precision, name='batch_total') - total_update = state_ops.assign_add( - total_var, batch_total, name='update', use_locking=True - ) + total_update = state_ops.assign_add(total_var, batch_total, name='update', use_locking=True) # Divide total by max to get mean, for both vars and the update ops. def precision_across_replicas(_, total_var, max_var): @@ -3504,15 +3165,14 @@ def _clean_out_of_range_indices(labels, num_classes): def _labels_is_sparse(): """Returns true is `labels` is a sparse tensor.""" - return isinstance( - labels, (sparse_tensor.SparseTensor, sparse_tensor.SparseTensorValue) - ) + return isinstance(labels, (sparse_tensor.SparseTensor, sparse_tensor.SparseTensorValue)) def _clean_out_of_range(values): """Replaces by -1 any large out-of-range `values`.""" return array_ops.where_v2( math_ops.greater_equal(values, num_classes), - -1 * array_ops.ones_like(values), values + -1 * array_ops.ones_like(values), + values, ) def _clean_labels_out_of_range(): @@ -3521,17 +3181,16 @@ def _clean_labels_out_of_range(): return type(labels)( indices=labels.indices, values=_clean_out_of_range(labels.values), - dense_shape=labels.dense_shape + dense_shape=labels.dense_shape, ) else: return _clean_out_of_range(labels) - max_labels = math_ops.reduce_max( - labels.values if _labels_is_sparse() else labels - ) + max_labels = math_ops.reduce_max(labels.values if _labels_is_sparse() else labels) return control_flow_ops.cond( math_ops.greater_equal(max_labels, num_classes), - _clean_labels_out_of_range, lambda: labels + _clean_labels_out_of_range, + lambda: labels, ) @@ -3544,7 +3203,7 @@ def sparse_average_precision_at_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Renamed to `average_precision_at_k`, please use that method instead.""" return average_precision_at_k( @@ -3554,7 +3213,7 @@ def sparse_average_precision_at_k( weights=weights, metrics_collections=metrics_collections, updates_collections=updates_collections, - name=name + name=name, ) @@ -3566,7 +3225,7 @@ def average_precision_at_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes average precision@k of predictions with respect to sparse labels. @@ -3621,38 +3280,29 @@ def average_precision_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.sparse_average_precision_at_k is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.sparse_average_precision_at_k is not ' 'supported when eager execution is enabled.') if k < 1: raise ValueError('Invalid k=%s.' % k) - with ops.name_scope( - name, _at_k_name('average_precision', k), (predictions, labels, weights) - ) as scope: + with ops.name_scope(name, _at_k_name('average_precision', k), (predictions, labels, weights)) as scope: # Calculate top k indices to produce [D1, ... DN, k] tensor. _, predictions_idx = nn.top_k(predictions, k) # The documentation states that labels should be in [0, ..., num_classes), # but num_classes is lost when predictions_idx replaces predictions. # For conformity with the documentation, any label >= num_classes, which is # ignored, is replaced by -1. - labels = _clean_out_of_range_indices( - labels, math_ops.cast(array_ops.shape(predictions)[-1], dtypes.int64) - ) + labels = _clean_out_of_range_indices(labels, math_ops.cast(array_ops.shape(predictions)[-1], dtypes.int64)) return _streaming_sparse_average_precision_at_top_k( labels=labels, predictions_idx=predictions_idx, weights=weights, metrics_collections=metrics_collections, updates_collections=updates_collections, - name=scope + name=scope, ) -def _sparse_false_positive_at_k( - labels, predictions_idx, class_id=None, weights=None -): +def _sparse_false_positive_at_k(labels, predictions_idx, class_id=None, weights=None): """Calculates false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3678,28 +3328,18 @@ def _sparse_false_positive_at_k( Returns: A [D1, ... DN] `Tensor` of false positive counts. """ - with ops.name_scope( - None, 'false_positives', (predictions_idx, labels, weights) - ): - labels, predictions_idx = _maybe_select_class_id( - labels, predictions_idx, class_id - ) - fp = sets.set_size( - sets.set_difference(predictions_idx, labels, aminusb=True) - ) + with ops.name_scope(None, 'false_positives', (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) + fp = sets.set_size(sets.set_difference(predictions_idx, labels, aminusb=True)) fp = math_ops.cast(fp, dtypes.float64) if weights is not None: - with ops.control_dependencies( - (weights_broadcast_ops.assert_broadcastable(weights, fp), ) - ): + with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, fp),)): weights = math_ops.cast(weights, dtypes.float64) fp = math_ops.multiply(fp, weights) return fp -def _streaming_sparse_false_positive_at_k( - labels, predictions_idx, k=None, class_id=None, weights=None, name=None -): +def _streaming_sparse_false_positive_at_k(labels, predictions_idx, k=None, class_id=None, weights=None, name=None): """Calculates weighted per step false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3733,21 +3373,20 @@ def _streaming_sparse_false_positive_at_k( ValueError: If `weights` is not `None` and has an incompatible shape. """ with ops.name_scope( - name, _at_k_name('false_positive', k, class_id=class_id), - (predictions_idx, labels, weights) + name, + _at_k_name('false_positive', k, class_id=class_id), + (predictions_idx, labels, weights), ) as scope: fp = _sparse_false_positive_at_k( predictions_idx=predictions_idx, labels=labels, class_id=class_id, - weights=weights + weights=weights, ) batch_total_fp = math_ops.cast(math_ops.reduce_sum(fp), dtypes.float64) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add( - var, batch_total_fp, name='update', use_locking=True - ) + return var, state_ops.assign_add(var, batch_total_fp, name='update', use_locking=True) @tf_export(v1=['metrics.precision_at_top_k']) @@ -3759,7 +3398,7 @@ def precision_at_top_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes precision@k of the predictions with respect to sparse labels. @@ -3808,14 +3447,12 @@ def precision_at_top_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.precision_at_top_k is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.precision_at_top_k is not ' 'supported when eager execution is enabled.') with ops.name_scope( - name, _at_k_name('precision', k, class_id=class_id), - (predictions_idx, labels, weights) + name, + _at_k_name('precision', k, class_id=class_id), + (predictions_idx, labels, weights), ) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.cast(predictions_idx, dtypes.int64) @@ -3824,26 +3461,22 @@ def precision_at_top_k( labels=labels, k=k, class_id=class_id, - weights=weights + weights=weights, ) fp, fp_update = _streaming_sparse_false_positive_at_k( predictions_idx=top_k_idx, labels=labels, k=k, class_id=class_id, - weights=weights + weights=weights, ) def precision_across_replicas(_, tp, fp): return math_ops.div(tp, math_ops.add(tp, fp), name=scope) - metric = _aggregate_across_replicas( - metrics_collections, precision_across_replicas, tp, fp - ) + metric = _aggregate_across_replicas(metrics_collections, precision_across_replicas, tp, fp) - update = math_ops.div( - tp_update, math_ops.add(tp_update, fp_update), name='update' - ) + update = math_ops.div(tp_update, math_ops.add(tp_update, fp_update), name='update') if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @@ -3859,7 +3492,7 @@ def sparse_precision_at_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Renamed to `precision_at_k`, please use that method instead.""" return precision_at_k( @@ -3870,7 +3503,7 @@ def sparse_precision_at_k( weights=weights, metrics_collections=metrics_collections, updates_collections=updates_collections, - name=name + name=name, ) @@ -3883,7 +3516,7 @@ def precision_at_k( weights=None, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes precision@k of the predictions with respect to sparse labels. @@ -3953,14 +3586,12 @@ def precision_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.sparse_precision_at_k is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.sparse_precision_at_k is not ' 'supported when eager execution is enabled.') with ops.name_scope( - name, _at_k_name('precision', k, class_id=class_id), - (predictions, labels, weights) + name, + _at_k_name('precision', k, class_id=class_id), + (predictions, labels, weights), ) as scope: _, top_k_idx = nn.top_k(predictions, k) return precision_at_top_k( @@ -3971,7 +3602,7 @@ def precision_at_k( weights=weights, metrics_collections=metrics_collections, updates_collections=updates_collections, - name=scope + name=scope, ) @@ -3984,7 +3615,7 @@ def specificity_at_sensitivity( num_thresholds=200, metrics_collections=None, updates_collections=None, - name=None + name=None, ): """Computes the specificity at a given sensitivity. @@ -4037,26 +3668,17 @@ def specificity_at_sensitivity( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError( - 'tf.metrics.specificity_at_sensitivity is not ' - 'supported when eager execution is enabled.' - ) + raise RuntimeError('tf.metrics.specificity_at_sensitivity is not ' 'supported when eager execution is enabled.') if sensitivity < 0 or sensitivity > 1: raise ValueError('`sensitivity` must be in the range [0, 1].') - with variable_scope.variable_scope( - name, 'specificity_at_sensitivity', (predictions, labels, weights) - ): + with variable_scope.variable_scope(name, 'specificity_at_sensitivity', (predictions, labels, weights)): kepsilon = 1e-7 # to account for floating point imprecisions - thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) - ] + thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] thresholds = [0.0 - kepsilon] + thresholds + [1.0 - kepsilon] - values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights - ) + values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): """Computes the specificity at the given sensitivity. @@ -4076,31 +3698,26 @@ def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): # We'll need to use this trick until tf.argmax allows us to specify # whether we should use the first or last index in case of ties. min_val = math_ops.reduce_min(math_ops.abs(sensitivities - sensitivity)) - indices_at_minval = math_ops.equal( - math_ops.abs(sensitivities - sensitivity), min_val - ) + indices_at_minval = math_ops.equal(math_ops.abs(sensitivities - sensitivity), min_val) indices_at_minval = math_ops.cast(indices_at_minval, dtypes.int64) indices_at_minval = math_ops.cumsum(indices_at_minval) tf_index = math_ops.argmax(indices_at_minval, 0) tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the specificity: - return math_ops.div( - tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, name - ) + return math_ops.div(tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, name) def specificity_across_replicas(_, values): - return compute_specificity_at_sensitivity( - values['tp'], values['tn'], values['fp'], values['fn'], 'value' - ) + return compute_specificity_at_sensitivity(values['tp'], values['tn'], values['fp'], values['fn'], 'value') - specificity = _aggregate_across_replicas( - metrics_collections, specificity_across_replicas, values - ) + specificity = _aggregate_across_replicas(metrics_collections, specificity_across_replicas, values) update_op = compute_specificity_at_sensitivity( - update_ops['tp'], update_ops['tn'], update_ops['fp'], update_ops['fn'], - 'update_op' + update_ops['tp'], + update_ops['tn'], + update_ops['fp'], + update_ops['fn'], + 'update_op', ) if updates_collections: ops.add_to_collections(updates_collections, update_op) diff --git a/easy_rec/python/core/learning_schedules.py b/easy_rec/python/core/learning_schedules.py index d8944fc27..d1b1fa2e1 100644 --- a/easy_rec/python/core/learning_schedules.py +++ b/easy_rec/python/core/learning_schedules.py @@ -30,7 +30,7 @@ def exponential_decay_with_burnin( burnin_learning_rate=0.0, burnin_steps=0, min_learning_rate=0.0, - staircase=True + staircase=True, ): """Exponential decay schedule with burn-in period. @@ -59,23 +59,22 @@ def exponential_decay_with_burnin( burnin_rate = learning_rate_base else: slope = (learning_rate_base - burnin_learning_rate) / burnin_steps - burnin_rate = slope * tf.cast( - global_step, tf.float32 - ) + burnin_learning_rate + burnin_rate = slope * tf.cast(global_step, tf.float32) + burnin_learning_rate post_burnin_learning_rate = tf.train.exponential_decay( learning_rate_base, global_step - burnin_steps, learning_rate_decay_steps, learning_rate_decay_factor, - staircase=staircase + staircase=staircase, ) return tf.maximum( tf.where( tf.less(tf.cast(global_step, tf.int32), tf.constant(burnin_steps)), - burnin_rate, post_burnin_learning_rate + burnin_rate, + post_burnin_learning_rate, ), min_learning_rate, - name='learning_rate' + name='learning_rate', ) @@ -85,7 +84,7 @@ def cosine_decay_with_warmup( total_steps, warmup_learning_rate=0.0, warmup_steps=0, - hold_base_rate_steps=0 + hold_base_rate_steps=0, ): """Cosine decay schedule with warm up period. @@ -113,38 +112,32 @@ def cosine_decay_with_warmup( or if warmup_steps is larger than total_steps. """ if learning_rate_base < warmup_learning_rate: - raise ValueError( - 'learning_rate_base must be larger ' - 'or equal to warmup_learning_rate.' - ) + raise ValueError('learning_rate_base must be larger ' 'or equal to warmup_learning_rate.') if total_steps < warmup_steps: - raise ValueError( - 'total_steps must be larger or equal to ' - 'warmup_steps.' - ) - learning_rate = 0.5 * learning_rate_base * ( - 1 + tf.cos( - np.pi * - (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps) - / float(total_steps - warmup_steps - hold_base_rate_steps) + raise ValueError('total_steps must be larger or equal to ' 'warmup_steps.') + learning_rate = ( + 0.5 + * learning_rate_base + * ( + 1 + + tf.cos( + np.pi + * (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps) + / float(total_steps - warmup_steps - hold_base_rate_steps) + ) ) ) if hold_base_rate_steps > 0: learning_rate = tf.where( - global_step > warmup_steps + hold_base_rate_steps, learning_rate, - learning_rate_base + global_step > warmup_steps + hold_base_rate_steps, + learning_rate, + learning_rate_base, ) if warmup_steps > 0: slope = (learning_rate_base - warmup_learning_rate) / warmup_steps - warmup_rate = slope * tf.cast( - global_step, tf.float32 - ) + warmup_learning_rate - learning_rate = tf.where( - global_step < warmup_steps, warmup_rate, learning_rate - ) - return tf.where( - global_step > total_steps, 0.0, learning_rate, name='learning_rate' - ) + warmup_rate = slope * tf.cast(global_step, tf.float32) + warmup_learning_rate + learning_rate = tf.where(global_step < warmup_steps, warmup_rate, learning_rate) + return tf.where(global_step > total_steps, 0.0, learning_rate, name='learning_rate') def manual_stepping(global_step, boundaries, rates, warmup=False): @@ -175,18 +168,14 @@ def manual_stepping(global_step, boundaries, rates, warmup=False): 2. len(rates) == len(boundaries) + 1 3. boundaries[0] != 0 """ - if any([b < 0 for b in boundaries] - ) or any([not isinstance(b, int) for b in boundaries]): + if any([b < 0 for b in boundaries]) or any([not isinstance(b, int) for b in boundaries]): raise ValueError('boundaries must be a list of positive integers') if any([bnext <= b for bnext, b in zip(boundaries[1:], boundaries[:-1])]): raise ValueError('Entries in boundaries must be strictly increasing.') if any([not isinstance(r, float) for r in rates]): raise ValueError('Learning rates must be floats') if len(rates) != len(boundaries) + 1: - raise ValueError( - 'Number of provided learning rates must exceed ' - 'number of boundary points by exactly 1.' - ) + raise ValueError('Number of provided learning rates must exceed ' 'number of boundary points by exactly 1.') if boundaries and boundaries[0] == 0: raise ValueError('First step cannot be zero.') @@ -202,13 +191,12 @@ def manual_stepping(global_step, boundaries, rates, warmup=False): num_boundaries = len(boundaries) rate_index = tf.reduce_max( tf.where( - tf.greater_equal(global_step, boundaries), list(range(num_boundaries)), - [0] * num_boundaries + tf.greater_equal(global_step, boundaries), + list(range(num_boundaries)), + [0] * num_boundaries, ) ) - return tf.reduce_sum( - rates * tf.one_hot(rate_index, depth=num_boundaries), name='learning_rate' - ) + return tf.reduce_sum(rates * tf.one_hot(rate_index, depth=num_boundaries), name='learning_rate') def transformer_policy( @@ -219,7 +207,7 @@ def transformer_policy( step_scaling_rate=1.0, max_lr=None, coefficient=1.0, - dtype=tf.float32 + dtype=tf.float32, ): """Transformer's learning rate schedule. @@ -246,9 +234,7 @@ def transformer_policy( step_num *= step_scaling_rate ws *= step_scaling_rate - decay = coefficient * d_model**-0.5 * tf.minimum( - (step_num + 1) * ws**-1.5, (step_num + 1)**-0.5 - ) + decay = coefficient * d_model**-0.5 * tf.minimum((step_num + 1) * ws**-1.5, (step_num + 1) ** -0.5) new_lr = decay * learning_rate if max_lr is not None: diff --git a/easy_rec/python/core/metrics.py b/easy_rec/python/core/metrics.py index e1dea67a9..57a2b5945 100644 --- a/easy_rec/python/core/metrics.py +++ b/easy_rec/python/core/metrics.py @@ -8,14 +8,15 @@ import numpy as np import tensorflow as tf from sklearn import metrics as sklearn_metrics +from tensorflow.python.ops import array_ops, math_ops, state_ops, variable_scope # NOQA from easy_rec.python.utils.estimator_utils import get_task_index_and_num +from easy_rec.python.utils.io_util import ( # NOQA + read_data_from_json_path, + save_data_to_json_path, +) from easy_rec.python.utils.shape_utils import get_shape_list -from tensorflow.python.ops import array_ops, math_ops, state_ops, variable_scope # NOQA - -from easy_rec.python.utils.io_util import read_data_from_json_path, save_data_to_json_path # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -28,24 +29,21 @@ def max_f1(label, predictions): predictions: Estimated targets as returned by a model. """ from easy_rec.python.core.easyrec_metrics import metrics_tf + num_thresholds = 200 kepsilon = 1e-7 - thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) - ] + thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] f1_scores = [] precision_update_ops = [] recall_update_ops = [] for threshold in thresholds: - pred = (predictions > threshold) + pred = predictions > threshold precision, precision_update_op = metrics_tf.precision( labels=label, predictions=pred, name='precision_%s' % threshold ) - recall, recall_update_op = metrics_tf.recall( - labels=label, predictions=pred, name='recall_%s' % threshold - ) + recall, recall_update_op = metrics_tf.recall(labels=label, predictions=pred, name='recall_%s' % threshold) f1_score = (2 * precision * recall) / (precision + recall + 1e-12) precision_update_ops.append(precision_update_op) recall_update_ops.append(recall_update_op) @@ -70,8 +68,11 @@ def _separated_auc_impl(labels, predictions, keys, reduction='mean'): * "mean_by_sample_num": weighted mean with sample num of different keys * "mean_by_positive_num": weighted mean with positive sample num of different keys """ - assert reduction in ['mean', 'mean_by_sample_num', 'mean_by_positive_num'], \ - 'reduction method must in mean | mean_by_sample_num | mean_by_positive_num' + assert reduction in [ + 'mean', + 'mean_by_sample_num', + 'mean_by_positive_num', + ], 'reduction method must in mean | mean_by_sample_num | mean_by_positive_num' separated_label = defaultdict(list) separated_prediction = defaultdict(list) separated_weights = defaultdict(int) @@ -122,8 +123,7 @@ def value_pyfunc(pos_neg_arr, total_pos_neg): auc += pos_neg_arr[0][i] * pos_neg_arr[1][i] auc = np.double(auc) / np.double(total_pos * total_neg * 2) logging.info( - 'fast_auc[%s]: total_pos=%d total_neg=%d total=%d' % - (name, total_pos, total_neg, total_pos + total_neg) + 'fast_auc[%s]: total_pos=%d total_neg=%d total=%d' % (name, total_pos, total_neg, total_pos + total_neg) ) return np.float32(auc) @@ -134,7 +134,7 @@ def value_pyfunc(pos_neg_arr, total_pos_neg): trainable=False, collections=[tf.GraphKeys.METRIC_VARIABLES], initializer=tf.zeros_initializer(), - dtype=tf.int64 + dtype=tf.int64, ) total_var = variable_scope.get_variable( name='total_cnt', @@ -142,27 +142,25 @@ def value_pyfunc(pos_neg_arr, total_pos_neg): trainable=False, collections=[tf.GraphKeys.METRIC_VARIABLES], initializer=tf.zeros_initializer(), - dtype=tf.int64 + dtype=tf.int64, ) pred_bins = math_ops.cast(predictions * num_thresholds, dtype=tf.int32) labels = math_ops.cast(labels, dtype=tf.int32) labels = array_ops.reshape(labels, [-1, 1]) pred_bins = array_ops.reshape(pred_bins, [-1, 1]) update_op0 = state_ops.scatter_nd_add( - neg_pos_var, tf.concat([labels, pred_bins], axis=1), - array_ops.ones(tf.shape(labels)[0], dtype=tf.int64) + neg_pos_var, + tf.concat([labels, pred_bins], axis=1), + array_ops.ones(tf.shape(labels)[0], dtype=tf.int64), ) total_pos = math_ops.reduce_sum(labels) total_neg = array_ops.shape(labels)[0] - total_pos total_add = math_ops.cast(tf.stack([total_neg, total_pos]), dtype=tf.int64) update_op1 = state_ops.assign_add(total_var, total_add) - return tf.py_func(value_pyfunc, [neg_pos_var, total_var], - tf.float32), tf.group([update_op0, update_op1]) + return tf.py_func(value_pyfunc, [neg_pos_var, total_var], tf.float32), tf.group([update_op0, update_op1]) -def _distribute_separated_auc_impl( - labels, predictions, keys, reduction='mean', metric_name='sepatated_auc' -): +def _distribute_separated_auc_impl(labels, predictions, keys, reduction='mean', metric_name='sepatated_auc'): """Computes the AUC group by the key separately. Args: @@ -177,21 +175,20 @@ def _distribute_separated_auc_impl( * "mean_by_sample_num": weighted mean with sample num of different keys * "mean_by_positive_num": weighted mean with positive sample num of different keys """ - assert reduction in ['mean', 'mean_by_sample_num', 'mean_by_positive_num'], \ - 'reduction method must in mean | mean_by_sample_num | mean_by_positive_num' + assert reduction in [ + 'mean', + 'mean_by_sample_num', + 'mean_by_positive_num', + ], 'reduction method must in mean | mean_by_sample_num | mean_by_positive_num' separated_label = defaultdict(list) separated_prediction = defaultdict(list) separated_weights = defaultdict(int) tf_config = json.loads(os.environ['TF_CONFIG']) cur_job_name = tf_config['task']['type'] cur_task_index, task_num = get_task_index_and_num() - cur_work_device = 'job_' + cur_job_name + '__' + 'task_' + str( - cur_task_index - ) + cur_work_device = 'job_' + cur_job_name + '__' + 'task_' + str(cur_task_index) eval_tmp_results_dir = os.environ['eval_tmp_results_dir'] - assert tf.gfile.IsDirectory( - eval_tmp_results_dir - ), 'eval_tmp_results_dir not exists' + assert tf.gfile.IsDirectory(eval_tmp_results_dir), 'eval_tmp_results_dir not exists' def update_pyfunc(labels, predictions, keys): for label, prediction, key in zip(labels, predictions, keys): @@ -206,7 +203,7 @@ def update_pyfunc(labels, predictions, keys): separated_weights[key] += label.item() for name, data in zip( ['separated_label', 'separated_prediction', 'separated_weights'], - [separated_label, separated_prediction, separated_weights] + [separated_label, separated_prediction, separated_weights], ): cur_json_name = metric_name + '__' + cur_work_device + '__' + name + '.json' cur_json_path = os.path.join(eval_tmp_results_dir, cur_json_name) @@ -216,36 +213,35 @@ def value_pyfunc(): for task_i in range(1, task_num): work_device_i = 'job_worker__task_' + str(task_i) for name in [ - 'separated_label', 'separated_prediction', 'separated_weights' + 'separated_label', + 'separated_prediction', + 'separated_weights', ]: json_name_i = metric_name + '__' + work_device_i + '__' + name + '.json' json_path_i = os.path.join(eval_tmp_results_dir, json_name_i) data_i = read_data_from_json_path(json_path_i) - if (name == 'separated_label'): + if name == 'separated_label': separated_label.update( { key: separated_label.get(key, []) + data_i.get(key, []) - for key in - set(list(separated_label.keys()) + list(data_i.keys())) + for key in set(list(separated_label.keys()) + list(data_i.keys())) } ) - elif (name == 'separated_prediction'): + elif name == 'separated_prediction': separated_prediction.update( { key: separated_prediction.get(key, []) + data_i.get(key, []) - for key in - set(list(separated_prediction.keys()) + list(data_i.keys())) + for key in set(list(separated_prediction.keys()) + list(data_i.keys())) } ) - elif (name == 'separated_weights'): + elif name == 'separated_weights': if reduction == 'mean': separated_weights.update(data_i) else: separated_weights.update( { key: separated_weights.get(key, 0) + data_i.get(key, 0) - for key in - set(list(separated_weights.keys()) + list(data_i.keys())) + for key in set(list(separated_weights.keys()) + list(data_i.keys())) } ) else: @@ -285,9 +281,7 @@ def gauc(labels, predictions, uids, reduction='mean'): * "mean_by_positive_num": weighted mean with positive sample num of different users """ if os.environ.get('distribute_eval') == 'True': - return _distribute_separated_auc_impl( - labels, predictions, uids, reduction, metric_name='gauc' - ) + return _distribute_separated_auc_impl(labels, predictions, uids, reduction, metric_name='gauc') return _separated_auc_impl(labels, predictions, uids, reduction) @@ -306,15 +300,11 @@ def session_auc(labels, predictions, session_ids, reduction='mean'): * "mean_by_positive_num": weighted mean with positive sample num of different sessions """ if os.environ.get('distribute_eval') == 'True': - return _distribute_separated_auc_impl( - labels, predictions, session_ids, reduction, metric_name='session_auc' - ) + return _distribute_separated_auc_impl(labels, predictions, session_ids, reduction, metric_name='session_auc') return _separated_auc_impl(labels, predictions, session_ids, reduction) -def metric_learning_recall_at_k( - k, embeddings, labels, session_ids=None, embed_normed=False -): +def metric_learning_recall_at_k(k, embeddings, labels, session_ids=None, embed_normed=False): """Computes the recall_at_k metric for metric learning. Args: @@ -337,19 +327,13 @@ def metric_learning_recall_at_k( # Uses broadcasting where the 1st argument has shape (1, batch_size) and the 2nd (batch_size, 1) labels_equal = tf.equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1)) if session_ids is not None and session_ids is not labels: - sessions_equal = tf.equal( - tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1) - ) + sessions_equal = tf.equal(tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1)) labels_equal = tf.logical_and(sessions_equal, labels_equal) mask = tf.logical_and(indices_not_equal, labels_equal) - mask_pos = tf.where( - mask, sim_mat, -array_ops.ones_like(sim_mat) - ) # shape: (batch_size, batch_size) + mask_pos = tf.where(mask, sim_mat, -array_ops.ones_like(sim_mat)) # shape: (batch_size, batch_size) if isinstance(k, int): _, pos_top_k_idx = tf.nn.top_k(mask_pos, k) # shape: (batch_size, k) - return metrics_tf.recall_at_k( - labels=tf.to_int64(pos_top_k_idx), predictions=sim_mat, k=k - ) + return metrics_tf.recall_at_k(labels=tf.to_int64(pos_top_k_idx), predictions=sim_mat, k=k) if any((isinstance(k, list), isinstance(k, tuple), isinstance(k, set))): metrics = {} for kk in k: @@ -364,9 +348,7 @@ def metric_learning_recall_at_k( raise ValueError('k should be a `int` or a list/tuple/set of int.') -def metric_learning_average_precision_at_k( - k, embeddings, labels, session_ids=None, embed_normed=False -): +def metric_learning_average_precision_at_k(k, embeddings, labels, session_ids=None, embed_normed=False): from easy_rec.python.core.easyrec_metrics import metrics_tf # make sure embedding should be l2-normalized @@ -378,9 +360,7 @@ def metric_learning_average_precision_at_k( sim_mat = sim_mat - tf.eye(batch_size) * 2.0 mask = tf.equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1)) if session_ids is not None and session_ids is not labels: - sessions_equal = tf.equal( - tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1) - ) + sessions_equal = tf.equal(tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1)) mask = tf.logical_and(sessions_equal, mask) label_indices = _get_matrix_mask_indices(mask) if isinstance(k, int): @@ -390,9 +370,7 @@ def metric_learning_average_precision_at_k( for kk in k: if kk < 1: continue - metrics['MAP@' + str(kk)] = metrics_tf.average_precision_at_k( - label_indices, sim_mat, kk - ) + metrics['MAP@' + str(kk)] = metrics_tf.average_precision_at_k(label_indices, sim_mat, kk) return metrics else: raise ValueError('k should be a `int` or a list/tuple/set of int.') @@ -403,9 +381,7 @@ def _get_matrix_mask_indices(matrix, num_rows=None): num_rows = get_shape_list(matrix)[0] indices = tf.where(matrix) num_indices = tf.shape(indices)[0] - elem_per_row = tf.bincount( - tf.cast(indices[:, 0], tf.int32), minlength=num_rows - ) + elem_per_row = tf.bincount(tf.cast(indices[:, 0], tf.int32), minlength=num_rows) max_elem_per_row = tf.reduce_max(elem_per_row) row_start = tf.concat([[0], tf.cumsum(elem_per_row[:-1])], axis=0) r = tf.range(max_elem_per_row) @@ -413,9 +389,7 @@ def _get_matrix_mask_indices(matrix, num_rows=None): idx = tf.minimum(idx, num_indices - 1) result = tf.gather(indices[:, 1], idx) # replace invalid elements with -1 - result = tf.where( - tf.expand_dims(elem_per_row, 1) > r, result, -array_ops.ones_like(result) - ) + result = tf.where(tf.expand_dims(elem_per_row, 1) > r, result, -array_ops.ones_like(result)) max_index_per_row = tf.reduce_max(result, axis=1, keepdims=True) max_index_per_row = tf.tile(max_index_per_row, [1, max_elem_per_row]) result = tf.where(result >= 0, result, max_index_per_row) diff --git a/easy_rec/python/core/sampler.py b/easy_rec/python/core/sampler.py index 62193b289..890857fbc 100644 --- a/easy_rec/python/core/sampler.py +++ b/easy_rec/python/core/sampler.py @@ -42,6 +42,7 @@ def string_attrs(self, string_attrs): # NOQA try: import graphlearn as gl from graphlearn.python.data.values import Values + Values.string_attrs = string_attrs except Exception: logging.info( @@ -59,7 +60,7 @@ def _get_gl_type(field_type): DatasetConfig.STRING: 'string', DatasetConfig.BOOL: 'int', DatasetConfig.FLOAT: 'float', - DatasetConfig.DOUBLE: 'float' + DatasetConfig.DOUBLE: 'float', } assert field_type in type_map, 'invalid type: %s' % field_type return type_map[field_type] @@ -72,7 +73,7 @@ def _get_np_type(field_type): DatasetConfig.STRING: str, DatasetConfig.BOOL: bool, DatasetConfig.FLOAT: np.float32, - DatasetConfig.DOUBLE: np.double + DatasetConfig.DOUBLE: np.double, } assert field_type in type_map, 'invalid type: %s' % field_type return type_map[field_type] @@ -91,9 +92,7 @@ def __init__(self, fields, num_sample, num_eval_sample=None): self._is_on_ds = ds_util.is_on_ds() def set_eval_num_sample(self): - print( - 'set_eval_num_sample: %d %d' % (self._num_sample, self._num_eval_sample) - ) + print('set_eval_num_sample: %d %d' % (self._num_sample, self._num_eval_sample)) self._num_sample = self._num_eval_sample def _init_graph(self): @@ -108,13 +107,10 @@ def _init_graph(self): task_count = 2 if self._is_on_ds: gl.set_tracker_mode(0) - server_hosts = [ - host.split(':')[0] + ':888' + str(i) - for i, host in enumerate(tf_config['cluster']['ps']) - ] + server_hosts = [host.split(':')[0] + ':888' + str(i) for i, host in enumerate(tf_config['cluster']['ps'])] cluster = { 'server': ','.join(server_hosts), - 'client_count': task_count + 'client_count': task_count, } else: ps_count = len(tf_config['cluster']['ps']) @@ -125,14 +121,14 @@ def _init_graph(self): self._g.init( cluster=cluster, job_name='client', - task_index=tf_config['task']['index'] + 2 + task_index=tf_config['task']['index'] + 2, ) # TODO(hongsheng.jhs): check cluster has evaluator or not? elif tf_config['task']['type'] == 'evaluator': self._g.init( cluster=cluster, job_name='client', - task_index=tf_config['task']['index'] + 1 + task_index=tf_config['task']['index'] + 1, ) if self._num_eval_sample is not None and self._num_eval_sample > 0: self._num_sample = self._num_eval_sample @@ -140,7 +136,7 @@ def _init_graph(self): self._g.init( cluster=cluster, job_name='server', - task_index=tf_config['task']['index'] + task_index=tf_config['task']['index'], ) else: # worker mode @@ -150,32 +146,30 @@ def _init_graph(self): self._g.init(task_index=0, task_count=task_count) elif tf_config['task']['type'] == 'worker': self._g.init( - task_index=tf_config['task']['index'] + 1, task_count=task_count + task_index=tf_config['task']['index'] + 1, + task_count=task_count, ) else: gl.set_tracker_mode(0) if tf_config['cluster'].get('chief', ''): - chief_host = tf_config['cluster']['chief'][0].split(':' - )[0] + ':8880' + chief_host = tf_config['cluster']['chief'][0].split(':')[0] + ':8880' else: - chief_host = tf_config['cluster']['master'][0].split(':' - )[0] + ':8880' + chief_host = tf_config['cluster']['master'][0].split(':')[0] + ':8880' worker_hosts = chief_host + [ - host.split(':')[0] + ':888' + str(i) - for i, host in enumerate(tf_config['cluster']['worker']) + host.split(':')[0] + ':888' + str(i) for i, host in enumerate(tf_config['cluster']['worker']) ] if tf_config['task']['type'] in ['chief', 'master']: self._g.init( task_index=0, task_count=task_count, - hosts=','.join(worker_hosts) + hosts=','.join(worker_hosts), ) elif tf_config['task']['type'] == 'worker': self._g.init( task_index=tf_config['task']['index'] + 1, task_count=task_count, - hosts=worker_hosts + hosts=worker_hosts, ) # TODO(hongsheng.jhs): check cluster has evaluator or not? @@ -210,17 +204,14 @@ def __del__(self): def _parse_nodes(self, nodes): if self._log_first_n > 0: logging.info( - 'num_example=%d num_eval_example=%d node_num=%d' % - (self._num_sample, self._num_eval_sample, len(nodes.ids)) + 'num_example=%d num_eval_example=%d node_num=%d' % (self._num_sample, self._num_eval_sample, len(nodes.ids)) ) self._log_first_n -= 1 features = [] int_idx = 0 float_idx = 0 string_idx = 0 - for attr_gl_type, attr_np_type in zip( - self._attr_gl_types, self._attr_np_types - ): + for attr_gl_type, attr_np_type in zip(self._attr_gl_types, self._attr_np_types): if attr_gl_type == 'int': feature = nodes.int_attrs[:, :, int_idx] int_idx += 1 @@ -234,8 +225,7 @@ def _parse_nodes(self, nodes): string_idx += 1 else: raise ValueError('Unknown attr type %s' % attr_gl_type) - feature = np.reshape(feature, - [-1])[:self._num_sample].astype(attr_np_type) + feature = np.reshape(feature, [-1])[: self._num_sample].astype(attr_np_type) if attr_gl_type == 'string': feature = feature.tolist() features.append(feature) @@ -246,9 +236,7 @@ def _parse_sparse_nodes(self, nodes): int_idx = 0 float_idx = 0 string_idx = 0 - for attr_gl_type, attr_np_type in zip( - self._attr_gl_types, self._attr_np_types - ): + for attr_gl_type, attr_np_type in zip(self._attr_gl_types, self._attr_np_types): if attr_gl_type == 'int': feature = nodes.int_attrs[:, int_idx] int_idx += 1 @@ -288,7 +276,7 @@ def __init__( num_sample, batch_size, attr_delimiter=':', - num_eval_sample=None + num_eval_sample=None, ): super(NegativeSampler, self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size @@ -298,15 +286,13 @@ def __init__( decoder=gl.Decoder( attr_types=self._attr_gl_types, weighted=True, - attr_delimiter=attr_delimiter - ) + attr_delimiter=attr_delimiter, + ), ) self._init_graph() expand_factor = int(math.ceil(self._num_sample / batch_size)) - self._sampler = self._g.negative_sampler( - 'item', expand_factor, strategy='node_weight' - ) + self._sampler = self._g.negative_sampler('item', expand_factor, strategy='node_weight') def _get_impl(self, ids): ids = np.array(ids, dtype=np.int64) @@ -353,16 +339,15 @@ def __init__( num_sample, batch_size, attr_delimiter=':', - num_eval_sample=None + num_eval_sample=None, ): - super(NegativeSamplerInMemory, - self).__init__(fields, num_sample, num_eval_sample) + super(NegativeSamplerInMemory, self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size self._item_ids = [] self._cols = [[] for x in fields] - if six.PY2 and isinstance(attr_delimiter, type(u'')): + if six.PY2 and isinstance(attr_delimiter, type('')): attr_delimiter = attr_delimiter.encode('utf-8') if data_path.startswith('odps://'): self._load_table(data_path, attr_delimiter) @@ -376,12 +361,11 @@ def __init__( if np_type != str: self._cols[col_id] = np.array(self._cols[col_id], dtype=np_type) else: - self._cols[col_id] = np.asarray( - self._cols[col_id], order='C', dtype=object - ) + self._cols[col_id] = np.asarray(self._cols[col_id], order='C', dtype=object) def _load_table(self, data_path, attr_delimiter): import common_io + reader = common_io.table.TableReader(data_path) schema = reader.get_schema() item_id_col = 0 @@ -394,10 +378,7 @@ def _load_table(self, data_path, attr_delimiter): if schema[tid][0].startswith('id'): item_id_col = tid break - print( - 'NegativeSamplerInMemory: feature_id_col = %d, item_id_col = %d' % - (fea_id_col, item_id_col) - ) + print('NegativeSamplerInMemory: feature_id_col = %d, item_id_col = %d' % (fea_id_col, item_id_col)) while True: try: row_arr = reader.read(num_records=1024, allow_smaller_final_batch=True) @@ -405,10 +386,11 @@ def _load_table(self, data_path, attr_delimiter): # item_id, weight, feature self._item_ids.append(int(row[item_id_col])) col_vals = row[fea_id_col].split(attr_delimiter) - assert len(col_vals) == len( - self._cols - ), 'invalid row[%d %d]: %s %s' % ( - len(col_vals), len(self._cols), row[item_id_col], row[fea_id_col] + assert len(col_vals) == len(self._cols), 'invalid row[%d %d]: %s %s' % ( + len(col_vals), + len(self._cols), + row[item_id_col], + row[fea_id_col], ) for col_id in range(len(col_vals)): self._cols[col_id].append(col_vals[col_id]) @@ -431,18 +413,17 @@ def _load_data(self, data_path, attr_delimiter): item_id_col = tid if schema[tid][0].startswith('feature'): fea_id_col = tid - print( - 'feature_id_col = %d, item_id_col = %d' % - (fea_id_col, item_id_col) - ) + print('feature_id_col = %d, item_id_col = %d' % (fea_id_col, item_id_col)) else: self._item_ids.append(int(cols[item_id_col])) fea_vals = cols[fea_id_col].split(attr_delimiter) - assert len(fea_vals) == len(self._cols - ), 'invalid row[%d][%d %d]:%s %s' % ( - line_id, len(fea_vals), len(self._cols), - cols[item_id_col], cols[fea_id_col] - ) + assert len(fea_vals) == len(self._cols), 'invalid row[%d][%d %d]:%s %s' % ( + line_id, + len(fea_vals), + len(self._cols), + cols[item_id_col], + cols[fea_id_col], + ) for col_id in range(len(fea_vals)): self._cols[col_id].append(fea_vals[col_id]) @@ -452,11 +433,7 @@ def _get_impl(self, ids): ids = [int(x) for x in ids] assert self._num_sample > 0, 'invalid num_sample: %d' % self._num_sample - indices = np.random.choice( - len(self._item_ids), - size=self._num_sample + self._batch_size, - replace=False - ) + indices = np.random.choice(len(self._item_ids), size=self._num_sample + self._batch_size, replace=False) sel_ids = [] for tid in indices: @@ -474,9 +451,7 @@ def _get_impl(self, ids): sel_feas = tmp_col[sel_ids] features.append(sel_feas) else: - features.append( - np.asarray([tmp_col[x] for x in sel_ids], order='C', dtype=object) - ) + features.append(np.asarray([tmp_col[x] for x in sel_ids], order='C', dtype=object)) return features def get(self, ids): @@ -523,30 +498,36 @@ def __init__( num_sample, batch_size, attr_delimiter=':', - num_eval_sample=None + num_eval_sample=None, ): - super(NegativeSamplerV2, - self).__init__(fields, num_sample, num_eval_sample) + super(NegativeSamplerV2, self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size - self._g = gl.Graph() \ - .node(tf.compat.as_str(user_data_path), - node_type='user', - decoder=gl.Decoder(weighted=True)) \ - .node(tf.compat.as_str(item_data_path), - node_type='item', - decoder=gl.Decoder( - attr_types=self._attr_gl_types, - weighted=True, - attr_delimiter=attr_delimiter)) \ - .edge(tf.compat.as_str(edge_data_path), - edge_type=('user', 'item', 'edge'), - decoder=gl.Decoder(weighted=True)) + self._g = ( + gl.Graph() + .node( + tf.compat.as_str(user_data_path), + node_type='user', + decoder=gl.Decoder(weighted=True), + ) + .node( + tf.compat.as_str(item_data_path), + node_type='item', + decoder=gl.Decoder( + attr_types=self._attr_gl_types, + weighted=True, + attr_delimiter=attr_delimiter, + ), + ) + .edge( + tf.compat.as_str(edge_data_path), + edge_type=('user', 'item', 'edge'), + decoder=gl.Decoder(weighted=True), + ) + ) self._init_graph() expand_factor = int(math.ceil(self._num_sample / batch_size)) - self._sampler = self._g.negative_sampler( - 'edge', expand_factor, strategy='random', conditional=True - ) + self._sampler = self._g.negative_sampler('edge', expand_factor, strategy='random', conditional=True) def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) @@ -567,9 +548,7 @@ def get(self, src_ids, dst_ids): Returns: Negative sampled feature dict. """ - sampled_values = tf.py_func( - self._get_impl, [src_ids, dst_ids], self._attr_tf_types - ) + sampled_values = tf.py_func(self._get_impl, [src_ids, dst_ids], self._attr_tf_types) result_dict = {} for k, t, v in zip(self._attr_names, self._attr_tf_types, sampled_values): v.set_shape([self._num_sample]) @@ -605,33 +584,37 @@ def __init__( num_hard_sample, batch_size, attr_delimiter=':', - num_eval_sample=None + num_eval_sample=None, ): - super(HardNegativeSampler, - self).__init__(fields, num_sample, num_eval_sample) + super(HardNegativeSampler, self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size - self._g = gl.Graph() \ - .node(tf.compat.as_str(user_data_path), - node_type='user', - decoder=gl.Decoder(weighted=True)) \ - .node(tf.compat.as_str(item_data_path), - node_type='item', - decoder=gl.Decoder( - attr_types=self._attr_gl_types, - weighted=True, - attr_delimiter=attr_delimiter)) \ - .edge(tf.compat.as_str(hard_neg_edge_data_path), - edge_type=('user', 'item', 'hard_neg_edge'), - decoder=gl.Decoder(weighted=True)) + self._g = ( + gl.Graph() + .node( + tf.compat.as_str(user_data_path), + node_type='user', + decoder=gl.Decoder(weighted=True), + ) + .node( + tf.compat.as_str(item_data_path), + node_type='item', + decoder=gl.Decoder( + attr_types=self._attr_gl_types, + weighted=True, + attr_delimiter=attr_delimiter, + ), + ) + .edge( + tf.compat.as_str(hard_neg_edge_data_path), + edge_type=('user', 'item', 'hard_neg_edge'), + decoder=gl.Decoder(weighted=True), + ) + ) self._init_graph() expand_factor = int(math.ceil(self._num_sample / batch_size)) - self._neg_sampler = self._g.negative_sampler( - 'item', expand_factor, strategy='node_weight' - ) - self._hard_neg_sampler = self._g.neighbor_sampler( - ['hard_neg_edge'], num_hard_sample, strategy='full' - ) + self._neg_sampler = self._g.negative_sampler('item', expand_factor, strategy='node_weight') + self._hard_neg_sampler = self._g.neighbor_sampler(['hard_neg_edge'], num_hard_sample, strategy='full') def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) @@ -640,16 +623,12 @@ def _get_impl(self, src_ids, dst_ids): nodes = self._neg_sampler.get(dst_ids) neg_features = self._parse_nodes(nodes) sparse_nodes = self._hard_neg_sampler.get(src_ids).layer_nodes(1) - hard_neg_features, hard_neg_indices = self._parse_sparse_nodes( - sparse_nodes - ) + hard_neg_features, hard_neg_indices = self._parse_sparse_nodes(sparse_nodes) results = [] for i, v in enumerate(hard_neg_features): if type(v) == list: - results.append( - np.asarray(neg_features[i] + v, order='C', dtype=object) - ) + results.append(np.asarray(neg_features[i] + v, order='C', dtype=object)) else: results.append(np.concatenate([neg_features[i], v], axis=0)) results.append(hard_neg_indices) @@ -666,13 +645,9 @@ def get(self, src_ids, dst_ids): Sampled feature dict. The first batch_size is negative samples, remainder is hard negative samples """ output_types = self._attr_tf_types + [tf.int64] - output_values = tf.py_func( - self._get_impl, [src_ids, dst_ids], output_types - ) + output_values = tf.py_func(self._get_impl, [src_ids, dst_ids], output_types) result_dict = {} - for k, t, v in zip( - self._attr_names, self._attr_tf_types, output_values[:-1] - ): + for k, t, v in zip(self._attr_names, self._attr_tf_types, output_values[:-1]): v.set_shape([None]) result_dict[k] = v @@ -712,57 +687,57 @@ def __init__( num_hard_sample, batch_size, attr_delimiter=':', - num_eval_sample=None + num_eval_sample=None, ): - super(HardNegativeSamplerV2, - self).__init__(fields, num_sample, num_eval_sample) + super(HardNegativeSamplerV2, self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size - self._g = gl.Graph() \ - .node(tf.compat.as_str(user_data_path), - node_type='user', - decoder=gl.Decoder(weighted=True)) \ - .node(tf.compat.as_str(item_data_path), - node_type='item', - decoder=gl.Decoder( - attr_types=self._attr_gl_types, - weighted=True, - attr_delimiter=attr_delimiter)) \ - .edge(tf.compat.as_str(edge_data_path), - edge_type=('user', 'item', 'edge'), - decoder=gl.Decoder(weighted=True)) \ - .edge(tf.compat.as_str(hard_neg_edge_data_path), - edge_type=('user', 'item', 'hard_neg_edge'), - decoder=gl.Decoder(weighted=True)) + self._g = ( + gl.Graph() + .node( + tf.compat.as_str(user_data_path), + node_type='user', + decoder=gl.Decoder(weighted=True), + ) + .node( + tf.compat.as_str(item_data_path), + node_type='item', + decoder=gl.Decoder( + attr_types=self._attr_gl_types, + weighted=True, + attr_delimiter=attr_delimiter, + ), + ) + .edge( + tf.compat.as_str(edge_data_path), + edge_type=('user', 'item', 'edge'), + decoder=gl.Decoder(weighted=True), + ) + .edge( + tf.compat.as_str(hard_neg_edge_data_path), + edge_type=('user', 'item', 'hard_neg_edge'), + decoder=gl.Decoder(weighted=True), + ) + ) self._init_graph() expand_factor = int(math.ceil(self._num_sample / batch_size)) - self._neg_sampler = self._g.negative_sampler( - 'edge', expand_factor, strategy='random', conditional=True - ) - self._hard_neg_sampler = self._g.neighbor_sampler( - ['hard_neg_edge'], num_hard_sample, strategy='full' - ) + self._neg_sampler = self._g.negative_sampler('edge', expand_factor, strategy='random', conditional=True) + self._hard_neg_sampler = self._g.neighbor_sampler(['hard_neg_edge'], num_hard_sample, strategy='full') def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) - src_ids_padded = np.pad( - src_ids, (0, self._batch_size - len(src_ids)), 'edge' - ) + src_ids_padded = np.pad(src_ids, (0, self._batch_size - len(src_ids)), 'edge') dst_ids = np.array(dst_ids, dtype=np.int64) dst_ids = np.pad(dst_ids, (0, self._batch_size - len(dst_ids)), 'edge') nodes = self._neg_sampler.get(src_ids_padded, dst_ids) neg_features = self._parse_nodes(nodes) sparse_nodes = self._hard_neg_sampler.get(src_ids).layer_nodes(1) - hard_neg_features, hard_neg_indices = self._parse_sparse_nodes( - sparse_nodes - ) + hard_neg_features, hard_neg_indices = self._parse_sparse_nodes(sparse_nodes) results = [] for i, v in enumerate(hard_neg_features): if type(v) == list: - results.append( - np.asarray(neg_features[i] + v, order='C', dtype=object) - ) + results.append(np.asarray(neg_features[i] + v, order='C', dtype=object)) else: results.append(np.concatenate([neg_features[i], v], axis=0)) results.append(hard_neg_indices) @@ -779,13 +754,9 @@ def get(self, src_ids, dst_ids): Sampled feature dict. The first batch_size is negative samples, remainder is hard negative samples """ output_types = self._attr_tf_types + [tf.int64] - output_values = tf.py_func( - self._get_impl, [src_ids, dst_ids], output_types - ) + output_values = tf.py_func(self._get_impl, [src_ids, dst_ids], output_types) result_dict = {} - for k, t, v in zip( - self._attr_names, self._attr_tf_types, output_values[:-1] - ): + for k, t, v in zip(self._attr_names, self._attr_tf_types, output_values[:-1]): v.set_shape([None]) result_dict[k] = v @@ -796,7 +767,6 @@ def get(self, src_ids, dst_ids): def build(data_config): - if not data_config.HasField('sampler'): return None sampler_type = data_config.WhichOneof('sampler') @@ -817,7 +787,7 @@ def build(data_config): num_sample=sampler_config.num_sample, batch_size=data_config.batch_size, attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample + num_eval_sample=sampler_config.num_eval_sample, ) elif sampler_type == 'negative_sampler_in_memory': input_fields = {f.input_name: f for f in data_config.input_fields} @@ -830,21 +800,15 @@ def build(data_config): num_sample=sampler_config.num_sample, batch_size=data_config.batch_size, attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample + num_eval_sample=sampler_config.num_eval_sample, ) elif sampler_type == 'negative_sampler_v2': input_fields = {f.input_name: f for f in data_config.input_fields} attr_fields = [input_fields[name] for name in sampler_config.attr_fields] - user_input_path = process_multi_file_input_path( - sampler_config.user_input_path - ) - item_input_path = process_multi_file_input_path( - sampler_config.item_input_path - ) - pos_edge_input_path = process_multi_file_input_path( - sampler_config.pos_edge_input_path - ) + user_input_path = process_multi_file_input_path(sampler_config.user_input_path) + item_input_path = process_multi_file_input_path(sampler_config.item_input_path) + pos_edge_input_path = process_multi_file_input_path(sampler_config.pos_edge_input_path) return NegativeSamplerV2.instance( user_data_path=user_input_path, item_data_path=item_input_path, @@ -853,21 +817,15 @@ def build(data_config): num_sample=sampler_config.num_sample, batch_size=data_config.batch_size, attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample + num_eval_sample=sampler_config.num_eval_sample, ) elif sampler_type == 'hard_negative_sampler': input_fields = {f.input_name: f for f in data_config.input_fields} attr_fields = [input_fields[name] for name in sampler_config.attr_fields] - user_input_path = process_multi_file_input_path( - sampler_config.user_input_path - ) - item_input_path = process_multi_file_input_path( - sampler_config.item_input_path - ) - hard_neg_edge_input_path = process_multi_file_input_path( - sampler_config.hard_neg_edge_input_path - ) + user_input_path = process_multi_file_input_path(sampler_config.user_input_path) + item_input_path = process_multi_file_input_path(sampler_config.item_input_path) + hard_neg_edge_input_path = process_multi_file_input_path(sampler_config.hard_neg_edge_input_path) return HardNegativeSampler.instance( user_data_path=user_input_path, item_data_path=item_input_path, @@ -877,24 +835,16 @@ def build(data_config): num_hard_sample=sampler_config.num_hard_sample, batch_size=data_config.batch_size, attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample + num_eval_sample=sampler_config.num_eval_sample, ) elif sampler_type == 'hard_negative_sampler_v2': input_fields = {f.input_name: f for f in data_config.input_fields} attr_fields = [input_fields[name] for name in sampler_config.attr_fields] - user_input_path = process_multi_file_input_path( - sampler_config.user_input_path - ) - item_input_path = process_multi_file_input_path( - sampler_config.item_input_path - ) - pos_edge_input_path = process_multi_file_input_path( - sampler_config.pos_edge_input_path - ) - hard_neg_edge_input_path = process_multi_file_input_path( - sampler_config.hard_neg_edge_input_path - ) + user_input_path = process_multi_file_input_path(sampler_config.user_input_path) + item_input_path = process_multi_file_input_path(sampler_config.item_input_path) + pos_edge_input_path = process_multi_file_input_path(sampler_config.pos_edge_input_path) + hard_neg_edge_input_path = process_multi_file_input_path(sampler_config.hard_neg_edge_input_path) return HardNegativeSamplerV2.instance( user_data_path=user_input_path, item_data_path=item_input_path, @@ -905,7 +855,7 @@ def build(data_config): num_hard_sample=sampler_config.num_hard_sample, batch_size=data_config.batch_size, attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample + num_eval_sample=sampler_config.num_eval_sample, ) else: raise ValueError('Unknown sampler %s' % sampler_type) diff --git a/easy_rec/python/eval.py b/easy_rec/python/eval.py index 491df0f48..c7c1124dd 100644 --- a/easy_rec/python/eval.py +++ b/easy_rec/python/eval.py @@ -11,39 +11,33 @@ from easy_rec.python.main import distribute_evaluate, evaluate from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import config_util, ds_util, estimator_utils - -from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_distribute_eval_worker_num_on_ds # NOQA +from easy_rec.python.utils.distribution_utils import ( # NOQA + set_tf_config_and_get_distribute_eval_worker_num_on_ds, +) if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO + level=logging.INFO, ) +tf.app.flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') tf.app.flags.DEFINE_string( - 'pipeline_config_path', None, 'Path to pipeline config ' - 'file.' -) -tf.app.flags.DEFINE_string( - 'checkpoint_path', None, 'checkpoint to be evaled ' - ' if not specified, use the latest checkpoint in ' - 'train_config.model_dir' + 'checkpoint_path', + None, + 'checkpoint to be evaled ' ' if not specified, use the latest checkpoint in ' 'train_config.model_dir', ) tf.app.flags.DEFINE_multi_string( - 'eval_input_path', None, 'eval data path, if specified will ' - 'override pipeline_config.eval_input_path' + 'eval_input_path', + None, + 'eval data path, if specified will ' 'override pipeline_config.eval_input_path', ) tf.app.flags.DEFINE_string('model_dir', None, help='will update the model_dir') tf.app.flags.DEFINE_string('odps_config', None, help='odps config path') -tf.app.flags.DEFINE_string( - 'eval_result_path', 'eval_result.txt', 'eval result metric file' -) -tf.app.flags.DEFINE_bool( - 'distribute_eval', False, - 'use distribute parameter server for train and eval.' -) +tf.app.flags.DEFINE_string('eval_result_path', 'eval_result.txt', 'eval result metric file') +tf.app.flags.DEFINE_bool('distribute_eval', False, 'use distribute parameter server for train and eval.') tf.app.flags.DEFINE_bool('is_on_ds', False, help='is on ds') FLAGS = tf.app.flags.FLAGS @@ -67,9 +61,7 @@ def main(argv): else: pipeline_config_path = FLAGS.pipeline_config_path - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path - ) + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) if FLAGS.model_dir: pipeline_config.model_dir = FLAGS.model_dir @@ -79,7 +71,7 @@ def main(argv): estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.SokStrategy, ]: estimator_utils.init_hvd() estimator_utils.init_sok() @@ -87,14 +79,18 @@ def main(argv): if FLAGS.distribute_eval: os.environ['distribute_eval'] = 'True' eval_result = distribute_evaluate( - pipeline_config, FLAGS.checkpoint_path, FLAGS.eval_input_path, - FLAGS.eval_result_path + pipeline_config, + FLAGS.checkpoint_path, + FLAGS.eval_input_path, + FLAGS.eval_result_path, ) else: os.environ['distribute_eval'] = 'False' eval_result = evaluate( - pipeline_config, FLAGS.checkpoint_path, FLAGS.eval_input_path, - FLAGS.eval_result_path + pipeline_config, + FLAGS.checkpoint_path, + FLAGS.eval_input_path, + FLAGS.eval_result_path, ) if eval_result is not None: # when distribute evaluate, only master has eval_result. diff --git a/easy_rec/python/export.py b/easy_rec/python/export.py index 72726af34..5df4a6f19 100644 --- a/easy_rec/python/export.py +++ b/easy_rec/python/export.py @@ -20,55 +20,30 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO -) -tf.app.flags.DEFINE_string( - 'pipeline_config_path', None, 'Path to pipeline config ' - 'file.' + level=logging.INFO, ) +tf.app.flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') tf.app.flags.DEFINE_string('checkpoint_path', '', 'checkpoint to be exported') -tf.app.flags.DEFINE_string( - 'export_dir', None, 'directory where model should be exported to' -) +tf.app.flags.DEFINE_string('export_dir', None, 'directory where model should be exported to') tf.app.flags.DEFINE_string('redis_url', None, 'export to redis url, host:port') tf.app.flags.DEFINE_string('redis_passwd', None, 'export to redis passwd') tf.app.flags.DEFINE_integer('redis_threads', 0, 'export to redis threads') -tf.app.flags.DEFINE_integer( - 'redis_batch_size', 256, 'export to redis batch_size' -) -tf.app.flags.DEFINE_integer( - 'redis_timeout', 600, 'export to redis time_out in seconds' -) -tf.app.flags.DEFINE_integer( - 'redis_expire', 24, 'export to redis expire time in hour' -) -tf.app.flags.DEFINE_string( - 'redis_embedding_version', '', 'redis embedding version' -) -tf.app.flags.DEFINE_integer( - 'redis_write_kv', 1, 'whether to write embedding to redis' -) +tf.app.flags.DEFINE_integer('redis_batch_size', 256, 'export to redis batch_size') +tf.app.flags.DEFINE_integer('redis_timeout', 600, 'export to redis time_out in seconds') +tf.app.flags.DEFINE_integer('redis_expire', 24, 'export to redis expire time in hour') +tf.app.flags.DEFINE_string('redis_embedding_version', '', 'redis embedding version') +tf.app.flags.DEFINE_integer('redis_write_kv', 1, 'whether to write embedding to redis') -tf.app.flags.DEFINE_string( - 'oss_path', None, 'write embed objects to oss folder, oss://bucket/folder' -) +tf.app.flags.DEFINE_string('oss_path', None, 'write embed objects to oss folder, oss://bucket/folder') tf.app.flags.DEFINE_string('oss_endpoint', None, 'oss endpoint') tf.app.flags.DEFINE_string('oss_ak', None, 'oss ak') tf.app.flags.DEFINE_string('oss_sk', None, 'oss sk') -tf.app.flags.DEFINE_integer( - 'oss_threads', 10, '# threads access oss at the same time' -) -tf.app.flags.DEFINE_integer( - 'oss_timeout', 10, 'connect to oss, time_out in seconds' -) +tf.app.flags.DEFINE_integer('oss_threads', 10, '# threads access oss at the same time') +tf.app.flags.DEFINE_integer('oss_timeout', 10, 'connect to oss, time_out in seconds') tf.app.flags.DEFINE_integer('oss_expire', 24, 'oss expire time in hours') -tf.app.flags.DEFINE_integer( - 'oss_write_kv', 1, 'whether to write embedding to oss' -) -tf.app.flags.DEFINE_string( - 'oss_embedding_version', '', 'oss embedding version' -) +tf.app.flags.DEFINE_integer('oss_write_kv', 1, 'whether to write embedding to oss') +tf.app.flags.DEFINE_string('oss_embedding_version', '', 'oss embedding version') tf.app.flags.DEFINE_string('asset_files', '', 'more files to add to asset') tf.app.flags.DEFINE_bool('verbose', False, 'print more debug information') @@ -77,9 +52,7 @@ tf.app.flags.mark_flag_as_required('export_dir') tf.app.flags.DEFINE_bool('clear_export', False, 'remove export_dir if exists') -tf.app.flags.DEFINE_string( - 'export_done_file', '', 'a flag file to signal that export model is done' -) +tf.app.flags.DEFINE_string('export_done_file', '', 'a flag file to signal that export model is done') FLAGS = tf.app.flags.FLAGS @@ -131,16 +104,14 @@ def main(argv): if FLAGS.oss_embedding_version: extra_params['oss_embedding_version'] = FLAGS.oss_embedding_version - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path - ) + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) if pipeline_config.train_config.train_distribute in [ DistributionStrategy.HorovodStrategy, ]: estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.SokStrategy, ]: estimator_utils.init_hvd() estimator_utils.init_sok() @@ -151,8 +122,12 @@ def main(argv): gfile.DeleteRecursively(FLAGS.export_dir) export_out_dir = export( - FLAGS.export_dir, pipeline_config_path, FLAGS.checkpoint_path, - FLAGS.asset_files, FLAGS.verbose, **extra_params + FLAGS.export_dir, + pipeline_config_path, + FLAGS.checkpoint_path, + FLAGS.asset_files, + FLAGS.verbose, + **extra_params, ) if FLAGS.export_done_file: diff --git a/easy_rec/python/feature_column/feature_column.py b/easy_rec/python/feature_column/feature_column.py index bea5f20d3..2ca60da1d 100644 --- a/easy_rec/python/feature_column/feature_column.py +++ b/easy_rec/python/feature_column/feature_column.py @@ -9,23 +9,22 @@ from tensorflow.python.platform import gfile from easy_rec.python.builders import hyperparams_builder -from easy_rec.python.compat.feature_column import sequence_feature_column +from easy_rec.python.compat.feature_column import feature_column_v2 as feature_column # NOQA +from easy_rec.python.compat.feature_column import ( + sequence_feature_column, +) from easy_rec.python.protos.feature_config_pb2 import FeatureConfig, WideOrDeep from easy_rec.python.utils.proto_util import copy_obj -from easy_rec.python.compat.feature_column import feature_column_v2 as feature_column # NOQA - MAX_HASH_BUCKET_SIZE = 9223372036854775807 class FeatureKeyError(KeyError): - def __init__(self, feature_name): super(FeatureKeyError, self).__init__(feature_name) class SharedEmbedding(object): - def __init__(self, embedding_name, index, sequence_combiner=None): self.embedding_name = embedding_name self.index = index @@ -33,23 +32,15 @@ def __init__(self, embedding_name, index, sequence_combiner=None): EVParams = collections.namedtuple( - 'EVParams', [ - 'filter_freq', 'steps_to_live', 'use_cache', 'init_capacity', - 'max_capacity' - ] + 'EVParams', + ['filter_freq', 'steps_to_live', 'use_cache', 'init_capacity', 'max_capacity'], ) class FeatureColumnParser(object): """Parse and generate feature columns.""" - def __init__( - self, - feature_configs, - wide_deep_dict={}, - wide_output_dim=-1, - ev_params=None - ): + def __init__(self, feature_configs, wide_deep_dict={}, wide_output_dim=-1, ev_params=None): """Initializes a `FeatureColumnParser`. Args: @@ -79,9 +70,13 @@ def __init__( self._global_ev_params = self._build_ev_params(ev_params) def _cmp_embed_config(a, b): - return a.embedding_dim == b.embedding_dim and a.combiner == b.combiner and\ - a.initializer == b.initializer and a.max_partitions == b.max_partitions and\ - a.embedding_name == b.embedding_name + return ( + a.embedding_dim == b.embedding_dim + and a.combiner == b.combiner + and a.initializer == b.initializer + and a.max_partitions == b.max_partitions + and a.embedding_name == b.embedding_name + ) for config in self._feature_configs: if not config.HasField('embedding_name'): @@ -89,9 +84,14 @@ def _cmp_embed_config(a, b): embed_name = config.embedding_name if embed_name in self._share_embed_names: - assert _cmp_embed_config(config, self._share_embed_infos[embed_name]),\ - 'shared embed info of [%s] is not matched [%s] vs [%s]' % ( - embed_name, config, self._share_embed_infos[embed_name]) + assert _cmp_embed_config(config, self._share_embed_infos[embed_name]), ( + 'shared embed info of [%s] is not matched [%s] vs [%s]' + % ( + embed_name, + config, + self._share_embed_infos[embed_name], + ) + ) self._share_embed_names[embed_name] += 1 if config.feature_type == FeatureConfig.FeatureType.SequenceFeature: self._share_embed_infos[embed_name] = copy_obj(config) @@ -100,9 +100,7 @@ def _cmp_embed_config(a, b): self._share_embed_infos[embed_name] = copy_obj(config) # remove not shared embedding names - not_shared = [ - x for x in self._share_embed_names if self._share_embed_names[x] == 1 - ] + not_shared = [x for x in self._share_embed_names if self._share_embed_names[x] == 1] for embed_name in not_shared: del self._share_embed_names[embed_name] del self._share_embed_infos[embed_name] @@ -110,19 +108,15 @@ def _cmp_embed_config(a, b): logging.info('shared embeddings[num=%d]' % len(self._share_embed_names)) for embed_name in self._share_embed_names: logging.info( - '\t%s: share_num[%d], share_info[%s]' % ( - embed_name, self._share_embed_names[embed_name], - self._share_embed_infos[embed_name] + '\t%s: share_num[%d], share_info[%s]' + % ( + embed_name, + self._share_embed_names[embed_name], + self._share_embed_infos[embed_name], ) ) - self._deep_share_embed_columns = { - embed_name: [] - for embed_name in self._share_embed_names - } - self._wide_share_embed_columns = { - embed_name: [] - for embed_name in self._share_embed_names - } + self._deep_share_embed_columns = {embed_name: [] for embed_name in self._share_embed_names} + self._wide_share_embed_columns = {embed_name: [] for embed_name in self._share_embed_names} self._feature_vocab_size = {} for config in self._feature_configs: @@ -150,18 +144,12 @@ def _cmp_embed_config(a, b): for embed_name in self._share_embed_names: initializer = None if self._share_embed_infos[embed_name].HasField('initializer'): - initializer = hyperparams_builder.build_initializer( - self._share_embed_infos[embed_name].initializer - ) + initializer = hyperparams_builder.build_initializer(self._share_embed_infos[embed_name].initializer) - partitioner = self._build_partitioner( - self._share_embed_infos[embed_name] - ) + partitioner = self._build_partitioner(self._share_embed_infos[embed_name]) if self._share_embed_infos[embed_name].HasField('ev_params'): - ev_params = self._build_ev_params( - self._share_embed_infos[embed_name].ev_params - ) + ev_params = self._build_ev_params(self._share_embed_infos[embed_name].ev_params) else: ev_params = self._global_ev_params @@ -174,12 +162,10 @@ def _cmp_embed_config(a, b): shared_embedding_collection_name=embed_name, combiner=self._share_embed_infos[embed_name].combiner, partitioner=partitioner, - ev_params=ev_params + ev_params=ev_params, ) config = self._share_embed_infos[embed_name] - max_seq_len = config.max_seq_len if config.HasField( - 'max_seq_len' - ) else -1 + max_seq_len = config.max_seq_len if config.HasField('max_seq_len') else -1 for fc in share_embed_fcs: fc.max_seq_length = max_seq_len self._deep_share_embed_columns[embed_name] = share_embed_fcs @@ -193,12 +179,10 @@ def _cmp_embed_config(a, b): shared_embedding_collection_name=embed_name + '_wide', combiner='sum', partitioner=partitioner, - ev_params=ev_params + ev_params=ev_params, ) config = self._share_embed_infos[embed_name] - max_seq_len = config.max_seq_len if config.HasField( - 'max_seq_len' - ) else -1 + max_seq_len = config.max_seq_len if config.HasField('max_seq_len') else -1 for fc in share_embed_fcs: fc.max_seq_length = max_seq_len self._wide_share_embed_columns[embed_name] = share_embed_fcs @@ -211,9 +195,7 @@ def _cmp_embed_config(a, b): for fc_name in self._wide_columns: fc = self._wide_columns[fc_name] if isinstance(fc, SharedEmbedding): - self._wide_columns[fc_name] = self._get_shared_embedding_column( - fc, deep=False - ) + self._wide_columns[fc_name] = self._get_shared_embedding_column(fc, deep=False) for fc_name in self._sequence_columns: fc = self._sequence_columns[fc_name] @@ -240,7 +222,8 @@ def is_wide(self, config): if feature_name not in self._wide_deep_dict: raise FeatureKeyError(feature_name) return self._wide_deep_dict[feature_name] in [ - WideOrDeep.WIDE, WideOrDeep.WIDE_AND_DEEP + WideOrDeep.WIDE, + WideOrDeep.WIDE_AND_DEEP, ] def is_deep(self, config): @@ -252,7 +235,8 @@ def is_deep(self, config): if feature_name not in self._wide_deep_dict: raise FeatureKeyError(feature_name) return self._wide_deep_dict[feature_name] in [ - WideOrDeep.DEEP, WideOrDeep.WIDE_AND_DEEP + WideOrDeep.DEEP, + WideOrDeep.WIDE_AND_DEEP, ] def get_feature_vocab_size(self, feature): @@ -284,21 +268,20 @@ def parse_id_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] hash_bucket_size = self._get_hash_bucket_size(config) if hash_bucket_size > 0: fc = feature_column.categorical_column_with_hash_bucket( feature_name, hash_bucket_size=hash_bucket_size, - feature_name=feature_name + feature_name=feature_name, ) elif config.vocab_list: fc = feature_column.categorical_column_with_vocabulary_list( feature_name, default_value=0, vocabulary_list=config.vocab_list, - feature_name=feature_name + feature_name=feature_name, ) elif config.vocab_file: fc = feature_column.categorical_column_with_vocabulary_file( @@ -306,7 +289,7 @@ def parse_id_feature(self, config): default_value=0, vocabulary_file=config.vocab_file, vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name + feature_name=feature_name, ) else: use_ev = self._global_ev_params or config.HasField('ev_params') @@ -330,22 +313,21 @@ def parse_tag_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] hash_bucket_size = self._get_hash_bucket_size(config) if hash_bucket_size > 0: tag_fc = feature_column.categorical_column_with_hash_bucket( feature_name, hash_bucket_size, dtype=tf.string, - feature_name=feature_name + feature_name=feature_name, ) elif config.vocab_list: tag_fc = feature_column.categorical_column_with_vocabulary_list( feature_name, default_value=0, vocabulary_list=config.vocab_list, - feature_name=feature_name + feature_name=feature_name, ) elif config.vocab_file: tag_fc = feature_column.categorical_column_with_vocabulary_file( @@ -353,7 +335,7 @@ def parse_tag_feature(self, config): default_value=0, vocabulary_file=config.vocab_file, vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name + feature_name=feature_name, ) else: use_ev = self._global_ev_params or config.HasField('ev_params') @@ -384,13 +366,8 @@ def parse_raw_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] - fc = feature_column.numeric_column( - key=feature_name, - shape=(config.raw_input_dim, ), - feature_name=feature_name - ) + feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] + fc = feature_column.numeric_column(key=feature_name, shape=(config.raw_input_dim,), feature_name=feature_name) bounds = None if config.boundaries: @@ -398,21 +375,14 @@ def parse_raw_feature(self, config): bounds.sort() elif config.num_buckets > 1 and config.max_val > config.min_val: # the feature values are already normalized into [0, 1] - bounds = [ - x / float(config.num_buckets) for x in range(0, config.num_buckets) - ] - logging.info( - 'discrete %s into %d buckets' % (feature_name, config.num_buckets) - ) + bounds = [x / float(config.num_buckets) for x in range(0, config.num_buckets)] + logging.info('discrete %s into %d buckets' % (feature_name, config.num_buckets)) if bounds: try: fc = feature_column.bucketized_column(fc, bounds) except Exception as e: - logging.error( - 'bucketized_column [%s] with bounds %s error' % - (fc.name, str(bounds)) - ) + logging.error('bucketized_column [%s] with bounds %s error' % (fc.name, str(bounds))) raise e if self.is_wide(config): self._add_wide_embedding_column(fc, config) @@ -423,12 +393,12 @@ def parse_raw_feature(self, config): feature_name + '_raw_proj_id', config.raw_input_dim, default_value=0, - feature_name=feature_name + feature_name=feature_name, ) wgt_fc = feature_column.weighted_categorical_column( tmp_id_col, weight_feature_key=feature_name + '_raw_proj_val', - dtype=tf.float32 + dtype=tf.float32, ) if self.is_wide(config): self._add_wide_embedding_column(wgt_fc, config) @@ -446,11 +416,8 @@ def parse_expr_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] - fc = feature_column.numeric_column( - feature_name, shape=(1, ), feature_name=feature_name - ) + feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] + fc = feature_column.numeric_column(feature_name, shape=(1,), feature_name=feature_name) if self.is_wide(config): self._add_wide_embedding_column(fc, config) if self.is_deep(config): @@ -462,8 +429,7 @@ def parse_combo_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else None + feature_name = config.feature_name if config.HasField('feature_name') else None assert len(config.input_names) >= 2 if len(config.combo_join_sep) == 0: @@ -477,13 +443,13 @@ def parse_combo_feature(self, config): input_names, self._get_hash_bucket_size(config), hash_key=None, - feature_name=feature_name + feature_name=feature_name, ) else: fc = feature_column.categorical_column_with_hash_bucket( feature_name, hash_bucket_size=self._get_hash_bucket_size(config), - feature_name=feature_name + feature_name=feature_name, ) if self.is_wide(config): @@ -497,15 +463,11 @@ def parse_lookup_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] assert config.HasField('hash_bucket_size') hash_bucket_size = self._get_hash_bucket_size(config) fc = feature_column.categorical_column_with_hash_bucket( - feature_name, - hash_bucket_size, - dtype=tf.string, - feature_name=feature_name + feature_name, hash_bucket_size, dtype=tf.string, feature_name=feature_name ) if self.is_wide(config): @@ -519,11 +481,12 @@ def parse_sequence_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] sub_feature_type = config.sub_feature_type - assert sub_feature_type in [config.IdFeature, config.RawFeature], \ - 'Current sub_feature_type only support IdFeature and RawFeature.' + assert sub_feature_type in [ + config.IdFeature, + config.RawFeature, + ], 'Current sub_feature_type only support IdFeature and RawFeature.' if sub_feature_type == config.IdFeature: if config.HasField('hash_bucket_size'): hash_bucket_size = self._get_hash_bucket_size(config) @@ -531,14 +494,14 @@ def parse_sequence_feature(self, config): feature_name, hash_bucket_size, dtype=tf.string, - feature_name=feature_name + feature_name=feature_name, ) elif config.vocab_list: fc = sequence_feature_column.sequence_categorical_column_with_vocabulary_list( feature_name, default_value=0, vocabulary_list=config.vocab_list, - feature_name=feature_name + feature_name=feature_name, ) elif config.vocab_file: fc = sequence_feature_column.sequence_categorical_column_with_vocabulary_file( @@ -546,7 +509,7 @@ def parse_sequence_feature(self, config): default_value=0, vocabulary_file=config.vocab_file, vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name + feature_name=feature_name, ) else: use_ev = self._global_ev_params or config.HasField('ev_params') @@ -555,39 +518,28 @@ def parse_sequence_feature(self, config): feature_name, num_buckets, default_value=0, - feature_name=feature_name + feature_name=feature_name, ) else: # raw feature bounds = None - fc = sequence_feature_column.sequence_numeric_column( - feature_name, shape=(1, ), feature_name=feature_name - ) + fc = sequence_feature_column.sequence_numeric_column(feature_name, shape=(1,), feature_name=feature_name) if config.hash_bucket_size > 0: hash_bucket_size = self._get_hash_bucket_size(config) - assert sub_feature_type == config.IdFeature, \ - 'You should set sub_feature_type to IdFeature to use hash_bucket_size.' + assert ( + sub_feature_type == config.IdFeature + ), 'You should set sub_feature_type to IdFeature to use hash_bucket_size.' elif config.boundaries: bounds = list(config.boundaries) bounds.sort() elif config.num_buckets > 1 and config.max_val > config.min_val: # the feature values are already normalized into [0, 1] - bounds = [ - x / float(config.num_buckets) for x in range(0, config.num_buckets) - ] - logging.info( - 'sequence feature discrete %s into %d buckets' % - (feature_name, config.num_buckets) - ) + bounds = [x / float(config.num_buckets) for x in range(0, config.num_buckets)] + logging.info('sequence feature discrete %s into %d buckets' % (feature_name, config.num_buckets)) if bounds: try: - fc = sequence_feature_column.sequence_numeric_column_with_bucketized_column( - fc, bounds - ) + fc = sequence_feature_column.sequence_numeric_column_with_bucketized_column(fc, bounds) except Exception as e: - logging.error( - 'sequence features bucketized_column [%s] with bounds %s error' % - (feature_name, str(bounds)) - ) + logging.error('sequence features bucketized_column [%s] with bounds %s error' % (feature_name, str(bounds))) raise e elif config.hash_bucket_size <= 0: if config.embedding_dim > 0: @@ -595,18 +547,16 @@ def parse_sequence_feature(self, config): feature_name + '_raw_proj_id', config.raw_input_dim, default_value=0, - feature_name=feature_name + feature_name=feature_name, ) wgt_fc = sequence_feature_column.sequence_weighted_categorical_column( tmp_id_col, weight_feature_key=feature_name + '_raw_proj_val', - dtype=tf.float32 + dtype=tf.float32, ) fc = wgt_fc else: - fc = sequence_feature_column.sequence_numeric_column_with_raw_column( - fc, config.sequence_length - ) + fc = sequence_feature_column.sequence_numeric_column_with_raw_column(fc, config.sequence_length) if config.embedding_dim > 0: self._add_deep_embedding_column(fc, config) @@ -617,13 +567,9 @@ def _build_partitioner(self, config): if config.max_partitions > 1: if self._global_ev_params is not None or config.HasField('ev_params'): # pai embedding_variable should use fixed_size_partitioner - return partitioned_variables.fixed_size_partitioner( - num_shards=config.max_partitions - ) + return partitioned_variables.fixed_size_partitioner(num_shards=config.max_partitions) else: - return partitioned_variables.min_max_variable_partitioner( - max_partitions=config.max_partitions - ) + return partitioned_variables.min_max_variable_partitioner(max_partitions=config.max_partitions) else: return None @@ -651,13 +597,10 @@ def _add_wide_embedding_column(self, fc, config): We use embedding to simulate wide column, which is more efficient than indicator column for sparse features """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] assert self._wide_output_dim > 0, 'wide_output_dim is not set' if config.embedding_name in self._wide_share_embed_columns: - wide_fc = self._add_shared_embedding_column( - config.embedding_name, fc, deep=False - ) + wide_fc = self._add_shared_embedding_column(config.embedding_name, fc, deep=False) else: initializer = None if config.HasField('initializer'): @@ -672,14 +615,13 @@ def _add_wide_embedding_column(self, fc, config): combiner='sum', initializer=initializer, partitioner=self._build_partitioner(config), - ev_params=ev_params + ev_params=ev_params, ) self._wide_columns[feature_name] = wide_fc def _add_deep_embedding_column(self, fc, config): """Generate deep feature columns.""" - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] assert config.embedding_dim > 0, 'embedding_dim is not set for %s' % feature_name self._feature_vocab_size[feature_name] = fc.num_buckets if config.embedding_name in self._deep_share_embed_columns: @@ -698,11 +640,9 @@ def _add_deep_embedding_column(self, fc, config): combiner=config.combiner, initializer=initializer, partitioner=self._build_partitioner(config), - ev_params=ev_params + ev_params=ev_params, ) - fc.max_seq_length = config.max_seq_len if config.HasField( - 'max_seq_len' - ) else -1 + fc.max_seq_length = config.max_seq_len if config.HasField('max_seq_len') else -1 if config.feature_type != config.SequenceFeature: self._deep_columns[feature_name] = fc @@ -716,6 +656,8 @@ def _build_ev_params(self, ev_params): ev_params = EVParams( ev_params.filter_freq, ev_params.steps_to_live if ev_params.steps_to_live > 0 else None, - ev_params.use_cache, ev_params.init_capacity, ev_params.max_capacity + ev_params.use_cache, + ev_params.init_capacity, + ev_params.max_capacity, ) return ev_params diff --git a/easy_rec/python/feature_column/feature_group.py b/easy_rec/python/feature_column/feature_group.py index a30cd820d..19552542e 100644 --- a/easy_rec/python/feature_column/feature_group.py +++ b/easy_rec/python/feature_column/feature_group.py @@ -2,11 +2,13 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import re -from easy_rec.python.protos.feature_config_pb2 import FeatureGroupConfig, WideOrDeep # NOQA +from easy_rec.python.protos.feature_config_pb2 import ( # NOQA + FeatureGroupConfig, + WideOrDeep, +) class FeatureGroup(object): - def __init__(self, feature_group_config): self._config = feature_group_config assert isinstance(self._config, FeatureGroupConfig) diff --git a/easy_rec/python/hpo/emr_hpo.py b/easy_rec/python/hpo/emr_hpo.py index 5ec95d748..796e603ca 100644 --- a/easy_rec/python/hpo/emr_hpo.py +++ b/easy_rec/python/hpo/emr_hpo.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Hyperparameter search for easy_rec on emr.""" + import argparse import json import logging @@ -13,24 +14,12 @@ from easy_rec.python.utils import hpo_util file_dir, _ = os.path.split(os.path.abspath(__file__)) -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) - - -def hpo_config( - config_path, hyperparams, exp_dir, metric_name, el_submit_params -): - earlystop = { - 'type': 'large_is_better', - 'threshold': 0.99, - 'max_runtime': 2400 - } - algorithm = { - 'type': 'gp', - 'initial_trials_num': 4, - 'stop_when_exception': True - } +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') + + +def hpo_config(config_path, hyperparams, exp_dir, metric_name, el_submit_params): + earlystop = {'type': 'large_is_better', 'threshold': 0.99, 'max_runtime': 2400} + algorithm = {'type': 'gp', 'initial_trials_num': 4, 'stop_when_exception': True} tmp_dir = '/tmp/emr_easy_rec_hpo_%d' % time.time() os.makedirs(tmp_dir) @@ -41,34 +30,49 @@ def hpo_config( model_path = '%s/trail_{{ trial.id }}' % exp_dir metric_path = os.path.join(model_path, 'res.metric') - pre_task = { - 'type': 'BashTask', - 'cmd': ['hadoop', 'fs', '-mkdir', '-p', model_path] - } + pre_task = {'type': 'BashTask', 'cmd': ['hadoop', 'fs', '-mkdir', '-p', model_path]} adapter_task = { 'type': 'localadaptertask', # hpo_param_path for easy_rec 'param_file': param_path, } - el_params = [ - x.strip() for x in el_submit_params.split(' ') if x.strip() != '' - ] - assert len(el_params - ) % 2 == 0, 'invalid number of el_submit params: %d[%s]' % ( - len(el_params), str(el_params) - ) + el_params = [x.strip() for x in el_submit_params.split(' ') if x.strip() != ''] + assert len(el_params) % 2 == 0, 'invalid number of el_submit params: %d[%s]' % ( + len(el_params), + str(el_params), + ) for i in range(0, len(el_params), 2): assert el_params[i] in [ - '-t', '-m', '-pn', '-pc', '-pg', '-pm', '-wn', '-wc', '-wm', '-wg' + '-t', + '-m', + '-pn', + '-pc', + '-pg', + '-pm', + '-wn', + '-wc', + '-wm', + '-wg', + ] + cmd = ( + ['el_submit'] + + el_params + + [ + '-a', + 'easy_rec_hpo', + '-m', + 'local', + '-f', + '{},train_eval.py,{}'.format(config_path, param_path), + '--interact', + 'INTERACT', + '-c', + 'python -m easy_rec.python.train_eval --hpo_metric_save_path {} ' + '--hpo_param_path {} --pipeline_config_path {} --model_dir {}'.format( + metric_path, param_path_file, config_path, model_path + ), ] - cmd = ['el_submit'] + el_params + [ - '-a', 'easy_rec_hpo', '-m', 'local', '-f', - '{},train_eval.py,{}'.format(config_path, - param_path), '--interact', 'INTERACT', '-c', - 'python -m easy_rec.python.train_eval --hpo_metric_save_path {} ' - '--hpo_param_path {} --pipeline_config_path {} --model_dir {}'. - format(metric_path, param_path_file, config_path, model_path) - ] + ) train_task = { 'type': 'BashTask', @@ -76,8 +80,8 @@ def hpo_config( 'metric_reader': { 'type': 'hdfs_reader', 'location': metric_path, - 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name - } + 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name, + }, } tasks = [pre_task, adapter_task, train_task] @@ -85,48 +89,34 @@ def hpo_config( 'earlystop': earlystop, 'algorithm': algorithm, 'hyperparams': hyperparams, - 'tasks': tasks + 'tasks': tasks, } return data, tmp_dir if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( - '--hyperparams', type=str, help='hyper parameters', default=None - ) - parser.add_argument( - '--config_path', type=str, help='pipeline config', default=None - ) - parser.add_argument( - '--exp_dir', type=str, help='hpo experiment directory', default=None - ) + parser.add_argument('--hyperparams', type=str, help='hyper parameters', default=None) + parser.add_argument('--config_path', type=str, help='pipeline config', default=None) + parser.add_argument('--exp_dir', type=str, help='hpo experiment directory', default=None) parser.add_argument( '--el_submit_params', type=str, - help= - 'el_submit parameters(-t x -m x [-pn x -pc x -pm x] -wn x -wc x -wm x -wg x)', - default='-t standalone -m local -wn 1 -wc 6 -wm 20000 -wg 1' - ) - parser.add_argument( - '--metric_name', type=str, help='metric_name', default='auc' + help='el_submit parameters(-t x -m x [-pn x -pc x -pm x] -wn x -wc x -wm x -wg x)', + default='-t standalone -m local -wn 1 -wc 6 -wm 20000 -wg 1', ) + parser.add_argument('--metric_name', type=str, help='metric_name', default='auc') parser.add_argument( '--max_parallel', type=int, help='max number of trials run at the same time', - default=4 - ) - parser.add_argument( - '--total_trial_num', - type=int, - help='total number of trials will run', - default=6 + default=4, ) + parser.add_argument('--total_trial_num', type=int, help='total number of trials will run', default=6) parser.add_argument( '--debug', action='store_true', - help='debug mode, will keep the temporary folder' + help='debug mode, will keep the temporary folder', ) args = parser.parse_args() @@ -139,15 +129,16 @@ def hpo_config( hyperparams = json.load(fin) data, tmp_dir = hpo_config( - args.config_path, hyperparams, args.exp_dir, args.metric_name, - args.el_submit_params + args.config_path, + hyperparams, + args.exp_dir, + args.metric_name, + args.el_submit_params, ) hpo_util.kill_old_proc(tmp_dir, platform='emr') - tuner = AutoTuner.create_tuner( - data, max_parallel=args.max_parallel, max_trial_num=args.total_trial_num - ) + tuner = AutoTuner.create_tuner(data, max_parallel=args.max_parallel, max_trial_num=args.total_trial_num) tuner.fit(synchronize=True) if not args.debug: diff --git a/easy_rec/python/hpo/generate_hpo_sql.py b/easy_rec/python/hpo/generate_hpo_sql.py index c91e0df9c..2a0fcf6ea 100644 --- a/easy_rec/python/hpo/generate_hpo_sql.py +++ b/easy_rec/python/hpo/generate_hpo_sql.py @@ -6,54 +6,31 @@ import argparse parser = argparse.ArgumentParser() - parser.add_argument( - '--sql_path', type=str, help='output sql path', default=None - ) - parser.add_argument( - '--config_path', type=str, help='config path', default=None - ) - parser.add_argument( - '--tables', type=str, help='train_table and test_table', default=None - ) - parser.add_argument( - '--train_tables', type=str, help='train_tables', default=None - ) - parser.add_argument( - '--eval_tables', type=str, help='eval_tables', default=None - ) + parser.add_argument('--sql_path', type=str, help='output sql path', default=None) + parser.add_argument('--config_path', type=str, help='config path', default=None) + parser.add_argument('--tables', type=str, help='train_table and test_table', default=None) + parser.add_argument('--train_tables', type=str, help='train_tables', default=None) + parser.add_argument('--eval_tables', type=str, help='eval_tables', default=None) parser.add_argument( '--cluster', type=str, help='specify tensorflow train jobs cluster parameter', - default=None + default=None, ) parser.add_argument('--bucket', type=str, help='oss bucket', default=None) - parser.add_argument( - '--hpo_param_path', type=str, help='hpo param path', default=None - ) - parser.add_argument( - '--hpo_metric_save_path', - type=str, - help='hpo metric save path', - default=None - ) + parser.add_argument('--hpo_param_path', type=str, help='hpo param path', default=None) + parser.add_argument('--hpo_metric_save_path', type=str, help='hpo metric save path', default=None) parser.add_argument('--model_dir', type=str, help='model_dir', default=None) - parser.add_argument( - '--oss_host', type=str, help='oss endpoint', default=None - ) + parser.add_argument('--oss_host', type=str, help='oss endpoint', default=None) parser.add_argument('--role_arn', type=str, help='role arn', default=None) parser.add_argument( '--algo_proj_name', type=str, help='algorithm project name', - default='algo_public' - ) - parser.add_argument( - '--algo_res_proj', type=str, help='algo resource project', default=None - ) - parser.add_argument( - '--algo_version', type=str, help='algo version', default=None + default='algo_public', ) + parser.add_argument('--algo_res_proj', type=str, help='algo resource project', default=None) + parser.add_argument('--algo_version', type=str, help='algo version', default=None) args = parser.parse_args() @@ -72,7 +49,7 @@ else: fout.write(' -Dtrain_tables=%s\n' % args.train_tables) fout.write(' -Deval_tables=%s\n' % args.eval_tables) - fout.write(' -Dcluster=\'%s\'\n' % args.cluster) + fout.write(" -Dcluster='%s'\n" % args.cluster) fout.write(' -Darn=%s\n' % args.role_arn) fout.write(' -Dbuckets=%s\n' % args.bucket) fout.write(' -Dhpo_param_path=%s\n' % args.hpo_param_path) diff --git a/easy_rec/python/hpo/pai_hpo.py b/easy_rec/python/hpo/pai_hpo.py index d2c9e3f3c..7a909faf8 100644 --- a/easy_rec/python/hpo/pai_hpo.py +++ b/easy_rec/python/hpo/pai_hpo.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Hyperparameter search demo for easy_rec on pai.""" + import json import logging import os @@ -12,18 +13,14 @@ from easy_rec.python.utils import hpo_util file_dir, _ = os.path.split(os.path.abspath(__file__)) -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') try: import subprocess subprocess.check_output('which odpscmd', shell=True) except Exception: - logging.error( - 'odpscmd is not in path, please install from https://help.aliyun.com/document_detail/27971.html' - ) + logging.error('odpscmd is not in path, please install from https://help.aliyun.com/document_detail/27971.html') def get_tuner(data, max_parallel, max_trial_num): @@ -67,26 +64,32 @@ def get_tuner(data, max_parallel, max_trial_num): max_parallel=max_parallel, max_trial_num=max_trial_num, mode='local', - user_id='your_cloud_id' + user_id='your_cloud_id', ) return tuner def hpo_config( - config_path, hyperparams, environment, exp_dir, tables, train_tables, - eval_tables, cluster, algo_proj_name, algo_res_proj, algo_version, - metric_name, odps_config_path + config_path, + hyperparams, + environment, + exp_dir, + tables, + train_tables, + eval_tables, + cluster, + algo_proj_name, + algo_res_proj, + algo_version, + metric_name, + odps_config_path, ): earlystop = {'type': 'large_is_better', 'max_runtime': 3600 * 12} - algorithm = { - 'type': 'gp', - 'initial_trials_num': 4, - 'stop_when_exception': True - } + algorithm = {'type': 'gp', 'initial_trials_num': 4, 'stop_when_exception': True} if exp_dir.startswith('oss://'): exp_dir = exp_dir.replace('oss://', '') - exp_dir = exp_dir[exp_dir.find('/') + 1:] + exp_dir = exp_dir[exp_dir.find('/') + 1 :] param_path = '%s/hpo_test_{{ trial.id }}.json' % exp_dir metric_path = '%s/easy_rec_hpo_{{ trial.id }}.metric' % exp_dir @@ -122,22 +125,35 @@ def _add_prefix(table_name): sql_path = '%s/train_ext_hpo_{{ trial.id }}.sql' % tmp_dir cmd_args = [ - 'python', '-m', 'easy_rec.python.hpo.generate_hpo_sql', '--sql_path', - sql_path, '--config_path', config_path, '--cluster', cluster, '--bucket', - bucket, '--hpo_param_path', - os.path.join(bucket, param_path), '--hpo_metric_save_path', - os.path.join(bucket, metric_path), '--model_dir', - os.path.join(bucket, - model_path), '--oss_host', environment['oss_endpoint'], - '--role_arn', environment['role_arn'], '--algo_proj_name', algo_proj_name + 'python', + '-m', + 'easy_rec.python.hpo.generate_hpo_sql', + '--sql_path', + sql_path, + '--config_path', + config_path, + '--cluster', + cluster, + '--bucket', + bucket, + '--hpo_param_path', + os.path.join(bucket, param_path), + '--hpo_metric_save_path', + os.path.join(bucket, metric_path), + '--model_dir', + os.path.join(bucket, model_path), + '--oss_host', + environment['oss_endpoint'], + '--role_arn', + environment['role_arn'], + '--algo_proj_name', + algo_proj_name, ] if tables: cmd_args.extend(['--tables', tables]) if train_tables and eval_tables: - cmd_args.extend( - ['--train_tables', train_tables, '--eval_tables', eval_tables] - ) + cmd_args.extend(['--train_tables', train_tables, '--eval_tables', eval_tables]) if algo_res_proj: cmd_args.extend(['--algo_res_proj', algo_res_proj]) @@ -147,13 +163,12 @@ def _add_prefix(table_name): train_task = { 'type': 'BashTask', - 'cmd': ['odpscmd', - '--config=%s' % odps_config_path, '-f', sql_path], + 'cmd': ['odpscmd', '--config=%s' % odps_config_path, '-f', sql_path], 'metric_reader': { 'type': 'oss_reader', 'location': metric_path, - 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name - } + 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name, + }, } tasks = [adapter_task, prepare_sql_task, train_task] @@ -162,7 +177,7 @@ def _add_prefix(table_name): 'algorithm': algorithm, 'hyperparams': hyperparams, 'tasks': tasks, - 'environment': environment + 'environment': environment, } return data, tmp_dir @@ -171,70 +186,37 @@ def _add_prefix(table_name): import argparse parser = argparse.ArgumentParser() - parser.add_argument( - '--odps_config', type=str, help='odps_config.ini', default=None - ) - parser.add_argument( - '--oss_config', type=str, help='excel config path', default='' - ) + parser.add_argument('--odps_config', type=str, help='odps_config.ini', default=None) + parser.add_argument('--oss_config', type=str, help='excel config path', default='') parser.add_argument('--bucket', type=str, help='bucket name', default=None) parser.add_argument('--role_arn', type=str, help='role arn', default=None) - parser.add_argument( - '--hyperparams', type=str, help='hyper parameters', default=None - ) - parser.add_argument( - '--config_path', type=str, help='pipeline config', default=None - ) - parser.add_argument( - '--tables', type=str, help='train table and test table', default=None - ) - parser.add_argument( - '--train_tables', type=str, help='train tables', default=None - ) - parser.add_argument( - '--eval_tables', type=str, help='eval tables', default=None - ) - parser.add_argument( - '--exp_dir', type=str, help='hpo experiment directory', default=None - ) + parser.add_argument('--hyperparams', type=str, help='hyper parameters', default=None) + parser.add_argument('--config_path', type=str, help='pipeline config', default=None) + parser.add_argument('--tables', type=str, help='train table and test table', default=None) + parser.add_argument('--train_tables', type=str, help='train tables', default=None) + parser.add_argument('--eval_tables', type=str, help='eval tables', default=None) + parser.add_argument('--exp_dir', type=str, help='hpo experiment directory', default=None) parser.add_argument( '--cluster', type=str, help='cluster spec', - default= - '{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}' - ) - parser.add_argument( - '--algo_proj_name', - type=str, - help='algo project name', - default='algo_public' - ) - parser.add_argument( - '--algo_version', type=str, help='algo version', default=None - ) - parser.add_argument( - '--algo_res_proj', type=str, help='algo resource project', default=None - ) - parser.add_argument( - '--metric_name', type=str, help='evaluate metric name', default='auc' + default='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}', ) + parser.add_argument('--algo_proj_name', type=str, help='algo project name', default='algo_public') + parser.add_argument('--algo_version', type=str, help='algo version', default=None) + parser.add_argument('--algo_res_proj', type=str, help='algo resource project', default=None) + parser.add_argument('--metric_name', type=str, help='evaluate metric name', default='auc') parser.add_argument( '--max_parallel', type=int, help='max number of trials run at the same time', - default=4 - ) - parser.add_argument( - '--total_trial_num', - type=int, - help='total number of trials will run', - default=6 + default=4, ) + parser.add_argument('--total_trial_num', type=int, help='total number of trials will run', default=6) parser.add_argument( '--debug', action='store_true', - help='debug mode, will keep the temporary folder' + help='debug mode, will keep the temporary folder', ) args = parser.parse_args() @@ -251,7 +233,7 @@ def _add_prefix(table_name): if '=' in line_str: tmp_id = line_str.find('=') key = line_str[:tmp_id].strip() - val = line_str[(tmp_id + 1):].strip() + val = line_str[(tmp_id + 1) :].strip() odps_config[key] = val if args.oss_config is None: @@ -268,14 +250,14 @@ def _add_prefix(table_name): if '=' in line_str: tmp_id = line_str.find('=') key = line_str[:tmp_id].strip() - val = line_str[(tmp_id + 1):].strip() + val = line_str[(tmp_id + 1) :].strip() oss_config[key] = val assert args.bucket is not None assert args.role_arn is not None if args.bucket.startswith('oss://'): - args.bucket = args.bucket[len('oss://'):] + args.bucket = args.bucket[len('oss://') :] args.bucket = args.bucket.strip('/') environment = { @@ -288,7 +270,7 @@ def _add_prefix(table_name): 'biz_id': '147331^paistudio^xxxxxxx^2020-03-18', 'role_arn': args.role_arn, 'bucket': args.bucket, - 'oss_endpoint': oss_config['endpoint'] + 'oss_endpoint': oss_config['endpoint'], } assert args.hyperparams is not None @@ -297,14 +279,22 @@ def _add_prefix(table_name): assert args.config_path is not None assert args.exp_dir is not None - assert args.tables is not None or ( - args.train_tables is not None and args.eval_tables is not None - ) + assert args.tables is not None or (args.train_tables is not None and args.eval_tables is not None) data, tmp_dir = hpo_config( - args.config_path, hyperparams, environment, args.exp_dir, args.tables, - args.train_tables, args.eval_tables, args.cluster, args.algo_proj_name, - args.algo_res_proj, args.algo_version, args.metric_name, args.odps_config + args.config_path, + hyperparams, + environment, + args.exp_dir, + args.tables, + args.train_tables, + args.eval_tables, + args.cluster, + args.algo_proj_name, + args.algo_res_proj, + args.algo_version, + args.metric_name, + args.odps_config, ) hpo_util.kill_old_proc(tmp_dir, platform='pai') diff --git a/easy_rec/python/inference/client/client_demo.py b/easy_rec/python/inference/client/client_demo.py index 13e5e9726..f2608bdcf 100644 --- a/easy_rec/python/inference/client/client_demo.py +++ b/easy_rec/python/inference/client/client_demo.py @@ -8,9 +8,7 @@ from easy_rec.python.inference.client.easyrec_request import EasyrecRequest from easy_rec.python.protos.predict_pb2 import PBFeature, PBRequest -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') try: from eas_prediction import PredictClient # TFRequest @@ -49,8 +47,8 @@ def build_request(table_cols, table_data, item_ids=None): def parse_table_schema(create_table_sql): create_table_sql = create_table_sql.lower() spos = create_table_sql.index('(') - epos = create_table_sql[spos + 1:].index(')') - cols = create_table_sql[(spos + 1):epos] + epos = create_table_sql[spos + 1 :].index(')') + cols = create_table_sql[(spos + 1) : epos] cols = [x.strip().lower() for x in cols.split(',')] col_info_arr = [] for col in cols: @@ -74,26 +72,12 @@ def send_request(req_pb, client, debug_level=0): '--endpoint', type=str, default=None, - help='eas endpoint, such as 12345.cn-beijing.pai-eas.aliyuncs.com' - ) - parser.add_argument( - '--service_name', type=str, default=None, help='eas service name' - ) - parser.add_argument( - '--token', type=str, default=None, help='eas service token' - ) - parser.add_argument( - '--table_schema', - type=str, - default=None, - help='user feature table schema path' - ) - parser.add_argument( - '--table_data', - type=str, - default=None, - help='user feature table data path' + help='eas endpoint, such as 12345.cn-beijing.pai-eas.aliyuncs.com', ) + parser.add_argument('--service_name', type=str, default=None, help='eas service name') + parser.add_argument('--token', type=str, default=None, help='eas service token') + parser.add_argument('--table_schema', type=str, default=None, help='user feature table schema path') + parser.add_argument('--table_data', type=str, default=None, help='user feature table data path') parser.add_argument('--item_lst', type=str, default=None, help='item list') args, _ = parser.parse_known_args() diff --git a/easy_rec/python/inference/csv_predictor.py b/easy_rec/python/inference/csv_predictor.py index 83dbb5f2a..0a1ffe69d 100644 --- a/easy_rec/python/inference/csv_predictor.py +++ b/easy_rec/python/inference/csv_predictor.py @@ -8,17 +8,18 @@ import tensorflow as tf from tensorflow.python.platform import gfile +from easy_rec.python.inference.predictor import ( # NOQA + SINGLE_PLACEHOLDER_FEATURE_KEY, + Predictor, +) from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils.check_utils import check_split -from easy_rec.python.inference.predictor import SINGLE_PLACEHOLDER_FEATURE_KEY, Predictor # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 class CSVPredictor(Predictor): - def __init__( self, model_path, @@ -28,10 +29,9 @@ def __init__( fg_json_path=None, profiling_file=None, selected_cols=None, - output_sep=chr(1) + output_sep=chr(1), ): - super(CSVPredictor, - self).__init__(model_path, profiling_file, fg_json_path) + super(CSVPredictor, self).__init__(model_path, profiling_file, fg_json_path) self._output_sep = output_sep self._ds_vector_recall = ds_vector_recall input_type = DatasetConfig.InputType.Name(data_config.input_type).lower() @@ -75,9 +75,9 @@ def _get_reserved_cols(self, reserved_cols): def _parse_line(self, line): check_list = [ tf.py_func( - check_split, [line, self._input_sep, - len(self._record_defaults)], - Tout=tf.bool + check_split, + [line, self._input_sep, len(self._record_defaults)], + Tout=tf.bool, ) ] with tf.control_dependencies(check_list): @@ -85,7 +85,7 @@ def _parse_line(self, line): line, field_delim=self._input_sep, record_defaults=self._record_defaults, - name='decode_csv' + name='decode_csv', ) if self._is_rtp: if self._with_header: @@ -113,7 +113,7 @@ def _get_num_cols(self, file_paths): line_tok = line_str.strip().split(self._input_sep) if num_cols != -1: assert num_cols == len(line_tok), ( - 'num selected cols is %d, not equal to %d, current line is: %s, please check input_sep and data.' + 'num selected cols is %d, not equal to %d, current line is: %s, please check input_sep and data' % (num_cols, len(line_tok), line_str) ) num_cols = len(line_tok) @@ -123,9 +123,7 @@ def _get_num_cols(self, file_paths): logging.info('num selected cols = %d' % num_cols) return num_cols - def _get_dataset( - self, input_path, num_parallel_calls, batch_size, slice_num, slice_id - ): + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): file_paths = [] for path in input_path.split(','): for x in gfile.Glob(path): @@ -155,16 +153,14 @@ def _get_dataset( default_val = self._get_defaults(col_name) self._record_defaults[col_idx] = default_val else: - self._record_defaults = [ - self._get_defaults(col_name) for col_name in self._all_fields - ] + self._record_defaults = [self._get_defaults(col_name) for col_name in self._all_fields] dataset = tf.data.Dataset.from_tensor_slices(file_paths) parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( lambda x: tf.data.TextLineDataset(x).skip(int(self._with_header)), cycle_length=parallel_num, - num_parallel_calls=parallel_num + num_parallel_calls=parallel_num, ) dataset = dataset.shard(slice_num, slice_id) dataset = dataset.batch(batch_size) @@ -176,22 +172,17 @@ def _get_writer(self, output_path, slice_id): gfile.MakeDirs(output_path) res_path = os.path.join(output_path, 'part-%d.csv' % slice_id) table_writer = gfile.GFile(res_path, 'w') - table_writer.write( - self._output_sep.join(self._output_cols + self._reserved_cols) + '\n' - ) + table_writer.write(self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') return table_writer def _write_lines(self, table_writer, outputs): - outputs = '\n'.join( - [self._output_sep.join([str(i) for i in output]) for output in outputs] - ) + outputs = '\n'.join([self._output_sep.join([str(i) for i in output]) for output in outputs]) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [outputs[x] for x in output_cols] + \ - [all_vals[k] for k in reserved_cols] + reserve_vals = [outputs[x] for x in output_cols] + [all_vals[k] for k in reserved_cols] return reserve_vals @property def out_of_range_exception(self): - return (tf.errors.OutOfRangeError) + return tf.errors.OutOfRangeError diff --git a/easy_rec/python/inference/hive_parquet_predictor.py b/easy_rec/python/inference/hive_parquet_predictor.py index f2d7976de..33f9ae84e 100644 --- a/easy_rec/python/inference/hive_parquet_predictor.py +++ b/easy_rec/python/inference/hive_parquet_predictor.py @@ -21,7 +21,6 @@ class HiveParquetPredictor(Predictor): - def __init__( self, model_path, @@ -31,10 +30,9 @@ def __init__( profiling_file=None, output_sep=chr(1), all_cols=None, - all_col_types=None + all_col_types=None, ): - super(HiveParquetPredictor, - self).__init__(model_path, profiling_file, fg_json_path) + super(HiveParquetPredictor, self).__init__(model_path, profiling_file, fg_json_path) self._data_config = data_config self._hive_config = hive_config @@ -47,8 +45,7 @@ def __init__( self._all_cols = [x.strip() for x in all_cols if x != ''] self._all_col_types = [x.strip() for x in all_col_types if x != ''] self._record_defaults = [ - self._get_defaults(col_name, col_type) - for col_name, col_type in zip(self._all_cols, self._all_col_types) + self._get_defaults(col_name, col_type) for col_name, col_type in zip(self._all_cols, self._all_col_types) ] def _get_reserved_cols(self, reserved_cols): @@ -63,29 +60,20 @@ def _parse_line(self, *fields): field_dict = {self._all_cols[i]: fields[i] for i in range(len(fields))} return field_dict - def _get_dataset( - self, input_path, num_parallel_calls, batch_size, slice_num, slice_id - ): - self._hive_util = HiveUtils( - data_config=self._data_config, hive_config=self._hive_config - ) + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): + self._hive_util = HiveUtils(data_config=self._data_config, hive_config=self._hive_config) hdfs_path = self._hive_util.get_table_location(input_path) self._input_hdfs_path = gfile.Glob(os.path.join(hdfs_path, '*')) - assert len( - self._input_hdfs_path - ) > 0, 'match no files with %s' % input_path + assert len(self._input_hdfs_path) > 0, 'match no files with %s' % input_path list_type = [] - input_field_type_map = { - x.input_name: x.input_type - for x in self._data_config.input_fields - } + input_field_type_map = {x.input_name: x.input_type for x in self._data_config.input_fields} type_2_tftype = { 'string': tf.string, 'double': tf.double, 'float': tf.float32, 'bigint': tf.int32, - 'boolean': tf.bool + 'boolean': tf.bool, } for col_name, col_type in zip(self._all_cols, self._all_col_types): if col_name in input_field_type_map: @@ -117,9 +105,7 @@ def parquet_read(): inputs.append(batch_data[k].to_numpy()) yield tuple(inputs) - dataset = tf.data.Dataset.from_generator( - parquet_read, output_types=list_type, output_shapes=list_shapes - ) + dataset = tf.data.Dataset.from_generator(parquet_read, output_types=list_type, output_shapes=list_shapes) dataset = dataset.shard(slice_num, slice_id) dataset = dataset.prefetch(buffer_size=64) return dataset @@ -134,17 +120,14 @@ def get_table_info(self, output_path): return table_name, partition_name, partition_val def _get_writer(self, output_path, slice_id): - table_name, partition_name, partition_val = self.get_table_info( - output_path - ) - is_exist = self._hive_util.is_table_or_partition_exist( - table_name, partition_name, partition_val - ) + table_name, partition_name, partition_val = self.get_table_info(output_path) + is_exist = self._hive_util.is_table_or_partition_exist(table_name, partition_name, partition_val) assert not is_exist, '%s is already exists. Please drop it.' % output_path output_path = output_path.replace('.', '/') self._hdfs_path = 'hdfs://%s:9000/user/easy_rec/%s_tmp' % ( - self._hive_config.host, output_path + self._hive_config.host, + output_path, ) if not gfile.Exists(self._hdfs_path): gfile.MakeDirs(self._hdfs_path) @@ -153,14 +136,11 @@ def _get_writer(self, output_path, slice_id): return table_writer def _write_lines(self, table_writer, outputs): - outputs = '\n'.join( - [self._output_sep.join([str(i) for i in output]) for output in outputs] - ) + outputs = '\n'.join([self._output_sep.join([str(i) for i in output]) for output in outputs]) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [outputs[x] for x in output_cols] + \ - [all_vals[k] for k in reserved_cols] + reserve_vals = [outputs[x] for x in output_cols] + [all_vals[k] for k in reserved_cols] return reserve_vals def load_to_table(self, output_path, slice_num, slice_id): @@ -177,9 +157,7 @@ def load_to_table(self, output_path, slice_num, slice_id): while not gfile.Exists(res_path): time.sleep(10) - table_name, partition_name, partition_val = self.get_table_info( - output_path - ) + table_name, partition_name, partition_val = self.get_table_info(output_path) schema = '' for output_col_name in self._output_cols: tf_type = self._predictor_impl._outputs_map[output_col_name].dtype @@ -194,20 +172,28 @@ def load_to_table(self, output_path, slice_num, slice_id): schema = schema.rstrip(',') if partition_name and partition_val: - sql = 'create table if not exists %s (%s) PARTITIONED BY (%s string)' % \ - (table_name, schema, partition_name) + sql = 'create table if not exists %s (%s) PARTITIONED BY (%s string)' % ( + table_name, + schema, + partition_name, + ) self._hive_util.run_sql(sql) - sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s PARTITION (%s=%s)" % \ - (self._hdfs_path, table_name, partition_name, partition_val) + sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s PARTITION (%s=%s)" % ( + self._hdfs_path, + table_name, + partition_name, + partition_val, + ) self._hive_util.run_sql(sql) else: - sql = 'create table if not exists %s (%s)' % \ - (table_name, schema) + sql = 'create table if not exists %s (%s)' % (table_name, schema) self._hive_util.run_sql(sql) - sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s" % \ - (self._hdfs_path, table_name) + sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s" % ( + self._hdfs_path, + table_name, + ) self._hive_util.run_sql(sql) @property def out_of_range_exception(self): - return (tf.errors.OutOfRangeError) + return tf.errors.OutOfRangeError diff --git a/easy_rec/python/inference/hive_predictor.py b/easy_rec/python/inference/hive_predictor.py index a00b4b98e..f199bad6c 100644 --- a/easy_rec/python/inference/hive_predictor.py +++ b/easy_rec/python/inference/hive_predictor.py @@ -18,7 +18,6 @@ class HivePredictor(Predictor): - def __init__( self, model_path, @@ -28,10 +27,9 @@ def __init__( profiling_file=None, output_sep=chr(1), all_cols=None, - all_col_types=None + all_col_types=None, ): - super(HivePredictor, - self).__init__(model_path, profiling_file, fg_json_path) + super(HivePredictor, self).__init__(model_path, profiling_file, fg_json_path) self._data_config = data_config self._hive_config = hive_config @@ -44,8 +42,7 @@ def __init__( self._all_cols = [x.strip() for x in all_cols if x != ''] self._all_col_types = [x.strip() for x in all_col_types if x != ''] self._record_defaults = [ - self._get_defaults(col_name, col_type) - for col_name, col_type in zip(self._all_cols, self._all_col_types) + self._get_defaults(col_name, col_type) for col_name, col_type in zip(self._all_cols, self._all_col_types) ] def _get_reserved_cols(self, reserved_cols): @@ -61,17 +58,13 @@ def _parse_line(self, line): line, field_delim=field_delim, record_defaults=self._record_defaults, - name='decode_csv' + name='decode_csv', ) inputs = {self._all_cols[x]: fields[x] for x in range(len(fields))} return inputs - def _get_dataset( - self, input_path, num_parallel_calls, batch_size, slice_num, slice_id - ): - self._hive_util = HiveUtils( - data_config=self._data_config, hive_config=self._hive_config - ) + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): + self._hive_util = HiveUtils(data_config=self._data_config, hive_config=self._hive_config) self._input_hdfs_path = self._hive_util.get_table_location(input_path) file_paths = tf.gfile.Glob(os.path.join(self._input_hdfs_path, '*')) assert len(file_paths) > 0, 'match no files with %s' % input_path @@ -81,7 +74,7 @@ def _get_dataset( dataset = dataset.interleave( tf.data.TextLineDataset, cycle_length=parallel_num, - num_parallel_calls=parallel_num + num_parallel_calls=parallel_num, ) dataset = dataset.shard(slice_num, slice_id) dataset = dataset.batch(batch_size) @@ -98,17 +91,14 @@ def get_table_info(self, output_path): return table_name, partition_name, partition_val def _get_writer(self, output_path, slice_id): - table_name, partition_name, partition_val = self.get_table_info( - output_path - ) - is_exist = self._hive_util.is_table_or_partition_exist( - table_name, partition_name, partition_val - ) + table_name, partition_name, partition_val = self.get_table_info(output_path) + is_exist = self._hive_util.is_table_or_partition_exist(table_name, partition_name, partition_val) assert not is_exist, '%s is already exists. Please drop it.' % output_path output_path = output_path.replace('.', '/') self._hdfs_path = 'hdfs://%s:9000/user/easy_rec/%s_tmp' % ( - self._hive_config.host, output_path + self._hive_config.host, + output_path, ) if not gfile.Exists(self._hdfs_path): gfile.MakeDirs(self._hdfs_path) @@ -117,14 +107,11 @@ def _get_writer(self, output_path, slice_id): return table_writer def _write_lines(self, table_writer, outputs): - outputs = '\n'.join( - [self._output_sep.join([str(i) for i in output]) for output in outputs] - ) + outputs = '\n'.join([self._output_sep.join([str(i) for i in output]) for output in outputs]) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [outputs[x] for x in output_cols] + \ - [all_vals[k] for k in reserved_cols] + reserve_vals = [outputs[x] for x in output_cols] + [all_vals[k] for k in reserved_cols] return reserve_vals def load_to_table(self, output_path, slice_num, slice_id): @@ -141,9 +128,7 @@ def load_to_table(self, output_path, slice_num, slice_id): while not gfile.Exists(res_path): time.sleep(10) - table_name, partition_name, partition_val = self.get_table_info( - output_path - ) + table_name, partition_name, partition_val = self.get_table_info(output_path) schema = '' for output_col_name in self._output_cols: tf_type = self._predictor_impl._outputs_map[output_col_name].dtype @@ -158,20 +143,28 @@ def load_to_table(self, output_path, slice_num, slice_id): schema = schema.rstrip(',') if partition_name and partition_val: - sql = 'create table if not exists %s (%s) PARTITIONED BY (%s string)' % \ - (table_name, schema, partition_name) + sql = 'create table if not exists %s (%s) PARTITIONED BY (%s string)' % ( + table_name, + schema, + partition_name, + ) self._hive_util.run_sql(sql) - sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s PARTITION (%s=%s)" % \ - (self._hdfs_path, table_name, partition_name, partition_val) + sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s PARTITION (%s=%s)" % ( + self._hdfs_path, + table_name, + partition_name, + partition_val, + ) self._hive_util.run_sql(sql) else: - sql = 'create table if not exists %s (%s)' % \ - (table_name, schema) + sql = 'create table if not exists %s (%s)' % (table_name, schema) self._hive_util.run_sql(sql) - sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s" % \ - (self._hdfs_path, table_name) + sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s" % ( + self._hdfs_path, + table_name, + ) self._hive_util.run_sql(sql) @property def out_of_range_exception(self): - return (tf.errors.OutOfRangeError) + return tf.errors.OutOfRangeError diff --git a/easy_rec/python/inference/odps_predictor.py b/easy_rec/python/inference/odps_predictor.py index e38ff7ff7..737ea52b5 100644 --- a/easy_rec/python/inference/odps_predictor.py +++ b/easy_rec/python/inference/odps_predictor.py @@ -8,24 +8,19 @@ class ODPSPredictor(Predictor): - def __init__( self, model_path, fg_json_path=None, profiling_file=None, all_cols='', - all_col_types='' + all_col_types='', ): - super(ODPSPredictor, - self).__init__(model_path, profiling_file, fg_json_path) + super(ODPSPredictor, self).__init__(model_path, profiling_file, fg_json_path) self._all_cols = [x.strip() for x in all_cols.split(',') if x != ''] - self._all_col_types = [ - x.strip() for x in all_col_types.split(',') if x != '' - ] + self._all_col_types = [x.strip() for x in all_col_types.split(',') if x != ''] self._record_defaults = [ - self._get_defaults(col_name, col_type) - for col_name, col_type in zip(self._all_cols, self._all_col_types) + self._get_defaults(col_name, col_type) for col_name, col_type in zip(self._all_cols, self._all_col_types) ] def _get_reserved_cols(self, reserved_cols): @@ -37,16 +32,14 @@ def _parse_line(self, *fields): field_dict = {self._all_cols[i]: fields[i] for i in range(len(fields))} return field_dict - def _get_dataset( - self, input_path, num_parallel_calls, batch_size, slice_num, slice_id - ): + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): input_list = input_path.split(',') dataset = tf.data.TableRecordDataset( input_list, record_defaults=self._record_defaults, slice_id=slice_id, slice_count=slice_num, - selected_cols=','.join(self._all_cols) + selected_cols=','.join(self._all_cols), ) dataset = dataset.batch(batch_size) dataset = dataset.prefetch(buffer_size=64) @@ -54,6 +47,7 @@ def _get_dataset( def _get_writer(self, output_path, slice_id): import common_io + table_writer = common_io.table.TableWriter(output_path, slice_id=slice_id) return table_writer @@ -67,6 +61,5 @@ def out_of_range_exception(self): return (tf.python_io.OutOfRangeException, tf.errors.OutOfRangeError) def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [all_vals[k] for k in reserved_cols] + \ - [outputs[x] for x in output_cols] + reserve_vals = [all_vals[k] for k in reserved_cols] + [outputs[x] for x in output_cols] return reserve_vals diff --git a/easy_rec/python/inference/parquet_predictor.py b/easy_rec/python/inference/parquet_predictor.py index d808b0c9b..4669f26ed 100644 --- a/easy_rec/python/inference/parquet_predictor.py +++ b/easy_rec/python/inference/parquet_predictor.py @@ -19,6 +19,7 @@ from tensorflow.python.framework.load_library import load_op_library import easy_rec + load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) except Exception as ex: @@ -26,7 +27,6 @@ class ParquetPredictor(Predictor): - def __init__( self, model_path, @@ -36,10 +36,9 @@ def __init__( profiling_file=None, selected_cols=None, output_sep=chr(1), - pipeline_config=None + pipeline_config=None, ): - super(ParquetPredictor, - self).__init__(model_path, profiling_file, fg_json_path) + super(ParquetPredictor, self).__init__(model_path, profiling_file, fg_json_path) self._output_sep = output_sep self._ds_vector_recall = ds_vector_recall input_type = DatasetConfig.InputType.Name(data_config.input_type).lower() @@ -73,12 +72,8 @@ def _get_reserved_cols(self, reserved_cols): # already parsed in _get_dataset return self._reserved_cols - def _get_dataset( - self, input_path, num_parallel_calls, batch_size, slice_num, slice_id - ): - feature_configs = config_util.get_compatible_feature_configs( - self.pipeline_config - ) + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): + feature_configs = config_util.get_compatible_feature_configs(self.pipeline_config) kwargs = {} if self._reserved_args is not None and len(self._reserved_args) > 0: @@ -90,22 +85,17 @@ def _get_dataset( kwargs['reserve_fields'] = all_cols self._all_fields = all_cols self._reserved_cols = all_cols - kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( - all_cols, parquet_file - ) + kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file(all_cols, parquet_file) else: - self._reserved_cols = [ - x.strip() for x in self._reserved_args.split(',') if x.strip() != '' - ] + self._reserved_cols = [x.strip() for x in self._reserved_args.split(',') if x.strip() != ''] kwargs['reserve_fields'] = self._reserved_cols parquet_file = gfile.Glob(input_path.split(',')[0])[0] - kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( - self._reserved_cols, parquet_file - ) + kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file(self._reserved_cols, parquet_file) logging.info( - 'reserve_fields=%s reserve_types=%s' % ( - ','.join(self._reserved_cols - ), ','.join([str(x) for x in kwargs['reserve_types']]) + 'reserve_fields=%s reserve_types=%s' + % ( + ','.join(self._reserved_cols), + ','.join([str(x) for x in kwargs['reserve_types']]), ) ) else: @@ -120,7 +110,7 @@ def _get_dataset( task_index=slice_id, task_num=slice_num, pipeline_config=self.pipeline_config, - **kwargs + **kwargs, ) return parquet_input._build(tf.estimator.ModeKeys.PREDICT, {}) @@ -129,15 +119,11 @@ def _get_writer(self, output_path, slice_id): gfile.MakeDirs(output_path) res_path = os.path.join(output_path, 'part-%d.csv' % slice_id) table_writer = gfile.GFile(res_path, 'w') - table_writer.write( - self._output_sep.join(self._output_cols + self._reserved_cols) + '\n' - ) + table_writer.write(self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') return table_writer def _write_lines(self, table_writer, outputs): - outputs = '\n'.join( - [self._output_sep.join([str(i) for i in output]) for output in outputs] - ) + outputs = '\n'.join([self._output_sep.join([str(i) for i in output]) for output in outputs]) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): @@ -154,4 +140,4 @@ def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): @property def out_of_range_exception(self): - return (tf.errors.OutOfRangeError) + return tf.errors.OutOfRangeError diff --git a/easy_rec/python/inference/parquet_predictor_v2.py b/easy_rec/python/inference/parquet_predictor_v2.py index 9b77fb3ad..b936e6740 100644 --- a/easy_rec/python/inference/parquet_predictor_v2.py +++ b/easy_rec/python/inference/parquet_predictor_v2.py @@ -19,6 +19,7 @@ from tensorflow.python.framework.load_library import load_op_library import easy_rec + load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) except Exception as ex: @@ -26,7 +27,6 @@ class ParquetPredictorV2(Predictor): - def __init__( self, model_path, @@ -36,10 +36,9 @@ def __init__( profiling_file=None, selected_cols=None, output_sep=chr(1), - pipeline_config=None + pipeline_config=None, ): - super(ParquetPredictorV2, - self).__init__(model_path, profiling_file, fg_json_path) + super(ParquetPredictorV2, self).__init__(model_path, profiling_file, fg_json_path) self._output_sep = output_sep self._ds_vector_recall = ds_vector_recall input_type = DatasetConfig.InputType.Name(data_config.input_type).lower() @@ -73,12 +72,8 @@ def _get_reserved_cols(self, reserved_cols): # already parsed in _get_dataset return self._reserved_cols - def _get_dataset( - self, input_path, num_parallel_calls, batch_size, slice_num, slice_id - ): - feature_configs = config_util.get_compatible_feature_configs( - self.pipeline_config - ) + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): + feature_configs = config_util.get_compatible_feature_configs(self.pipeline_config) kwargs = {} if self._reserved_args is not None and len(self._reserved_args) > 0: @@ -90,22 +85,17 @@ def _get_dataset( kwargs['reserve_fields'] = all_cols self._all_fields = all_cols self._reserved_cols = all_cols - kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( - all_cols, parquet_file - ) + kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file(all_cols, parquet_file) else: - self._reserved_cols = [ - x.strip() for x in self._reserved_args.split(',') if x.strip() != '' - ] + self._reserved_cols = [x.strip() for x in self._reserved_args.split(',') if x.strip() != ''] kwargs['reserve_fields'] = self._reserved_cols parquet_file = gfile.Glob(input_path.split(',')[0])[0] - kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( - self._reserved_cols, parquet_file - ) + kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file(self._reserved_cols, parquet_file) logging.info( - 'reserve_fields=%s reserve_types=%s' % ( - ','.join(self._reserved_cols - ), ','.join([str(x) for x in kwargs['reserve_types']]) + 'reserve_fields=%s reserve_types=%s' + % ( + ','.join(self._reserved_cols), + ','.join([str(x) for x in kwargs['reserve_types']]), ) ) else: @@ -120,7 +110,7 @@ def _get_dataset( task_index=slice_id, task_num=slice_num, pipeline_config=self.pipeline_config, - **kwargs + **kwargs, ) return parquet_input._build(tf.estimator.ModeKeys.PREDICT, {}) @@ -129,15 +119,11 @@ def _get_writer(self, output_path, slice_id): gfile.MakeDirs(output_path) res_path = os.path.join(output_path, 'part-%d.csv' % slice_id) table_writer = gfile.GFile(res_path, 'w') - table_writer.write( - self._output_sep.join(self._output_cols + self._reserved_cols) + '\n' - ) + table_writer.write(self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') return table_writer def _write_lines(self, table_writer, outputs): - outputs = '\n'.join( - [self._output_sep.join([str(i) for i in output]) for output in outputs] - ) + outputs = '\n'.join([self._output_sep.join([str(i) for i in output]) for output in outputs]) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): @@ -154,4 +140,4 @@ def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): @property def out_of_range_exception(self): - return (tf.errors.OutOfRangeError) + return tf.errors.OutOfRangeError diff --git a/easy_rec/python/inference/predictor.py b/easy_rec/python/inference/predictor.py index e77351a88..24aafc02b 100644 --- a/easy_rec/python/inference/predictor.py +++ b/easy_rec/python/inference/predictor.py @@ -18,11 +18,14 @@ import easy_rec from easy_rec.python.utils import numpy_utils +from easy_rec.python.utils.config_util import ( # NOQA + get_configs_from_pipeline_file, + get_input_name_from_fg_json, + search_fg_json, +) from easy_rec.python.utils.input_utils import get_type_defaults from easy_rec.python.utils.load_class import get_register_class_meta -from easy_rec.python.utils.config_util import get_configs_from_pipeline_file, get_input_name_from_fg_json, search_fg_json # NOQA - try: tf.load_op_library(os.path.join(easy_rec.ops_dir, 'libcustom_ops.so')) except Exception as ex: @@ -34,9 +37,7 @@ SINGLE_PLACEHOLDER_FEATURE_KEY = 'features' _PREDICTOR_CLASS_MAP = {} -_register_abc_meta = get_register_class_meta( - _PREDICTOR_CLASS_MAP, have_abstract_class=True -) +_register_abc_meta = get_register_class_meta(_PREDICTOR_CLASS_MAP, have_abstract_class=True) class PredictorInterface(six.with_metaclass(_register_abc_meta, object)): @@ -49,7 +50,6 @@ def __init__(self, model_path, model_config=None): model_path: init model from this directory model_config: config string for model to init, in json format """ - pass @abc.abstractmethod def predict(self, input_data, batch_size): @@ -65,7 +65,6 @@ def predict(self, input_data, batch_size): eg, {"output1": value1, "output2": value2}, the value type can be python int str float, and numpy array """ - pass def get_output_type(self): """Get output types of prediction. @@ -80,7 +79,8 @@ def get_output_type(self): * type image, data will be converted to encode image binary and write to oss file, whose name is output_dir/${key}/${input_filename}_${idx}.jpg, where input_filename is extracted from url, key corresponds to the key in the dict of output_type, - if the type of data indexed by key is a list, idx is the index of element in list, otherwhile ${idx} will be empty + if the type of data indexed by key is a list, idx is the index of element in list, + otherwhile ${idx} will be empty * type video, data will be converted to encode video binary and write to oss file, @@ -96,7 +96,6 @@ def get_output_type(self): class PredictorImpl(object): - def __init__(self, model_path, profiling_file=None, use_latest=False): """Impl class for predictor. @@ -151,18 +150,14 @@ def search_pb(self, directory): raise ValueError('savedmodel is not found in directory %s' % directory) elif len(dir_list) > 1: if self._use_latest: - logging.info( - 'find %d models: %s' % (len(dir_list), ','.join(dir_list)) - ) + logging.info('find %d models: %s' % (len(dir_list), ','.join(dir_list))) dir_list = sorted( dir_list, - key=lambda x: int(x.split('/')[(-2 if (x[-1] == '/') else -1)]) + key=lambda x: int(x.split('/')[(-2 if (x[-1] == '/') else -1)]), ) return dir_list[-1] else: - raise ValueError( - 'multiple saved model found in directory %s' % directory - ) + raise ValueError('multiple saved model found in directory %s' % directory) return dir_list[0] @@ -170,20 +165,15 @@ def _get_input_fields_from_pipeline_config(self, model_path): pipeline_path = os.path.join(model_path, 'assets/pipeline.config') if not gfile.Exists(pipeline_path): logging.warning( - '%s not exists, default values maybe inconsistent with the values used in training.' - % pipeline_path + '%s not exists, default values maybe inconsistent with the values used in training.' % pipeline_path ) return {} pipeline_config = get_configs_from_pipeline_file(pipeline_path) input_fields = pipeline_config.data_config.input_fields input_fields_info = { - input_field.input_name: - (input_field.input_type, input_field.default_val) - for input_field in input_fields + input_field.input_name: (input_field.input_type, input_field.default_val) for input_field in input_fields } - input_fields_list = [ - input_field.input_name for input_field in input_fields - ] + input_fields_list = [input_field.input_name for input_field in input_fields] return input_fields_info, input_fields_list @@ -195,7 +185,7 @@ def _build_model(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=(self._profiling_file is not None) + log_device_placement=(self._profiling_file is not None), ) self._session = tf.Session(config=session_config, graph=self._graph) @@ -207,18 +197,21 @@ def _build_model(self): if gfile.IsDirectory(model_path): model_path = self.search_pb(model_path) logging.info('model find in %s' % model_path) - self._input_fields_info, self._input_fields_list = self._get_input_fields_from_pipeline_config( - model_path + ( + self._input_fields_info, + self._input_fields_list, + ) = self._get_input_fields_from_pipeline_config(model_path) + assert tf.saved_model.loader.maybe_saved_model_directory(model_path), ( + 'saved model does not exists in %s' % model_path ) - assert tf.saved_model.loader.maybe_saved_model_directory(model_path), \ - 'saved model does not exists in %s' % model_path self._is_saved_model = True meta_graph_def = tf.saved_model.loader.load( - self._session, [tf.saved_model.tag_constants.SERVING], model_path + self._session, + [tf.saved_model.tag_constants.SERVING], + model_path, ) # parse signature - signature_def = meta_graph_def.signature_def[ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] + signature_def = meta_graph_def.signature_def[signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] inputs = signature_def.inputs # each input_info is a tuple of input_id, name, data_type input_info = [] @@ -226,9 +219,7 @@ def _build_model(self): if self._is_multi_placeholder: for gid, item in enumerate(inputs.items()): name, tensor = item - logging.info( - 'Load input binding: %s -> %s' % (name, tensor.name) - ) + logging.info('Load input binding: %s -> %s' % (name, tensor.name)) input_name = tensor.name input_name, _ = input_name.split(':') try: @@ -239,25 +230,16 @@ def _build_model(self): # in which case, the order of inputs may not be the # same as they are defined, thereforce, list input # could not be supported, only dict input could be supported - logging.warning( - 'could not determine input_id from input_name: %s' % - input_name - ) + logging.warning('could not determine input_id from input_name: %s' % input_name) input_id = gid input_info.append((input_id, name, tensor.dtype)) - self._inputs_map[name] = self._graph.get_tensor_by_name( - tensor.name - ) + self._inputs_map[name] = self._graph.get_tensor_by_name(tensor.name) else: # only one input, all features concatenate together for name, tensor in inputs.items(): - logging.info( - 'Load input binding: %s -> %s' % (name, tensor.name) - ) + logging.info('Load input binding: %s -> %s' % (name, tensor.name)) input_info.append((0, name, tensor.dtype)) - self._inputs_map[name] = self._graph.get_tensor_by_name( - tensor.name - ) + self._inputs_map[name] = self._graph.get_tensor_by_name(tensor.name) # sort inputs by input_ids so as to match the order of csv data input_info.sort(key=lambda t: t[0]) self._input_names = [t[1] for t in input_info] @@ -265,9 +247,7 @@ def _build_model(self): outputs = signature_def.outputs for name, tensor in outputs.items(): logging.info('Load output binding: %s -> %s' % (name, tensor.name)) - self._outputs_map[name] = self._graph.get_tensor_by_name( - tensor.name - ) + self._outputs_map[name] = self._graph.get_tensor_by_name(tensor.name) # get assets self._assets = {} @@ -276,9 +256,7 @@ def _build_model(self): asset_file = meta_graph_pb2.AssetFileDef() any_proto.Unpack(asset_file) type_name = asset_file.tensor_info.name.split(':')[0] - asset_path = os.path.join( - model_path, constants.ASSETS_DIRECTORY, asset_file.filename - ) + asset_path = os.path.join(model_path, constants.ASSETS_DIRECTORY, asset_file.filename) # assert gfile.Exists( # asset_path), '%s is missing in saved model' % asset_path if gfile.Exists(asset_path): @@ -311,15 +289,19 @@ def predict(self, input_data_dict, output_names=None): assert input_name in input_data_dict, 'input data %s is missing' % input_name tensor_shape = tensor.get_shape().as_list() input_shape = input_data_dict[input_name].shape - assert tensor_shape[0] is None or (tensor_shape[0] == input_shape[0]), \ - 'input %s batchsize %d is not the same as the exported batch_size %d' % \ - (input_name, input_shape[0], tensor_shape[0]) + assert tensor_shape[0] is None or (tensor_shape[0] == input_shape[0]), ( + 'input %s batchsize %d is not the same as the exported batch_size %d' + % ( + input_name, + input_shape[0], + tensor_shape[0], + ) + ) feed_dict[tensor] = input_data_dict[input_name] fetch_dict = {} if output_names is not None: for output_name in output_names: - assert output_name in self._outputs_map, \ - 'invalid output name %s' % output_name + assert output_name in self._outputs_map, 'invalid output name %s' % output_name fetch_dict[output_name] = self._outputs_map[output_name] else: fetch_dict = self._outputs_map @@ -335,10 +317,11 @@ def predict(self, input_data_dict, output_names=None): fetch_dict, feed_dict, options=run_options, - run_metadata=run_metadata + run_metadata=run_metadata, ) # Create the Timeline object, and write it to a json from tensorflow.python.client import timeline + tl = timeline.Timeline(run_metadata.step_stats) ctf = tl.generate_chrome_trace_format() with gfile.GFile(self._profiling_file, 'w') as f: @@ -347,10 +330,7 @@ def predict(self, input_data_dict, output_names=None): class Predictor(PredictorInterface): - - def __init__( - self, model_path, profiling_file=None, fg_json_path=None, use_latest=True - ): + def __init__(self, model_path, profiling_file=None, fg_json_path=None, use_latest=True): """Initialize a `Predictor`. Args: @@ -361,9 +341,7 @@ def __init__( fg_json_path: fg.json file use_latest: use latest saved_model.pb if multiple one exists. """ - self._predictor_impl = PredictorImpl( - model_path, profiling_file, use_latest - ) + self._predictor_impl = PredictorImpl(model_path, profiling_file, use_latest) self._inputs_map = self._predictor_impl._inputs_map self._outputs_map = self._predictor_impl._outputs_map self._profiling_file = profiling_file @@ -402,7 +380,8 @@ def _get_defaults(self, col_name, col_type='string'): else: defaults = {'string': '', 'double': 0.0, 'bigint': 0} assert col_type in defaults, 'invalid col_type: %s, col_type: %s' % ( - col_name, col_type + col_name, + col_type, ) default_val = defaults[col_type] logging.info( @@ -414,9 +393,7 @@ def _get_defaults(self, col_name, col_type='string'): def _parse_line(self, line): pass - def _get_dataset( - self, input_path, num_parallel_calls, batch_size, slice_num, slice_id - ): + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): pass def _get_writer(self, output_path, slice_id): @@ -491,12 +468,8 @@ def predict_impl( with tf.Graph().as_default(), tf.Session() as sess: num_parallel_calls = 8 self._reserved_args = reserved_cols - dataset = self._get_dataset( - input_path, num_parallel_calls, batch_size, slice_num, slice_id - ) - dataset = dataset.map( - self._parse_line, num_parallel_calls=num_parallel_calls - ) + dataset = self._get_dataset(input_path, num_parallel_calls, batch_size, slice_num, slice_id) + dataset = dataset.map(self._parse_line, num_parallel_calls=num_parallel_calls) if hasattr(tf.data, 'make_one_shot_iterator'): iterator = tf.data.make_one_shot_iterator(dataset) else: @@ -552,44 +525,28 @@ def _parse_value(all_vals): # automatic flatten only one element array outputs[x] = [val[0] for val in outputs[x]] elif len(outputs[x].shape) > 1: - outputs[x] = [ - json.dumps(val, cls=numpy_utils.NumpyEncoder) - for val in outputs[x] - ] + outputs[x] = [json.dumps(val, cls=numpy_utils.NumpyEncoder) for val in outputs[x]] for k in self._reserved_cols: if k in all_vals and all_vals[k].dtype == np.object: - all_vals[k] = [ - val.decode('utf-8', errors='ignore') for val in all_vals[k] - ] + all_vals[k] = [val.decode('utf-8', errors='ignore') for val in all_vals[k]] ts2 = time.time() - reserve_vals = self._get_reserve_vals( - self._reserved_cols, self._output_cols, all_vals, outputs - ) + reserve_vals = self._get_reserve_vals(self._reserved_cols, self._output_cols, all_vals, outputs) outputs = [x for x in zip(*reserve_vals)] logging.info('predict size: %s' % len(outputs)) self._write_lines(table_writer, outputs) ts3 = time.time() progress += 1 - sum_t0 += (ts1 - ts0) - sum_t1 += (ts2 - ts1) - sum_t2 += (ts3 - ts2) + sum_t0 += ts1 - ts0 + sum_t1 += ts2 - ts1 + sum_t2 += ts3 - ts2 except self.out_of_range_exception: break if progress % 100 == 0: - logging.info( - 'progress: batch_num=%d sample_num=%d' % - (progress, progress * batch_size) - ) - logging.info( - 'time_stats: read: %.2f predict: %.2f write: %.2f' % - (sum_t0, sum_t1, sum_t2) - ) - logging.info( - 'Final_time_stats: read: %.2f predict: %.2f write: %.2f' % - (sum_t0, sum_t1, sum_t2) - ) + logging.info('progress: batch_num=%d sample_num=%d' % (progress, progress * batch_size)) + logging.info('time_stats: read: %.2f predict: %.2f write: %.2f' % (sum_t0, sum_t1, sum_t2)) + logging.info('Final_time_stats: read: %.2f predict: %.2f write: %.2f' % (sum_t0, sum_t1, sum_t2)) table_writer.close() self.load_to_table(output_path, slice_num, slice_id) logging.info('Predict %s done.' % input_path) @@ -607,9 +564,11 @@ def predict(self, input_data_dict_list, output_names=None, batch_size=1): """ num_example = len(input_data_dict_list) assert num_example > 0, 'input data should not be an empty list' - assert isinstance(input_data_dict_list[0], dict) or \ - isinstance(input_data_dict_list[0], list) or \ - isinstance(input_data_dict_list[0], str), 'input is not a list or dict or str' + assert ( + isinstance(input_data_dict_list[0], dict) + or isinstance(input_data_dict_list[0], list) + or isinstance(input_data_dict_list[0], str) + ), 'input is not a list or dict or str' if batch_size > 0: num_batches = int(math.ceil(float(num_example) / batch_size)) else: @@ -618,9 +577,7 @@ def predict(self, input_data_dict_list, output_names=None, batch_size=1): outputs_list = [] for batch_idx in range(num_batches): - batch_data_list = input_data_dict_list[batch_idx * - batch_size:(batch_idx + 1) * - batch_size] + batch_data_list = input_data_dict_list[batch_idx * batch_size : (batch_idx + 1) * batch_size] feed_dict = self.batch(batch_data_list) outputs = self._predictor_impl.predict(feed_dict, output_names) for idx in range(len(batch_data_list)): @@ -638,9 +595,13 @@ def batch(self, data_list): for key in data: batch_input[key].append(data[key]) elif isinstance(data, list): - assert len(self._predictor_impl.input_names) == len(data), \ - 'input fields number incorrect, should be %d, but %d' \ - % (len(self._predictor_impl.input_names), len(data)) + assert len(self._predictor_impl.input_names) == len(data), ( + 'input fields number incorrect, should be %d, but %d' + % ( + len(self._predictor_impl.input_names), + len(data), + ) + ) for key, v in zip(self._predictor_impl.input_names, data): if key != '': batch_input[key].append(v) diff --git a/easy_rec/python/inference/processor/test.py b/easy_rec/python/inference/processor/test.py index af3391644..413719560 100644 --- a/easy_rec/python/inference/processor/test.py +++ b/easy_rec/python/inference/processor/test.py @@ -14,9 +14,7 @@ from easy_rec.python.protos import dataset_pb2, pipeline_pb2, tf_predict_pb2 -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') PROCESSOR_VERSION = 'LaRec-0.9.5d-b1b1604-TF-2.5.0-Linux' PROCESSOR_FILE = PROCESSOR_VERSION + '.tar.gz' @@ -49,61 +47,37 @@ def build_array_proto(array_proto, data, dtype): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( - '--input_path', type=str, default=None, help='input data path' - ) - parser.add_argument( - '--output_path', type=str, default=None, help='output data path' - ) - parser.add_argument( - '--libc_path', type=str, default='/lib64/libc.so.6', help='libc.so.6 path' - ) - parser.add_argument( - '--saved_model_dir', type=str, default=None, help='saved model directory' - ) - parser.add_argument( - '--test_dir', type=str, default=None, help='test directory' - ) + parser.add_argument('--input_path', type=str, default=None, help='input data path') + parser.add_argument('--output_path', type=str, default=None, help='output data path') + parser.add_argument('--libc_path', type=str, default='/lib64/libc.so.6', help='libc.so.6 path') + parser.add_argument('--saved_model_dir', type=str, default=None, help='saved model directory') + parser.add_argument('--test_dir', type=str, default=None, help='test directory') args = parser.parse_args() if not os.path.exists('processor'): os.mkdir('processor') if not os.path.exists(PROCESSOR_ENTRY_LIB): if not os.path.exists('processor/' + PROCESSOR_FILE): - subprocess.check_output( - 'wget %s -O processor/%s' % (PROCESSOR_URL, PROCESSOR_FILE), - shell=True - ) - subprocess.check_output( - 'cd processor && tar -zvxf %s' % PROCESSOR_FILE, shell=True - ) - assert os.path.exists( - PROCESSOR_ENTRY_LIB - ), 'invalid processor path: %s' % PROCESSOR_ENTRY_LIB + subprocess.check_output('wget %s -O processor/%s' % (PROCESSOR_URL, PROCESSOR_FILE), shell=True) + subprocess.check_output('cd processor && tar -zvxf %s' % PROCESSOR_FILE, shell=True) + assert os.path.exists(PROCESSOR_ENTRY_LIB), 'invalid processor path: %s' % PROCESSOR_ENTRY_LIB assert os.path.exists(args.libc_path), '%s does not exist' % args.libc_path - assert args.saved_model_dir is not None and os.path.isdir( - args.saved_model_dir - ), '%s is not a valid directory' % args.saved_model_dir - assert args.input_path is not None and os.path.exists( - args.input_path - ), '%s does not exist' % args.input_path + assert args.saved_model_dir is not None and os.path.isdir(args.saved_model_dir), ( + '%s is not a valid directory' % args.saved_model_dir + ) + assert args.input_path is not None and os.path.exists(args.input_path), '%s does not exist' % args.input_path assert args.output_path is not None, 'output_path is not set' pipeline_config = pipeline_pb2.EasyRecConfig() - pipeline_config_path = os.path.join( - args.saved_model_dir, 'assets/pipeline.config' - ) + pipeline_config_path = os.path.join(args.saved_model_dir, 'assets/pipeline.config') with open(pipeline_config_path) as fin: config_str = fin.read() text_format.Merge(config_str, pipeline_config) data_config = pipeline_config.data_config - input_fields = [ - [] for x in data_config.input_fields - if x.input_name not in data_config.label_fields - ] + input_fields = [[] for x in data_config.input_fields if x.input_name not in data_config.label_fields] with open(args.input_path, 'r') as fin: for line_str in fin: @@ -116,8 +90,9 @@ def build_array_proto(array_proto, data, dtype): req.signature_name = 'serving_default' for i in range(len(input_fields)): build_array_proto( - req.inputs[data_config.input_fields[i + 1].input_name], input_fields[i], - data_config.input_fields[i + 1].input_type + req.inputs[data_config.input_fields[i + 1].input_name], + input_fields[i], + data_config.input_fields[i + 1].input_type, ) tf_predictor = ctypes.cdll.LoadLibrary(PROCESSOR_ENTRY_LIB) @@ -130,9 +105,7 @@ def build_array_proto(array_proto, data, dtype): # last_step could be greater than num_steps for sync_replicas: false train_dir = os.path.dirname(args.saved_model_dir.strip('/')) - all_models = glob.glob( - os.path.join(args.test_dir, 'train/model.ckpt-*.index') - ) + all_models = glob.glob(os.path.join(args.test_dir, 'train/model.ckpt-*.index')) iters = [int(x.split('-')[-1].replace('.index', '')) for x in all_models] iters.sort() last_step = iters[-1] @@ -142,16 +115,10 @@ def build_array_proto(array_proto, data, dtype): dense_step = ctypes.c_int(0) start_ts = time.time() while sparse_step.value < last_step or dense_step.value < last_step: - tf_predictor.saved_model_step( - ctypes.c_void_p(handle), ctypes.byref(sparse_step), - ctypes.byref(dense_step) - ) + tf_predictor.saved_model_step(ctypes.c_void_p(handle), ctypes.byref(sparse_step), ctypes.byref(dense_step)) time.sleep(1) if time.time() - start_ts > 300: - logging.warning( - 'could not reach last_step, sparse_step=%d dense_step=%d' % - (sparse_step.value, dense_step.value) - ) + logging.warning('could not reach last_step, sparse_step=%d dense_step=%d' % (sparse_step.value, dense_step.value)) break data_bin = req.SerializeToString() @@ -163,8 +130,10 @@ def build_array_proto(array_proto, data, dtype): tf_predictor.saved_model_predict.restype = ctypes.c_void_p out_len = ctypes.c_int(0) res_p = tf_predictor.saved_model_predict( - ctypes.c_void_p(handle), data_bin, ctypes.c_int32(len(data_bin)), - ctypes.byref(out_len) + ctypes.c_void_p(handle), + data_bin, + ctypes.c_int32(len(data_bin)), + ctypes.byref(out_len), ) res_bytes = bytearray(ctypes.string_at(res_p, out_len)) res = tf_predict_pb2.PredictResponse() diff --git a/easy_rec/python/inference/vector_retrieve.py b/easy_rec/python/inference/vector_retrieve.py index 4fc6a268e..19b50f87e 100644 --- a/easy_rec/python/inference/vector_retrieve.py +++ b/easy_rec/python/inference/vector_retrieve.py @@ -21,7 +21,6 @@ class VectorRetrieve(object): - def __init__( self, query_table, @@ -34,7 +33,7 @@ def __init__( nlist=10, nprobe=2, distance=1, - m=8 + m=8, ): """Retrieve top n neighbours by query vector. @@ -73,16 +72,12 @@ def __call__(self, top_n, task_index, task_count, *args, **kwargs): g.node( self.doc_table, 'doc', - decoder=gl.Decoder( - attr_types=['float'] * self.ndim, attr_delimiter=self.delimiter - ), - option=self.knn_option + decoder=gl.Decoder(attr_types=['float'] * self.ndim, attr_delimiter=self.delimiter), + option=self.knn_option, ) g.init(task_index=task_index, task_count=task_count) - query_reader = common_io.table.TableReader( - self.query_table, slice_id=task_index, slice_count=task_count - ) + query_reader = common_io.table.TableReader(self.query_table, slice_id=task_index, slice_count=task_count) num_records = query_reader.get_row_count() total_batch_num = num_records // self.batch_size + 1.0 batch_num = 0 @@ -90,31 +85,19 @@ def __call__(self, top_n, task_index, task_count, *args, **kwargs): print('total_batch_num: {}'.format(total_batch_num)) print('output_table: {}'.format(self.out_table)) - output_table_writer = common_io.table.TableWriter( - self.out_table, task_index - ) + output_table_writer = common_io.table.TableWriter(self.out_table, task_index) count = 0 while True: try: - batch_query_nodes, batch_query_feats = zip( - *query_reader.read(self.batch_size, allow_smaller_final_batch=True) - ) + batch_query_nodes, batch_query_feats = zip(*query_reader.read(self.batch_size, allow_smaller_final_batch=True)) batch_num += 1.0 - print( - '{} process: {:.2f}'.format( - datetime.now().time(), batch_num / total_batch_num - ) - ) + print('{} process: {:.2f}'.format(datetime.now().time(), batch_num / total_batch_num)) feats = to_np_array(batch_query_feats, self.delimiter) rt_ids, rt_dists = g.search('doc', feats, gl.KnnOption(k=top_n)) - for query_node, nodes, dists in zip( - batch_query_nodes, rt_ids, rt_dists - ): + for query_node, nodes, dists in zip(batch_query_nodes, rt_ids, rt_dists): query = np.array([query_node] * len(nodes), dtype='int64') - output_table_writer.write( - zip(query, nodes, dists), (0, 1, 2), allow_type_cast=False - ) + output_table_writer.write(zip(query, nodes, dists), (0, 1, 2), allow_type_cast=False) count += 1 if np.mod(count, 100) == 0: print('write ', count, ' query nodes totally') @@ -131,5 +114,5 @@ def __call__(self, top_n, task_index, task_count, *args, **kwargs): def to_np_array(batch_query_feats, attr_delimiter): return np.array( [map(float, feat.split(attr_delimiter)) for feat in batch_query_feats], - dtype='float32' + dtype='float32', ) diff --git a/easy_rec/python/input/batch_tfrecord_input.py b/easy_rec/python/input/batch_tfrecord_input.py index 74d7ea192..f739f0329 100644 --- a/easy_rec/python/input/batch_tfrecord_input.py +++ b/easy_rec/python/input/batch_tfrecord_input.py @@ -27,46 +27,48 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(BatchTFRecordInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) - assert data_config.HasField( - 'n_data_batch_tfrecord' - ), 'Need to set n_data_batch_tfrecord in config.' + assert data_config.HasField('n_data_batch_tfrecord'), 'Need to set n_data_batch_tfrecord in config.' self._input_shapes = [x.input_shape for x in data_config.input_fields] self.feature_desc = {} for x, t, d, s in zip( - self._input_fields, self._input_field_types, self._input_field_defaults, - self._input_shapes + self._input_fields, + self._input_field_types, + self._input_field_defaults, + self._input_shapes, ): d = self.get_type_defaults(t, d) t = get_tf_type(t) - self.feature_desc[x] = tf.io.FixedLenSequenceFeature( - dtype=t, shape=s, allow_missing=True - ) + self.feature_desc[x] = tf.io.FixedLenSequenceFeature(dtype=t, shape=s, allow_missing=True) def _parse_tfrecord(self, example): try: - _, features, _ = tf.parse_sequence_example( - example, sequence_features=self.feature_desc - ) + _, features, _ = tf.parse_sequence_example(example, sequence_features=self.feature_desc) except AttributeError: - _, features, _ = tf.io.parse_sequence_example( - example, sequence_features=self.feature_desc - ) + _, features, _ = tf.io.parse_sequence_example(example, sequence_features=self.feature_desc) # Below code will reduce one dimension when the data dimension > 2. features = dict( ( key, tf.reshape( - value, [ + value, + [ -1, - ] + [x for i, x in enumerate(value.shape) if i not in (0, 1)] - ) - ) for (key, value) in features.items() + ] + + [x for i, x in enumerate(value.shape) if i not in (0, 1)], + ), + ) + for (key, value) in features.items() ) return features @@ -81,9 +83,7 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls data_compression_type = self._data_config.data_compression_type if mode == tf.estimator.ModeKeys.TRAIN: - logging.info( - 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.shuffle: # shuffle input files @@ -92,47 +92,36 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data. - TFRecordDataset(x, compression_type=data_compression_type), + lambda x: tf.data.TFRecordDataset(x, compression_type=data_compression_type), cycle_length=parallel_num, - num_parallel_calls=parallel_num + num_parallel_calls=parallel_num, ) dataset = dataset.shard(self._task_num, self._task_index) if self._data_config.shuffle: dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info( - 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) - dataset = tf.data.TFRecordDataset( - file_paths, compression_type=data_compression_type - ) + logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + dataset = tf.data.TFRecordDataset(file_paths, compression_type=data_compression_type) dataset = dataset.repeat(1) # We read n data from tfrecord one time. cur_batch = self._data_config.batch_size // self._data_config.n_data_batch_tfrecord cur_batch = max(1, cur_batch) dataset = dataset.batch(cur_batch) - dataset = dataset.map( - self._parse_tfrecord, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(self._parse_tfrecord, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/criteo_binary_reader.py b/easy_rec/python/input/criteo_binary_reader.py index 26b1c8d95..00bd14708 100644 --- a/easy_rec/python/input/criteo_binary_reader.py +++ b/easy_rec/python/input/criteo_binary_reader.py @@ -13,7 +13,6 @@ class BinaryDataset: - def __init__( self, label_bins, @@ -36,39 +35,27 @@ def __init__( self._batch_size = batch_size - self._compute_global_start_pos( - total_sample_num, batch_size, global_rank, global_size, drop_last - ) + self._compute_global_start_pos(total_sample_num, batch_size, global_rank, global_size, drop_last) self._label_file_arr = [None for _ in self._sample_num_arr] self._dense_file_arr = [None for _ in self._sample_num_arr] self._category_file_arr = [None for _ in self._sample_num_arr] for tmp_file_id in range(self._start_file_id, self._end_file_id + 1): - self._label_file_arr[tmp_file_id] = os.open( - label_bins[tmp_file_id], os.O_RDONLY - ) - self._dense_file_arr[tmp_file_id] = os.open( - dense_bins[tmp_file_id], os.O_RDONLY - ) - self._category_file_arr[tmp_file_id] = os.open( - category_bins[tmp_file_id], os.O_RDONLY - ) + self._label_file_arr[tmp_file_id] = os.open(label_bins[tmp_file_id], os.O_RDONLY) + self._dense_file_arr[tmp_file_id] = os.open(dense_bins[tmp_file_id], os.O_RDONLY) + self._category_file_arr[tmp_file_id] = os.open(category_bins[tmp_file_id], os.O_RDONLY) self._prefetch = min(prefetch, self._num_entries) self._prefetch_queue = queue.Queue() - self._executor = concurrent.futures.ThreadPoolExecutor( - max_workers=self._prefetch - ) + self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=self._prefetch) self._os_close_func = os.close - def _compute_global_start_pos( - self, total_sample_num, batch_size, global_rank, global_size, drop_last - ): + def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, global_size, drop_last): # ensure all workers have the same number of samples - avg_sample_num = (total_sample_num // global_size) - res_num = (total_sample_num % global_size) + avg_sample_num = total_sample_num // global_size + res_num = total_sample_num % global_size self._num_samples = avg_sample_num if res_num > 0: self._num_samples += 1 @@ -85,10 +72,7 @@ def _compute_global_start_pos( if not drop_last and (self._num_samples % batch_size != 0): self._num_entries += 1 self._last_batch_size = self._num_samples % batch_size - logging.info( - 'num_batches = %d num_samples = %d' % - (self._num_entries, self._num_samples) - ) + logging.info('num_batches = %d num_samples = %d' % (self._num_entries, self._num_samples)) start_file_id = 0 curr_pos = 0 @@ -98,10 +82,7 @@ def _compute_global_start_pos( self._start_file_id = start_file_id self._start_file_pos = global_start_pos - curr_pos - logging.info( - 'start_file_id = %d start_file_pos = %d' % - (start_file_id, self._start_file_pos) - ) + logging.info('start_file_id = %d start_file_pos = %d' % (start_file_id, self._start_file_pos)) # find the start of each batch self._start_pos_arr = np.zeros([self._num_entries, 2], dtype=np.uint32) @@ -113,26 +94,19 @@ def _compute_global_start_pos( # the last batch if batch_id == self._num_entries: tmp_start_pos += self._last_batch_size - while start_file_id < len( - self._sample_num_arr - ) and tmp_start_pos > self._sample_num_arr[start_file_id]: + while start_file_id < len(self._sample_num_arr) and tmp_start_pos > self._sample_num_arr[start_file_id]: tmp_start_pos -= self._sample_num_arr[start_file_id] start_file_id += 1 else: tmp_start_pos += batch_size - while start_file_id < len( - self._sample_num_arr - ) and tmp_start_pos >= self._sample_num_arr[start_file_id]: + while start_file_id < len(self._sample_num_arr) and tmp_start_pos >= self._sample_num_arr[start_file_id]: tmp_start_pos -= self._sample_num_arr[start_file_id] start_file_id += 1 self._end_file_id = start_file_id self._end_file_pos = tmp_start_pos - logging.info( - 'end_file_id = %d end_file_pos = %d' % - (self._end_file_id, self._end_file_pos) - ) + logging.info('end_file_id = %d end_file_pos = %d' % (self._end_file_id, self._end_file_pos)) def __del__(self): for f in self._label_file_arr: @@ -160,9 +134,7 @@ def __getitem__(self, idx): self._prefetch_queue.put(self._executor.submit(self._get, (i))) if idx < (self._num_entries - self._prefetch): - self._prefetch_queue.put( - self._executor.submit(self._get, (idx + self._prefetch)) - ) + self._prefetch_queue.put(self._executor.submit(self._get, (idx + self._prefetch))) return self._prefetch_queue.get().result() @@ -176,38 +148,28 @@ def _get(self, idx): label_read_arr = [] dense_read_arr = [] cate_read_arr = [] - while total_read_num < self._batch_size and curr_file_id < len( - self._sample_num_arr - ): - tmp_read_num = min( - end_read_pos, self._sample_num_arr[curr_file_id] - ) - start_read_pos - - label_raw_data = os.pread( - self._label_file_arr[curr_file_id], 4 * tmp_read_num, - start_read_pos * 4 - ) - tmp_lbl_np = np.frombuffer(label_raw_data, - dtype=np.int32).reshape([tmp_read_num, 1]) + while total_read_num < self._batch_size and curr_file_id < len(self._sample_num_arr): + tmp_read_num = min(end_read_pos, self._sample_num_arr[curr_file_id]) - start_read_pos + + label_raw_data = os.pread(self._label_file_arr[curr_file_id], 4 * tmp_read_num, start_read_pos * 4) + tmp_lbl_np = np.frombuffer(label_raw_data, dtype=np.int32).reshape([tmp_read_num, 1]) label_read_arr.append(tmp_lbl_np) dense_raw_data = os.pread( - self._dense_file_arr[curr_file_id], 52 * tmp_read_num, - start_read_pos * 52 - ) - part_dense_np = np.frombuffer(dense_raw_data, dtype=np.float32).reshape( - [tmp_read_num, 13] + self._dense_file_arr[curr_file_id], + 52 * tmp_read_num, + start_read_pos * 52, ) + part_dense_np = np.frombuffer(dense_raw_data, dtype=np.float32).reshape([tmp_read_num, 13]) # part_dense_np = np.log(part_dense_np + 3, dtype=np.float32) dense_read_arr.append(part_dense_np) category_raw_data = os.pread( - self._category_file_arr[curr_file_id], 104 * tmp_read_num, - start_read_pos * 104 - ) - part_cate_np = np.frombuffer(category_raw_data, dtype=np.uint32).reshape( - [tmp_read_num, 26] + self._category_file_arr[curr_file_id], + 104 * tmp_read_num, + start_read_pos * 104, ) + part_cate_np = np.frombuffer(category_raw_data, dtype=np.uint32).reshape([tmp_read_num, 26]) cate_read_arr.append(part_cate_np) curr_file_id += 1 @@ -234,17 +196,11 @@ def _get(self, idx): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( - '--batch_size', type=int, default=1024, help='batch_size' - ) - parser.add_argument( - '--dataset_dir', type=str, default='./', help='dataset_dir' - ) + parser.add_argument('--batch_size', type=int, default=1024, help='batch_size') + parser.add_argument('--dataset_dir', type=str, default='./', help='dataset_dir') parser.add_argument('--task_num', type=int, default=1, help='task number') parser.add_argument('--task_index', type=int, default=0, help='task index') - parser.add_argument( - '--prefetch_size', type=int, default=10, help='prefetch size' - ) + parser.add_argument('--prefetch_size', type=int, default=10, help='prefetch size') args = parser.parse_args() batch_size = args.batch_size @@ -279,11 +235,8 @@ def _get(self, idx): start_time = time.time() if step == 1000: logging.info('1000 steps time = %.3f' % (time.time() - start_time)) + logging.info('total_steps = %d total_time = %.3f' % (step + 1, time.time() - start_time)) logging.info( - 'total_steps = %d total_time = %.3f' % - (step + 1, time.time() - start_time) - ) - logging.info( - 'final step[%d] dense_shape=%s category_shape=%s labels_shape=%s' % - (step, dense.shape, category.shape, labels.shape) + 'final step[%d] dense_shape=%s category_shape=%s labels_shape=%s' + % (step, dense.shape, category.shape, labels.shape) ) diff --git a/easy_rec/python/input/criteo_input.py b/easy_rec/python/input/criteo_input.py index ae345daf4..0fe0d758e 100644 --- a/easy_rec/python/input/criteo_input.py +++ b/easy_rec/python/input/criteo_input.py @@ -12,7 +12,6 @@ class CriteoInput(Input): - def __init__( self, data_config, @@ -21,21 +20,29 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(CriteoInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) all_label_paths = [] all_dense_paths = [] all_category_paths = [] if input_path is not None: - assert len(input_path.label_path) == len(input_path.dense_path) and \ - len(input_path.label_path) == len(input_path.category_path), \ - 'label_path_num(%d), dense_path_num(%d), category_path_num(%d) must be the same' % \ - (len(input_path.label_path), len(input_path.dense_path), len(input_path.category_path)) + assert len(input_path.label_path) == len(input_path.dense_path) and len(input_path.label_path) == len( + input_path.category_path + ), 'label_path_num(%d), dense_path_num(%d), category_path_num(%d) must be the same' % ( + len(input_path.label_path), + len(input_path.dense_path), + len(input_path.category_path), + ) for label_path, dense_path, category_path in zip( input_path.label_path, input_path.dense_path, input_path.category_path @@ -43,10 +50,10 @@ def __init__( label_paths = tf.gfile.Glob(input_path.label_path) dense_paths = tf.gfile.Glob(input_path.dense_path) category_paths = tf.gfile.Glob(input_path.category_path) - assert len(label_paths) == len(dense_paths) and len(label_paths) == \ - len(category_paths), 'label_path(%s) dense_path(%s) category_path(%s) ' + \ - 'matched different number of files(%d %d %d)' % ( - len(label_paths), len(dense_paths), len(category_paths)) + assert len(label_paths) == len(dense_paths) and len(label_paths) == len(category_paths), ( + 'label_path(%s) dense_path(%s) category_path(%s) ' + + 'matched different number of files(%d %d %d)' % (len(label_paths), len(dense_paths), len(category_paths)) + ) label_paths.sort() dense_paths.sort() category_paths.sort() @@ -63,7 +70,7 @@ def __init__( self._batch_size, prefetch=self._prefetch_size, global_rank=self._task_index, - global_size=self._task_num + global_size=self._task_num, ) else: self._binary_reader = None @@ -94,25 +101,19 @@ def _build(self, mode, params): self._sample_generator, output_types=(tf.float32, tf.int32, tf.int32), output_shapes=( - tf.TensorShape([None, - 13]), tf.TensorShape([None, - 26]), tf.TensorShape([None]) - ) + tf.TensorShape([None, 13]), + tf.TensorShape([None, 26]), + tf.TensorShape([None]), + ), ) num_parallel_calls = self._data_config.num_parallel_calls - dataset = dataset.map( - self._to_fea_dict, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(self._to_fea_dict, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/csv_input.py b/easy_rec/python/input/csv_input.py index ad25dcb09..08397e0ce 100644 --- a/easy_rec/python/input/csv_input.py +++ b/easy_rec/python/input/csv_input.py @@ -15,7 +15,6 @@ class CSVInput(Input): - def __init__( self, data_config, @@ -24,19 +23,23 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(CSVInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) self._with_header = data_config.with_header self._field_names = None def _parse_csv(self, line): record_defaults = [ - self.get_type_defaults(t, v) - for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) for t, v in zip(self._input_field_types, self._input_field_defaults) ] if self._field_names: @@ -47,32 +50,38 @@ def _parse_csv(self, line): tid = self._input_fields.index(field_name) record_defaults.append( self.get_type_defaults( - self._input_field_types[tid], self._input_field_defaults[tid] + self._input_field_types[tid], + self._input_field_defaults[tid], ) ) else: record_defaults.append('') - check_list = [ - tf.py_func( - check_split, [ - line, self._data_config.separator, - len(record_defaults), self._check_mode - ], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_split, + [ + line, + self._data_config.separator, + len(record_defaults), + self._check_mode, + ], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): fields = tf.decode_csv( line, field_delim=self._data_config.separator, record_defaults=record_defaults, - name='decode_csv' + name='decode_csv', ) if self._field_names is not None: - fields = [ - fields[self._field_names.index(x)] for x in self._input_fields - ] + fields = [fields[self._field_names.index(x)] for x in self._input_fields] # filter only valid fields inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids} @@ -92,9 +101,7 @@ def _build(self, mode, params): file_paths.append(x) assert len(file_paths) > 0, 'match no files with %s' % self._input_path - assert not file_paths[0].endswith( - '.tar.gz' - ), 'could only support .csv or .gz(not .tar.gz) files.' + assert not file_paths[0].endswith('.tar.gz'), 'could only support .csv or .gz(not .tar.gz) files.' compression_type = 'GZIP' if file_paths[0].endswith('.gz') else '' if compression_type: @@ -110,9 +117,7 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info( - 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -126,11 +131,9 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset( - x, compression_type=compression_type - ).skip(int(self._with_header)), + lambda x: tf.data.TextLineDataset(x, compression_type=compression_type).skip(int(self._with_header)), cycle_length=parallel_num, - num_parallel_calls=parallel_num + num_parallel_calls=parallel_num, ) if not self._data_config.file_shard: @@ -140,53 +143,41 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) elif self._task_num > 1: # For distribute evaluate dataset = tf.data.Dataset.from_tensor_slices(file_paths) parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset( - x, compression_type=compression_type - ).skip(int(self._with_header)), + lambda x: tf.data.TextLineDataset(x, compression_type=compression_type).skip(int(self._with_header)), cycle_length=parallel_num, - num_parallel_calls=parallel_num + num_parallel_calls=parallel_num, ) dataset = self._safe_shard(dataset) dataset = dataset.repeat(1) else: - logging.info( - 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset( - x, compression_type=compression_type - ).skip(int(self._with_header)), + lambda x: tf.data.TextLineDataset(x, compression_type=compression_type).skip(int(self._with_header)), cycle_length=parallel_num, - num_parallel_calls=parallel_num + num_parallel_calls=parallel_num, ) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map( - self._parse_csv, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(self._parse_csv, num_parallel_calls=num_parallel_calls) if self._data_config.ignore_error: dataset = dataset.apply(ignore_errors) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/csv_input_ex.py b/easy_rec/python/input/csv_input_ex.py index 8dd09fffa..88ac356e4 100644 --- a/easy_rec/python/input/csv_input_ex.py +++ b/easy_rec/python/input/csv_input_ex.py @@ -11,7 +11,6 @@ class CSVInputEx(CSVInput): - def __init__( self, data_config, @@ -20,17 +19,21 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(CSVInputEx, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) def _parse_csv(self, line): record_defaults = [ - self.get_type_defaults(t, v) - for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) for t, v in zip(self._input_field_types, self._input_field_defaults) ] def _check_data(line): @@ -38,45 +41,34 @@ def _check_data(line): if type(sep) != type(str): sep = sep.encode('utf-8') field_num = len(line[0].split(sep)) - assert field_num == len(record_defaults), \ - 'sep[%s] maybe invalid: field_num=%d, required_num=%d' % \ - (sep, field_num, len(record_defaults)) + assert field_num == len(record_defaults), 'sep[%s] maybe invalid: field_num=%d, required_num=%d' % ( + sep, + field_num, + len(record_defaults), + ) return True - fields = str_split_by_chr( - line, self._data_config.separator, skip_empty=False - ) + fields = str_split_by_chr(line, self._data_config.separator, skip_empty=False) tmp_fields = tf.reshape(fields.values, [-1, len(record_defaults)]) fields = [] for i in range(len(record_defaults)): if type(record_defaults[i]) == int: - fields.append( - tf.string_to_number( - tmp_fields[:, i], tf.int64, name='field_as_int_%d' % i - ) - ) + fields.append(tf.string_to_number(tmp_fields[:, i], tf.int64, name='field_as_int_%d' % i)) elif type(record_defaults[i]) in [float, np.float32, np.float64]: - fields.append( - tf.string_to_number( - tmp_fields[:, i], tf.float32, name='field_as_flt_%d' % i - ) - ) - elif type(record_defaults[i]) in [str, type(u''), bytes]: + fields.append(tf.string_to_number(tmp_fields[:, i], tf.float32, name='field_as_flt_%d' % i)) + elif type(record_defaults[i]) in [str, type(''), bytes]: fields.append(tmp_fields[:, i]) elif type(record_defaults[i]) == bool: fields.append( tf.logical_or( tf.equal(tmp_fields[:, i], 'True'), - tf.equal(tmp_fields[:, i], 'true') + tf.equal(tmp_fields[:, i], 'true'), ) ) else: assert 'invalid types: %s' % str(type(record_defaults[i])) - keep_ids = [ - self._input_fields.index(x) - for x in self._label_fields + self._effective_fields - ] + keep_ids = [self._input_fields.index(x) for x in self._label_fields + self._effective_fields] inputs = {self._input_fields[x]: fields[x] for x in keep_ids} return inputs diff --git a/easy_rec/python/input/csv_input_v2.py b/easy_rec/python/input/csv_input_v2.py index bd27df3d0..5f41cfa21 100644 --- a/easy_rec/python/input/csv_input_v2.py +++ b/easy_rec/python/input/csv_input_v2.py @@ -6,7 +6,6 @@ class CSVInputV2(Input): - def __init__( self, data_config, @@ -15,29 +14,31 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(CSVInputV2, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) def _build(self, mode, params): if type(self._input_path) != list: self._input_path = self._input_path.split(',') - assert len( - self._input_path - ) > 0, 'match no files with %s' % self._input_path + assert len(self._input_path) > 0, 'match no files with %s' % self._input_path if self._input_path[0].startswith('hdfs://'): # support hdfs input dataset = tf.data.TextLineDataset(self._input_path) else: num_epochs = self.num_epochs if mode == tf.estimator.ModeKeys.TRAIN else 1 - is_train = (mode == tf.estimator.ModeKeys.TRAIN) + is_train = mode == tf.estimator.ModeKeys.TRAIN record_defaults = [ - self.get_type_defaults(x, v) - for x, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(x, v) for x, v in zip(self._input_field_types, self._input_field_defaults) ] dataset = tf.data.experimental.make_csv_dataset( self._input_path, @@ -49,14 +50,12 @@ def _build(self, mode, params): num_epochs=num_epochs, shuffle=is_train and self._data_config.shuffle, num_parallel_reads=8, - sloppy=is_train + sloppy=is_train, ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.chief_redundant: - dataset = dataset.shard( - max(self._task_num - 1, 1), max(self._task_index - 1, 0) - ) + dataset = dataset.shard(max(self._task_num - 1, 1), max(self._task_index - 1, 0)) else: dataset = dataset.shard(self._task_num, self._task_index) else: @@ -67,9 +66,7 @@ def _build(self, mode, params): dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/datahub_input.py b/easy_rec/python/input/datahub_input.py index 706a1906d..b36db4f1d 100644 --- a/easy_rec/python/input/datahub_input.py +++ b/easy_rec/python/input/datahub_input.py @@ -26,13 +26,11 @@ from datahub import DataHub from datahub.exceptions import DatahubException from datahub.models import CursorType, RecordType + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) logging.getLogger('datahub.account').setLevel(logging.INFO) except Exception: - logging.warning( - 'DataHub is not installed[%s]. You can install it by: pip install pydatahub' - % traceback.format_exc() - ) + logging.warning('DataHub is not installed[%s]. You can install it by: pip install pydatahub' % traceback.format_exc()) DataHub = None @@ -47,16 +45,21 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(DataHubInput, self).__init__( - data_config, feature_config, '', task_index, task_num, check_mode, - pipeline_config + data_config, + feature_config, + '', + task_index, + task_num, + check_mode, + pipeline_config, ) if DataHub is None: logging.error( 'please install datahub: ', - 'pip install pydatahub ;Python 3.6 recommended' + 'pip install pydatahub ;Python 3.6 recommended', ) try: self._num_epoch = 0 @@ -74,17 +77,12 @@ def __init__( self._datahub = None except Exception as ex: logging.info('exception in init datahub: %s' % str(ex)) - pass self._offset_dict = {} if datahub_config: - shard_result = self._datahub.list_shard( - self._datahub_config.project, self._datahub_config.topic - ) + shard_result = self._datahub.list_shard(self._datahub_config.project, self._datahub_config.topic) shards = shard_result.shards self._all_shards = shards - self._shards = [ - shards[i] for i in range(len(shards)) if (i % task_num) == task_index - ] + self._shards = [shards[i] for i in range(len(shards)) if (i % task_num) == task_index] logging.info('all shards: %s' % str(self._shards)) offset_type = datahub_config.WhichOneof('offset') @@ -93,8 +91,11 @@ def __init__( for x in self._all_shards: ks = str(x.shard_id) cursor_result = self._datahub.get_cursor( - self._datahub_config.project, self._datahub_config.topic, ks, - CursorType.SYSTEM_TIME, ts + self._datahub_config.project, + self._datahub_config.topic, + ks, + CursorType.SYSTEM_TIME, + ts, ) logging.info('shard[%s] cursor = %s' % (ks, cursor_result)) self._offset_dict[ks] = cursor_result.cursor @@ -107,23 +108,19 @@ def __init__( self._dh_field_types = [] topic_info = self._datahub.get_topic( project_name=self._datahub_config.project, - topic_name=self._datahub_config.topic + topic_name=self._datahub_config.topic, ) for field in topic_info.record_schema.field_list: self._dh_field_names.append(field.name) self._dh_field_types.append(field.type.value) - assert len( - self._feature_fields - ) > 0, 'data_config.feature_fields are not set.' + assert len(self._feature_fields) > 0, 'data_config.feature_fields are not set.' for x in self._feature_fields: assert x in self._dh_field_names, 'feature_field[%s] is not in datahub' % x # feature column ids in datahub schema - self._dh_fea_ids = [ - self._dh_field_names.index(x) for x in self._feature_fields - ] + self._dh_fea_ids = [self._dh_field_names.index(x) for x in self._feature_fields] for x in self._label_fields: assert x in self._dh_field_names, 'label_field[%s] is not in datahub' % x @@ -135,27 +132,22 @@ def __init__( self._read_cnt = 32 if len(self._dh_fea_ids) > 1: - self._filter_fea_func = lambda record: ''.join( - [record.values[x] for x in self._dh_fea_ids] - ).split(chr(2))[1] == '-1024' + self._filter_fea_func = ( + lambda record: ''.join([record.values[x] for x in self._dh_fea_ids]).split(chr(2))[1] == '-1024' + ) else: dh_fea_id = self._dh_fea_ids[0] - self._filter_fea_func = lambda record: record.values[ - dh_fea_id].split(self._data_config.separator)[1] == '-1024' + self._filter_fea_func = lambda record: record.values[dh_fea_id].split(self._data_config.separator)[1] == '-1024' def _parse_record(self, *fields): field_dict = {} fields = list(fields) def _dump_offsets(): - all_offsets = { - x.shard_id: self._offset_dict[x.shard_id] - for x in self._shards if x.shard_id in self._offset_dict - } + all_offsets = {x.shard_id: self._offset_dict[x.shard_id] for x in self._shards if x.shard_id in self._offset_dict} return json.dumps(all_offsets) - field_dict[Input.DATA_OFFSET - ] = tf.py_func(_dump_offsets, [], dtypes.string) + field_dict[Input.DATA_OFFSET] = tf.py_func(_dump_offsets, [], dtypes.string) for x in self._label_fields: dh_id = self._dh_field_names.index(x) @@ -163,22 +155,15 @@ def _dump_offsets(): feature_inputs = self.get_feature_input_fields() # only for features, labels and sample_weight excluded - record_types = [ - t for x, t in zip(self._input_fields, self._input_field_types) - if x in feature_inputs - ] + record_types = [t for x, t in zip(self._input_fields, self._input_field_types) if x in feature_inputs] feature_num = len(record_types) - feature_fields = [ - fields[self._dh_field_names.index(x)] for x in self._feature_fields - ] + feature_fields = [fields[self._dh_field_names.index(x)] for x in self._feature_fields] feature = feature_fields[0] for fea_id in range(1, len(feature_fields)): feature = feature + self._data_config.separator + feature_fields[fea_id] - feature = tf.string_split( - feature, self._data_config.separator, skip_empty=False - ) + feature = tf.string_split(feature, self._data_config.separator, skip_empty=False) fields = tf.reshape(feature.values, [-1, feature_num]) @@ -236,16 +221,10 @@ def _datahub_generator(self): self._num_epoch += 1 try: - self._datahub.wait_shards_ready( - self._datahub_config.project, self._datahub_config.topic - ) - topic_result = self._datahub.get_topic( - self._datahub_config.project, self._datahub_config.topic - ) + self._datahub.wait_shards_ready(self._datahub_config.project, self._datahub_config.topic) + topic_result = self._datahub.get_topic(self._datahub_config.project, self._datahub_config.topic) if topic_result.record_type != RecordType.TUPLE: - logging.error( - 'datahub topic type(%s) illegal' % str(topic_result.record_type) - ) + logging.error('datahub topic type(%s) illegal' % str(topic_result.record_type)) record_schema = topic_result.record_schema tid = 0 @@ -257,35 +236,36 @@ def _datahub_generator(self): if shard_id not in self._offset_dict: cursor_result = self._datahub.get_cursor( - self._datahub_config.project, self._datahub_config.topic, shard_id, - CursorType.OLDEST + self._datahub_config.project, + self._datahub_config.topic, + shard_id, + CursorType.OLDEST, ) cursor = cursor_result.cursor else: cursor = self._offset_dict[shard_id] get_result = self._datahub.get_tuple_records( - self._datahub_config.project, self._datahub_config.topic, shard_id, - record_schema, cursor, self._read_cnt + self._datahub_config.project, + self._datahub_config.topic, + shard_id, + record_schema, + cursor, + self._read_cnt, ) count = get_result.record_count if count == 0: continue for row_id, record in enumerate(get_result.records): if self._is_data_empty(record): - logging.warning( - 'skip empty data record: %s' % self._dump_record(record) - ) + logging.warning('skip empty data record: %s' % self._dump_record(record)) continue if self._filter_fea_func is not None: if self._filter_fea_func(record): - logging.warning( - 'filter data record: %s' % self._dump_record(record) - ) + logging.warning('filter data record: %s' % self._dump_record(record)) continue yield tuple(list(record.values)) - if shard_id not in self._offset_dict or get_result.next_cursor > self._offset_dict[ - shard_id]: + if shard_id not in self._offset_dict or get_result.next_cursor > self._offset_dict[shard_id]: self._offset_dict[shard_id] = get_result.next_cursor except DatahubException as ex: logging.error('DatahubException: %s' % str(ex)) @@ -297,45 +277,34 @@ def _build(self, mode, params): assert self._datahub is not None, 'datahub_eval_input is not set' # get input types - list_types = [ - odps_util.odps_type_2_tf_type(x) for x in self._dh_field_types - ] + list_types = [odps_util.odps_type_2_tf_type(x) for x in self._dh_field_types] list_types = tuple(list_types) - list_shapes = [ - tf.TensorShape([]) for x in range(0, len(self._dh_field_types)) - ] + list_shapes = [tf.TensorShape([]) for x in range(0, len(self._dh_field_types))] list_shapes = tuple(list_shapes) # read datahub dataset = tf.data.Dataset.from_generator( - self._datahub_generator, - output_types=list_types, - output_shapes=list_shapes + self._datahub_generator, output_types=list_types, output_shapes=list_shapes ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.shuffle: dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map( - self._parse_record, - num_parallel_calls=self._data_config.num_parallel_calls - ) + dataset = dataset.map(self._parse_record, num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/dummy_input.py b/easy_rec/python/input/dummy_input.py index d12524fc0..9ede8ba89 100644 --- a/easy_rec/python/input/dummy_input.py +++ b/easy_rec/python/input/dummy_input.py @@ -25,11 +25,16 @@ def __init__( task_num=1, check_mode=False, pipeline_config=None, - input_vals={} + input_vals={}, ): super(DummyInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) self._input_vals = input_vals @@ -45,9 +50,7 @@ def _build(self, mode, params): label tensor dict """ features = {} - for field, field_type, def_val in zip( - self._input_fields, self._input_field_types, self._input_field_defaults - ): + for field, field_type, def_val in zip(self._input_fields, self._input_field_types, self._input_field_defaults): tf_type = get_tf_type(field_type) def_val = self.get_type_defaults(field_type, default_val=def_val) diff --git a/easy_rec/python/input/hive_input.py b/easy_rec/python/input/hive_input.py index b970d157d..4e35ae751 100644 --- a/easy_rec/python/input/hive_input.py +++ b/easy_rec/python/input/hive_input.py @@ -19,11 +19,16 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(HiveInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) if input_path is None: return @@ -31,26 +36,19 @@ def __init__( self._feature_config = feature_config self._hive_config = input_path - hive_util = HiveUtils( - data_config=self._data_config, hive_config=self._hive_config - ) - self._input_hdfs_path = hive_util.get_table_location( - self._hive_config.table_name - ) - self._input_table_col_names, self._input_table_col_types = hive_util.get_all_cols( - self._hive_config.table_name - ) + hive_util = HiveUtils(data_config=self._data_config, hive_config=self._hive_config) + self._input_hdfs_path = hive_util.get_table_location(self._hive_config.table_name) + ( + self._input_table_col_names, + self._input_table_col_types, + ) = hive_util.get_all_cols(self._hive_config.table_name) def _parse_csv(self, line): record_defaults = [] for field_name in self._input_table_col_names: if field_name in self._input_fields: tid = self._input_fields.index(field_name) - record_defaults.append( - self.get_type_defaults( - self._input_field_types[tid], self._input_field_defaults[tid] - ) - ) + record_defaults.append(self.get_type_defaults(self._input_field_types[tid], self._input_field_defaults[tid])) else: record_defaults.append('') @@ -58,13 +56,14 @@ def _parse_csv(self, line): line, field_delim=self._data_config.separator, record_defaults=record_defaults, - name='decode_csv' + name='decode_csv', ) fields = [] for x in self._input_fields: assert x in self._input_table_col_names, 'Column %s not in Table %s.' % ( - x, self._hive_config.table_name + x, + self._hive_config.table_name, ) fields.append(tmp_fields[self._input_table_col_names.index(x)]) @@ -76,15 +75,11 @@ def _parse_csv(self, line): def _build(self, mode, params): file_paths = tf.gfile.Glob(os.path.join(self._input_hdfs_path, '*')) - assert len( - file_paths - ) > 0, 'match no files with %s' % self._hive_config.table_name + assert len(file_paths) > 0, 'match no files with %s' % self._hive_config.table_name num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info( - 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -100,7 +95,7 @@ def _build(self, mode, params): dataset = dataset.interleave( lambda x: tf.data.TextLineDataset(x), cycle_length=parallel_num, - num_parallel_calls=parallel_num + num_parallel_calls=parallel_num, ) if not self._data_config.file_shard: @@ -110,32 +105,24 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info( - 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map( - self._parse_csv, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(self._parse_csv, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/hive_parquet_input.py b/easy_rec/python/input/hive_parquet_input.py index 931187394..85162f79c 100644 --- a/easy_rec/python/input/hive_parquet_input.py +++ b/easy_rec/python/input/hive_parquet_input.py @@ -22,11 +22,16 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(HiveParquetInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) if input_path is None: return @@ -34,25 +39,22 @@ def __init__( self._feature_config = feature_config self._hive_config = input_path - hive_util = HiveUtils( - data_config=self._data_config, hive_config=self._hive_config - ) - input_hdfs_path = hive_util.get_table_location( - self._hive_config.table_name - ) - self._input_table_col_names, self._input_table_col_types = hive_util.get_all_cols( - self._hive_config.table_name - ) + hive_util = HiveUtils(data_config=self._data_config, hive_config=self._hive_config) + input_hdfs_path = hive_util.get_table_location(self._hive_config.table_name) + ( + self._input_table_col_names, + self._input_table_col_types, + ) = hive_util.get_all_cols(self._hive_config.table_name) self._all_hdfs_path = tf.gfile.Glob(os.path.join(input_hdfs_path, '*')) for x in self._input_fields: assert x in self._input_table_col_names, 'Column %s not in Table %s.' % ( - x, self._hive_config.table_name + x, + self._hive_config.table_name, ) self._record_defaults = [ - self.get_type_defaults(t, v) - for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) for t, v in zip(self._input_field_types, self._input_field_defaults) ] def _file_shard(self, file_paths, task_num, task_index): @@ -77,12 +79,8 @@ def _parquet_read(self): for k, v in zip(self._input_fields, self._record_defaults): df[k].fillna(v, inplace=True) - for start_idx in range( - 0, total_records_num, self._data_config.batch_size - ): - end_idx = min( - total_records_num, start_idx + self._data_config.batch_size - ) + for start_idx in range(0, total_records_num, self._data_config.batch_size): + end_idx = min(total_records_num, start_idx + self._data_config.batch_size) batch_data = df[start_idx:end_idx] inputs = [] for k in self._input_fields: @@ -106,20 +104,14 @@ def _build(self, mode, params): if len(self._all_hdfs_path) >= 2 * self._task_num: file_shard = True - self._input_hdfs_path = self._file_shard( - self._all_hdfs_path, self._task_num, self._task_index - ) + self._input_hdfs_path = self._file_shard(self._all_hdfs_path, self._task_num, self._task_index) else: file_shard = False self._input_hdfs_path = self._all_hdfs_path logging.info('input path: %s' % self._input_hdfs_path) - assert len( - self._input_hdfs_path - ) > 0, 'match no files with %s' % self._hive_config.table_name + assert len(self._input_hdfs_path) > 0, 'match no files with %s' % self._hive_config.table_name - dataset = tf.data.Dataset.from_generator( - self._parquet_read, output_types=list_type, output_shapes=list_shapes - ) + dataset = tf.data.Dataset.from_generator(self._parquet_read, output_types=list_type, output_shapes=list_shapes) if not file_shard: dataset = self._safe_shard(dataset) @@ -128,29 +120,25 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) - dataset = dataset.map( - self._parse_csv, num_parallel_calls=self._data_config.num_parallel_calls - ) + dataset = dataset.map(self._parse_csv, num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/hive_rtp_input.py b/easy_rec/python/input/hive_rtp_input.py index e1a1b46b6..b3d70cc61 100644 --- a/easy_rec/python/input/hive_rtp_input.py +++ b/easy_rec/python/input/hive_rtp_input.py @@ -21,11 +21,16 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(HiveRTPInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) if input_path is None: return @@ -33,27 +38,22 @@ def __init__( self._feature_config = feature_config self._hive_config = input_path - logging.info( - 'input_fields: %s label_fields: %s' % - (','.join(self._input_fields), ','.join(self._label_fields)) - ) + logging.info('input_fields: %s label_fields: %s' % (','.join(self._input_fields), ','.join(self._label_fields))) self._rtp_separator = self._data_config.rtp_separator if not isinstance(self._rtp_separator, str): self._rtp_separator = self._rtp_separator.encode('utf-8') logging.info('rtp separator = %s' % self._rtp_separator) - self._selected_cols = [c.strip() for c in self._data_config.selected_cols.split(',')] \ - if self._data_config.selected_cols else None - logging.info('select cols: %s' % self._selected_cols) - hive_util = HiveUtils( - data_config=self._data_config, hive_config=self._hive_config - ) - self._input_hdfs_path = hive_util.get_table_location( - self._hive_config.table_name - ) - self._input_table_col_names, self._input_table_col_types = hive_util.get_all_cols( - self._hive_config.table_name + self._selected_cols = ( + [c.strip() for c in self._data_config.selected_cols.split(',')] if self._data_config.selected_cols else None ) + logging.info('select cols: %s' % self._selected_cols) + hive_util = HiveUtils(data_config=self._data_config, hive_config=self._hive_config) + self._input_hdfs_path = hive_util.get_table_location(self._hive_config.table_name) + ( + self._input_table_col_names, + self._input_table_col_types, + ) = hive_util.get_all_cols(self._hive_config.table_name) def _parse_csv(self, line): non_feature_cols = self._label_fields @@ -63,11 +63,7 @@ def _parse_csv(self, line): for tid, field_name in enumerate(self._input_table_col_names): if field_name in self._selected_cols[:-1]: idx = self._input_fields.index(field_name) - record_defaults.append( - self.get_type_defaults( - self._input_field_types[idx], self._input_field_defaults[idx] - ) - ) + record_defaults.append(self.get_type_defaults(self._input_field_types[idx], self._input_field_defaults[idx])) else: record_defaults.append('') print('record_defaults: ', record_defaults) @@ -75,7 +71,7 @@ def _parse_csv(self, line): line, field_delim=self._rtp_separator, record_defaults=record_defaults, - name='decode_csv' + name='decode_csv', ) print('tmp_fields: ', tmp_fields) @@ -88,36 +84,32 @@ def _parse_csv(self, line): labels = fields[:-1] # only for features, labels and sample_weight excluded - record_types = [ - t for x, t in zip(self._input_fields, self._input_field_types) - if x not in non_feature_cols - ] + record_types = [t for x, t in zip(self._input_fields, self._input_field_types) if x not in non_feature_cols] feature_num = len(record_types) - check_list = [ - tf.py_func( - check_split, - [fields[-1], self._data_config.separator, - len(record_types)], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_split, + [fields[-1], self._data_config.separator, len(record_types)], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): - fields = tf.string_split( - fields[-1], self._data_config.separator, skip_empty=False - ) + fields = tf.string_split(fields[-1], self._data_config.separator, skip_empty=False) tmp_fields = tf.reshape(fields.values, [-1, feature_num]) rtp_record_defaults = [ - str(self.get_type_defaults(t, v)) for x, t, v in zip( - self._input_fields, self._input_field_types, self._input_field_defaults - ) if x not in non_feature_cols + str(self.get_type_defaults(t, v)) + for x, t, v in zip(self._input_fields, self._input_field_types, self._input_field_defaults) + if x not in non_feature_cols ] - fields = labels[len(self._label_fields):] + fields = labels[len(self._label_fields) :] for i in range(feature_num): - field = string_to_number( - tmp_fields[:, i], record_types[i], rtp_record_defaults[i], i - ) + field = string_to_number(tmp_fields[:, i], record_types[i], rtp_record_defaults[i], i) fields.append(field) field_keys = [x for x in self._input_fields if x not in self._label_fields] @@ -130,15 +122,11 @@ def _parse_csv(self, line): def _build(self, mode, params): file_paths = tf.gfile.Glob(os.path.join(self._input_hdfs_path, '*')) - assert len( - file_paths - ) > 0, 'match no files with %s' % self._hive_config.table_name + assert len(file_paths) > 0, 'match no files with %s' % self._hive_config.table_name num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info( - 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -154,7 +142,7 @@ def _build(self, mode, params): dataset = dataset.interleave( lambda x: tf.data.TextLineDataset(x), cycle_length=parallel_num, - num_parallel_calls=parallel_num + num_parallel_calls=parallel_num, ) if not self._data_config.file_shard: @@ -164,32 +152,24 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info( - 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map( - self._parse_csv, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(self._parse_csv, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/input.py b/easy_rec/python/input/input.py index cc025c213..66032b22a 100644 --- a/easy_rec/python/input/input.py +++ b/easy_rec/python/input/input.py @@ -14,24 +14,26 @@ from easy_rec.python.core import sampler as sampler_lib from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils import conditional, config_util, constant +from easy_rec.python.utils.check_utils import ( # NOQA + check_split, + check_string_to_number, +) from easy_rec.python.utils.expr_util import get_expression from easy_rec.python.utils.input_utils import get_type_defaults +from easy_rec.python.utils.load_class import ( # NOQA + get_register_class_meta, + load_by_path, +) from easy_rec.python.utils.tf_utils import get_tf_type -from easy_rec.python.utils.check_utils import check_split, check_string_to_number # NOQA -from easy_rec.python.utils.load_class import get_register_class_meta, load_by_path # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 _INPUT_CLASS_MAP = {} -_meta_type = get_register_class_meta( - _INPUT_CLASS_MAP, have_abstract_class=True -) +_meta_type = get_register_class_meta(_INPUT_CLASS_MAP, have_abstract_class=True) class Input(six.with_metaclass(_meta_type, object)): - DATA_OFFSET = 'DATA_OFFSET' def __init__( @@ -43,7 +45,7 @@ def __init__( task_num=1, check_mode=False, pipeline_config=None, - **kwargs + **kwargs, ): self._pipeline_config = pipeline_config self._data_config = data_config @@ -52,9 +54,7 @@ def __init__( # tf.estimator.ModeKeys.*, only available before # calling self._build self._mode = None - if pipeline_config is not None and pipeline_config.model_config.HasField( - 'ev_params' - ): + if pipeline_config is not None and pipeline_config.model_config.HasField('ev_params'): self._has_ev = True else: self._has_ev = False @@ -74,9 +74,7 @@ def __init__( self._input_fields = [x.input_name for x in data_config.input_fields] self._input_dims = [x.input_dim for x in data_config.input_fields] self._input_field_types = [x.input_type for x in data_config.input_fields] - self._input_field_defaults = [ - x.default_val for x in data_config.input_fields - ] + self._input_field_defaults = [x.default_val for x in data_config.input_fields] self._label_fields = list(data_config.label_fields) self._feature_fields = list(data_config.feature_fields) self._label_sep = list(data_config.label_sep) @@ -110,15 +108,12 @@ def __init__( self._normalizer_fn = {} for fc in self._feature_configs: for input_name in fc.input_names: - assert input_name in self._input_fields, 'invalid input_name in %s' % str( - fc - ) + assert input_name in self._input_fields, 'invalid input_name in %s' % str(fc) if input_name not in self._effective_fields: self._effective_fields.append(input_name) if fc.feature_type in [fc.TagFeature, fc.SequenceFeature]: - if fc.hash_bucket_size > 0 or len(fc.vocab_list - ) > 0 or fc.HasField('vocab_file'): + if fc.hash_bucket_size > 0 or len(fc.vocab_list) > 0 or fc.HasField('vocab_file'): self._multi_value_types[fc.input_names[0]] = tf.string self._multi_value_fields.add(fc.input_names[0]) else: @@ -133,8 +128,7 @@ def __init__( self._multi_value_fields.add(fc.input_names[0]) if fc.HasField('normalizer_fn'): - feature_name = fc.feature_name if fc.HasField('feature_name' - ) else fc.input_names[0] + feature_name = fc.feature_name if fc.HasField('feature_name') else fc.input_names[0] self._normalizer_fn[feature_name] = load_by_path(fc.normalizer_fn) # add sample weight to effective fields @@ -160,9 +154,7 @@ def __init__( model_name = model_config.WhichOneof('model') if model_name in {'mmoe', 'esmm', 'dbmtl', 'simple_multi_task', 'ple'}: model = getattr(model_config, model_name) - towers = [ - model.ctr_tower, model.cvr_tower - ] if model_name == 'esmm' else model.task_towers + towers = [model.ctr_tower, model.cvr_tower] if model_name == 'esmm' else model.task_towers for tower in towers: metrics = tower.metrics_set for metric in metrics: @@ -176,18 +168,12 @@ def __init__( if sid not in self._effective_fields: self._effective_fields.append(sid) - self._effective_fids = [ - self._input_fields.index(x) for x in self._effective_fields - ] + self._effective_fids = [self._input_fields.index(x) for x in self._effective_fields] # sort fids from small to large self._effective_fids = list(set(self._effective_fids)) - self._effective_fields = [ - self._input_fields[x] for x in self._effective_fids - ] + self._effective_fields = [self._input_fields[x] for x in self._effective_fids] - self._label_fids = [ - self._input_fields.index(x) for x in self._label_fields - ] + self._label_fids = [self._input_fields.index(x) for x in self._label_fields] # virtual fields generated by self._preprocess # which will be inputs to feature columns @@ -203,12 +189,8 @@ def __init__( def _load_label_fn(self, config): udf_class = config.user_define_fn - udf_path = config.user_define_fn_path if config.HasField( - 'user_define_fn_path' - ) else None - dtype = config.user_define_fn_res_type if config.HasField( - 'user_define_fn_res_type' - ) else None + udf_path = config.user_define_fn_path if config.HasField('user_define_fn_path') else None + dtype = config.user_define_fn_res_type if config.HasField('user_define_fn_res_type') else None if udf_path: if udf_path.startswith('oss://') or udf_path.startswith('hdfs://'): @@ -237,10 +219,7 @@ def num_epochs(self): return None def get_feature_input_fields(self): - return [ - x for x in self._input_fields - if x not in self._label_fields and x != self._data_config.sample_weight - ] + return [x for x in self._input_fields if x not in self._label_fields and x != self._data_config.sample_weight] def should_stop(self, curr_epoch): """Check whether have run enough num epochs.""" @@ -273,9 +252,9 @@ def create_multi_placeholders(self, export_config): effective_fids = list(self._effective_fids) else: effective_fids = [ - fid for fid in range(len(self._input_fields)) - if self._input_fields[fid] not in self._label_fields - and self._input_fields[fid] != sample_weight_field + fid + for fid in range(len(self._input_fields)) + if self._input_fields[fid] not in self._label_fields and self._input_fields[fid] != sample_weight_field ] inputs = {} @@ -288,14 +267,13 @@ def create_multi_placeholders(self, export_config): else: placeholder_name = 'input_%d' % fid if input_name in export_fields_name: - tf_type = self._multi_value_types[input_name] if input_name in self._multi_value_types \ - else get_tf_type(self._input_field_types[fid]) - logging.info( - 'multi value input_name: %s, dtype: %s' % (input_name, tf_type) - ) - finput = array_ops.placeholder( - tf_type, [None, None], name=placeholder_name + tf_type = ( + self._multi_value_types[input_name] + if input_name in self._multi_value_types + else get_tf_type(self._input_field_types[fid]) ) + logging.info('multi value input_name: %s, dtype: %s' % (input_name, tf_type)) + finput = array_ops.placeholder(tf_type, [None, None], name=placeholder_name) else: ftype = self._input_field_types[fid] tf_type = get_tf_type(ftype) @@ -308,12 +286,8 @@ def create_multi_placeholders(self, export_config): def create_placeholders(self, export_config): self._mode = tf.estimator.ModeKeys.PREDICT - inputs_placeholder = array_ops.placeholder( - tf.string, [None], name='features' - ) - input_vals = tf.string_split( - inputs_placeholder, self._data_config.separator, skip_empty=False - ).values + inputs_placeholder = array_ops.placeholder(tf.string, [None], name='features') + input_vals = tf.string_split(inputs_placeholder, self._data_config.separator, skip_empty=False).values sample_weight_field = '' if self._data_config.HasField('sample_weight'): @@ -322,22 +296,16 @@ def create_placeholders(self, export_config): if export_config.filter_inputs: effective_fids = list(self._effective_fids) logging.info( - 'number of effective inputs:%d, total number inputs: %d' % - (len(effective_fids), len(self._input_fields)) + 'number of effective inputs:%d, total number inputs: %d' % (len(effective_fids), len(self._input_fields)) ) else: effective_fids = [ - fid for fid in range(len(self._input_fields)) - if self._input_fields[fid] not in self._label_fields - and self._input_fields[fid] != sample_weight_field + fid + for fid in range(len(self._input_fields)) + if self._input_fields[fid] not in self._label_fields and self._input_fields[fid] != sample_weight_field ] - logging.info( - 'will not filter any input[except labels], total number inputs:%d' % - len(effective_fids) - ) - input_vals = tf.reshape( - input_vals, [-1, len(effective_fids)], name='input_reshape' - ) + logging.info('will not filter any input[except labels], total number inputs:%d' % len(effective_fids)) + input_vals = tf.reshape(input_vals, [-1, len(effective_fids)], name='input_reshape') features = {} for tmp_id, fid in enumerate(effective_fids): ftype = self._input_field_types[fid] @@ -347,13 +315,11 @@ def create_placeholders(self, export_config): features[input_name] = tf.string_to_number( input_vals[:, tmp_id], tf_type, - name='input_str_to_%s' % tf_type.name + name='input_str_to_%s' % tf_type.name, ) else: if ftype not in [DatasetConfig.STRING]: - logging.warning( - 'unexpected field type: ftype=%s tf_type=%s' % (ftype, tf_type) - ) + logging.warning('unexpected field type: ftype=%s tf_type=%s' % (ftype, tf_type)) features[input_name] = input_vals[:, tmp_id] features = self._preprocess(features) return {'features': inputs_placeholder}, features['feature'] @@ -366,9 +332,14 @@ def _get_labels(self, fields): return OrderedDict( [ ( - x, tf.squeeze(labels[x], axis=1) if len(labels[x].get_shape()) == 2 - and labels[x].get_shape()[1] == 1 else labels[x] - ) for x in labels + x, + ( + tf.squeeze(labels[x], axis=1) + if len(labels[x].get_shape()) == 2 and labels[x].get_shape()[1] == 1 + else labels[x] + ), + ) + for x in labels ] ) @@ -376,13 +347,14 @@ def _as_string(self, field, fc): if field.dtype == tf.string: return field if field.dtype in [tf.float32, tf.double]: - feature_name = fc.feature_name if fc.HasField('feature_name' - ) else fc.input_names[0] - assert fc.precision > 0, 'fc.precision not set for feature[%s], it is dangerous to convert ' \ - 'float or double to string due to precision problem, it is suggested ' \ - ' to convert them into string format before using EasyRec; ' \ - 'if you really need to do so, please set precision (the number of ' \ - 'decimal digits) carefully.' % feature_name + feature_name = fc.feature_name if fc.HasField('feature_name') else fc.input_names[0] + assert fc.precision > 0, ( + 'fc.precision not set for feature[%s], it is dangerous to convert ' + 'float or double to string due to precision problem, it is suggested ' + ' to convert them into string format before using EasyRec; ' + 'if you really need to do so, please set precision (the number of ' + 'decimal digits) carefully.' % feature_name + ) precision = None if field.dtype in [tf.float32, tf.double]: if fc.precision > 0: @@ -396,13 +368,13 @@ def _as_string(self, field, fc): def _parse_combo_feature(self, fc, parsed_dict, field_dict): # for compatibility with existing implementations - feature_name = fc.feature_name if fc.HasField('feature_name' - ) else fc.input_names[0] + feature_name = fc.feature_name if fc.HasField('feature_name') else fc.input_names[0] if len(fc.combo_input_seps) > 0: - assert len(fc.combo_input_seps) == len(fc.input_names), \ - 'len(combo_separator)[%d] != len(fc.input_names)[%d]' % ( - len(fc.combo_input_seps), len(fc.input_names)) + assert len(fc.combo_input_seps) == len(fc.input_names), 'len(combo_separator)[%d] != len(fc.input_names)[%d]' % ( + len(fc.combo_input_seps), + len(fc.input_names), + ) def _get_input_sep(input_id): if input_id < len(fc.combo_input_seps): @@ -418,10 +390,12 @@ def _get_input_sep(input_id): key = feature_name input_sep = _get_input_sep(input_id) if input_sep != '': - assert field_dict[ - input_name - ].dtype == tf.string, 'could not apply string_split to input-name[%s] dtype=%s' % ( - input_name, field_dict[input_name].dtype + assert field_dict[input_name].dtype == tf.string, ( + 'could not apply string_split to input-name[%s] dtype=%s' + % ( + input_name, + field_dict[input_name].dtype, + ) ) parsed_dict[key] = tf.string_split(field_dict[input_name], input_sep) else: @@ -432,29 +406,20 @@ def _get_input_sep(input_id): for input_id, input_name in enumerate(fc.input_names): input_sep = fc.combo_input_seps[input_id] if len(input_sep) > 0: - assert field_dict[ - input_name - ].dtype == tf.string, 'could not apply string_split to input-name[%s] dtype=%s' % ( - input_name, field_dict[input_name].dtype - ) - split_inputs.append( - tf.string_split( - field_dict[input_name], fc.combo_input_seps[input_id] + assert field_dict[input_name].dtype == tf.string, ( + 'could not apply string_split to input-name[%s] dtype=%s' + % ( + input_name, + field_dict[input_name].dtype, ) ) + split_inputs.append(tf.string_split(field_dict[input_name], fc.combo_input_seps[input_id])) else: split_inputs.append(tf.reshape(field_dict[input_name], [-1, 1])) - parsed_dict[feature_name] = sparse_ops.sparse_cross( - split_inputs, fc.combo_join_sep - ) + parsed_dict[feature_name] = sparse_ops.sparse_cross(split_inputs, fc.combo_join_sep) else: - inputs = [ - self._as_string(field_dict[input_name], fc) - for input_name in fc.input_names - ] - parsed_dict[feature_name] = string_ops.string_join( - inputs, fc.combo_join_sep - ) + inputs = [self._as_string(field_dict[input_name], fc) for input_name in fc.input_names] + parsed_dict[feature_name] = string_ops.string_join(inputs, fc.combo_join_sep) def _parse_tag_feature(self, fc, parsed_dict, field_dict): input_0 = fc.input_names[0] @@ -468,8 +433,10 @@ def _parse_tag_feature(self, fc, parsed_dict, field_dict): elif len(field.get_shape()) == 2: field = tf.squeeze(field, axis=-1) if fc.HasField('kv_separator') and len(fc.input_names) > 1: - assert False, 'Tag Feature Error, ' \ - 'Cannot set kv_separator and multi input_names in one feature config. Feature: %s.' % input_0 + assert False, ( + 'Tag Feature Error, ' + 'Cannot set kv_separator and multi input_names in one feature config. Feature: %s.' % input_0 + ) parsed_dict[feature_name] = tf.string_split(field, fc.separator) if fc.HasField('kv_separator'): indices = parsed_dict[feature_name].indices @@ -478,36 +445,35 @@ def _parse_tag_feature(self, fc, parsed_dict, field_dict): tmp_kvs = tf.reshape(tmp_kvs.values, [-1, 2]) tmp_ks, tmp_vs = tmp_kvs[:, 0], tmp_kvs[:, 1] - check_list = [ - tf.py_func(check_string_to_number, [tmp_vs, input_0], Tout=tf.bool) - ] if self._check_mode else [] + check_list = [tf.py_func(check_string_to_number, [tmp_vs, input_0], Tout=tf.bool)] if self._check_mode else [] with tf.control_dependencies(check_list): - tmp_vs = tf.string_to_number( - tmp_vs, tf.float32, name='kv_tag_wgt_str_2_flt_%s' % input_0 - ) - parsed_dict[feature_name] = tf.sparse.SparseTensor( - indices, tmp_ks, parsed_dict[feature_name].dense_shape - ) + tmp_vs = tf.string_to_number(tmp_vs, tf.float32, name='kv_tag_wgt_str_2_flt_%s' % input_0) + parsed_dict[feature_name] = tf.sparse.SparseTensor(indices, tmp_ks, parsed_dict[feature_name].dense_shape) parsed_dict[feature_name + '_w'] = tf.sparse.SparseTensor( indices, tmp_vs, parsed_dict[feature_name].dense_shape ) if not fc.HasField('hash_bucket_size') and fc.num_buckets > 0: - check_list = [ - tf.py_func( - check_string_to_number, - [parsed_dict[feature_name].values, input_0], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_string_to_number, + [parsed_dict[feature_name].values, input_0], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): vals = tf.string_to_number( parsed_dict[feature_name].values, tf.int32, - name='tag_fea_%s' % input_0 + name='tag_fea_%s' % input_0, ) parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, vals, - parsed_dict[feature_name].dense_shape + parsed_dict[feature_name].indices, + vals, + parsed_dict[feature_name].dense_shape, ) if len(fc.input_names) > 1: input_1 = fc.input_names[1] @@ -515,26 +481,27 @@ def _parse_tag_feature(self, fc, parsed_dict, field_dict): if len(field.get_shape()) == 0: field = tf.expand_dims(field, axis=0) field = tf.string_split(field, fc.separator) - check_list = [ - tf.py_func( - check_string_to_number, [field.values, input_1], Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_string_to_number, + [field.values, input_1], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): - field_vals = tf.string_to_number( - field.values, tf.float32, name='tag_wgt_str_2_flt_%s' % input_1 - ) + field_vals = tf.string_to_number(field.values, tf.float32, name='tag_wgt_str_2_flt_%s' % input_1) assert_op = tf.assert_equal( tf.shape(field_vals)[0], tf.shape(parsed_dict[feature_name].values)[0], - message= - 'TagFeature Error: The size of %s not equal to the size of %s. Please check input: %s and %s.' - % (input_0, input_1, input_0, input_1) + message='TagFeature Error: The size of %s not equal to %s. Please check input: %s and %s.' + % (input_0, input_1, input_0, input_1), ) with tf.control_dependencies([assert_op]): - field = tf.sparse.SparseTensor( - field.indices, tf.identity(field_vals), field.dense_shape - ) + field = tf.sparse.SparseTensor(field.indices, tf.identity(field_vals), field.dense_shape) parsed_dict[feature_name + '_w'] = field else: parsed_dict[feature_name] = field_dict[input_0] @@ -548,28 +515,32 @@ def _parse_expr_feature(self, fc, parsed_dict, field_dict): for input_name in fc.input_names: new_input_name = prefix + input_name if field_dict[input_name].dtype == tf.string: - check_list = [ - tf.py_func( - check_string_to_number, [field_dict[input_name], input_name], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_string_to_number, + [field_dict[input_name], input_name], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): parsed_dict[new_input_name] = tf.string_to_number( field_dict[input_name], tf.float64, - name='%s_str_2_int_for_expr' % new_input_name + name='%s_str_2_int_for_expr' % new_input_name, ) elif field_dict[input_name].dtype in [ - tf.int32, tf.int64, tf.double, tf.float32 + tf.int32, + tf.int64, + tf.double, + tf.float32, ]: - parsed_dict[new_input_name] = tf.cast( - field_dict[input_name], tf.float64 - ) + parsed_dict[new_input_name] = tf.cast(field_dict[input_name], tf.float64) else: - assert False, 'invalid input dtype[%s] for expr feature' % str( - field_dict[input_name].dtype - ) + assert False, 'invalid input dtype[%s] for expr feature' % str(field_dict[input_name].dtype) expression = get_expression(fc.expression, fc.input_names, prefix=prefix) logging.info('expression: %s' % expression) @@ -585,15 +556,22 @@ def _parse_id_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name] = self._as_string(field_dict[input_0], fc) elif fc.num_buckets > 0: if parsed_dict[feature_name].dtype == tf.string: - check_list = [ - tf.py_func( - check_string_to_number, [parsed_dict[feature_name], input_0], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_string_to_number, + [parsed_dict[feature_name], input_0], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.string_to_number( - parsed_dict[feature_name], tf.int32, name='%s_str_2_int' % input_0 + parsed_dict[feature_name], + tf.int32, + name='%s_str_2_int' % input_0, ) def _parse_raw_feature(self, fc, parsed_dict, field_dict): @@ -608,24 +586,35 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): vals = field_dict[input_0] segment_ids = tf.range(0, tf.shape(vals)[0]) if fc.raw_input_dim > 1: - check_list = [ - tf.py_func( - check_split, [vals, fc.separator, fc.raw_input_dim, input_0], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_split, + [vals, fc.separator, fc.raw_input_dim, input_0], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): tmp_fea = tf.string_split(vals, fc.separator) - check_list = [ - tf.py_func( - check_string_to_number, [tmp_fea.values, input_0], Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_string_to_number, + [tmp_fea.values, input_0], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): tmp_vals = tf.string_to_number( tmp_fea.values, tf.float32, - name='multi_raw_fea_to_flt_%s' % input_0 + name='multi_raw_fea_to_flt_%s' % input_0, ) if fc.HasField('seq_multi_sep') and fc.HasField('combiner'): emb = tf.reshape(tmp_vals, [-1, fc.raw_input_dim]) @@ -645,16 +634,12 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): tmp_fea.indices, [tf.shape(field_dict[input_0])[0], fc.raw_input_dim], tmp_vals, - default_value=0 + default_value=0, ) elif fc.HasField('seq_multi_sep') and fc.HasField('combiner'): - check_list = [ - tf.py_func(check_string_to_number, [vals, input_0], Tout=tf.bool) - ] if self._check_mode else [] + check_list = [tf.py_func(check_string_to_number, [vals, input_0], Tout=tf.bool)] if self._check_mode else [] with tf.control_dependencies(check_list): - emb = tf.string_to_number( - vals, tf.float32, name='raw_fea_to_flt_%s' % input_0 - ) + emb = tf.string_to_number(vals, tf.float32, name='raw_fea_to_flt_%s' % input_0) if fc.combiner == 'max': emb = tf.segment_max(emb, segment_ids) elif fc.combiner == 'sum': @@ -667,39 +652,33 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): assert False, 'unsupported combine operator: ' + fc.combiner parsed_dict[feature_name] = emb else: - check_list = [ - tf.py_func( - check_string_to_number, [field_dict[input_0], input_0], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_string_to_number, + [field_dict[input_0], input_0], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): - parsed_dict[feature_name] = tf.string_to_number( - field_dict[input_0], tf.float32 - ) - elif field_dict[input_0].dtype in [ - tf.int32, tf.int64, tf.double, tf.float32 - ]: + parsed_dict[feature_name] = tf.string_to_number(field_dict[input_0], tf.float32) + elif field_dict[input_0].dtype in [tf.int32, tf.int64, tf.double, tf.float32]: parsed_dict[feature_name] = tf.to_float(field_dict[input_0]) else: - assert False, 'invalid dtype[%s] for raw feature' % str( - field_dict[input_0].dtype - ) + assert False, 'invalid dtype[%s] for raw feature' % str(field_dict[input_0].dtype) if fc.max_val > fc.min_val: - parsed_dict[feature_name] = (parsed_dict[feature_name] - - fc.min_val) / (fc.max_val - fc.min_val) + parsed_dict[feature_name] = (parsed_dict[feature_name] - fc.min_val) / (fc.max_val - fc.min_val) if fc.HasField('normalizer_fn'): - logging.info( - 'apply normalizer_fn %s to `%s`' % (fc.normalizer_fn, feature_name) - ) - parsed_dict[feature_name] = self._normalizer_fn[feature_name]( - parsed_dict[feature_name] - ) + logging.info('apply normalizer_fn %s to `%s`' % (fc.normalizer_fn, feature_name)) + parsed_dict[feature_name] = self._normalizer_fn[feature_name](parsed_dict[feature_name]) - if not fc.boundaries and fc.num_buckets <= 1 and \ - fc.embedding_dim > 0 and \ - self._data_config.sample_weight != input_0: + if ( + not fc.boundaries and fc.num_buckets <= 1 and fc.embedding_dim > 0 and self._data_config.sample_weight != input_0 + ): # may need by wide model and deep model to project # raw values to a vector, it maybe better implemented # by a ProjectionColumn later @@ -718,12 +697,12 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( indices=indices, values=indices_1[:, 0], - dense_shape=[sample_num, fc.raw_input_dim] + dense_shape=[sample_num, fc.raw_input_dim], ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( indices=indices, values=tf.reshape(tmp_parsed, [-1]), - dense_shape=[sample_num, fc.raw_input_dim] + dense_shape=[sample_num, fc.raw_input_dim], ) # self._appended_fields.append(input_0 + '_raw_proj_id') # self._appended_fields.append(input_0 + '_raw_proj_val') @@ -747,66 +726,80 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): # 3 dimensional sparse tensor out_shape = tf.concat( [parsed_dict[feature_name].dense_shape, multi_vals.dense_shape[1:]], - axis=0 + axis=0, ) - parsed_dict[feature_name] = tf.sparse.SparseTensor( - out_indices, multi_vals.values, out_shape + parsed_dict[feature_name] = tf.sparse.SparseTensor(out_indices, multi_vals.values, out_shape) + if fc.num_buckets > 1 and fc.max_val == fc.min_val: + check_list = ( + [ + tf.py_func( + check_string_to_number, + [parsed_dict[feature_name].values, input_0], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] ) - if (fc.num_buckets > 1 and fc.max_val == fc.min_val): - check_list = [ - tf.py_func( - check_string_to_number, - [parsed_dict[feature_name].values, input_0], - Tout=tf.bool - ) - ] if self._check_mode else [] with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.sparse.SparseTensor( parsed_dict[feature_name].indices, tf.string_to_number( parsed_dict[feature_name].values, tf.int64, - name='sequence_str_2_int_%s' % input_0 - ), parsed_dict[feature_name].dense_shape + name='sequence_str_2_int_%s' % input_0, + ), + parsed_dict[feature_name].dense_shape, ) elif sub_feature_type == fc.RawFeature: - check_list = [ - tf.py_func( - check_string_to_number, - [parsed_dict[feature_name].values, input_0], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_string_to_number, + [parsed_dict[feature_name].values, input_0], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.sparse.SparseTensor( parsed_dict[feature_name].indices, tf.string_to_number( parsed_dict[feature_name].values, tf.float32, - name='sequence_str_2_float_%s' % input_0 - ), parsed_dict[feature_name].dense_shape + name='sequence_str_2_float_%s' % input_0, + ), + parsed_dict[feature_name].dense_shape, ) if fc.num_buckets > 1 and fc.max_val > fc.min_val: - normalized_values = (parsed_dict[feature_name].values - - fc.min_val) / (fc.max_val - fc.min_val) + normalized_values = (parsed_dict[feature_name].values - fc.min_val) / (fc.max_val - fc.min_val) parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, normalized_values, - parsed_dict[feature_name].dense_shape + parsed_dict[feature_name].indices, + normalized_values, + parsed_dict[feature_name].dense_shape, ) else: parsed_dict[feature_name] = field - if not fc.boundaries and fc.num_buckets <= 1 and\ - self._data_config.sample_weight != input_0 and\ - sub_feature_type == fc.RawFeature and\ - fc.raw_input_dim == 1: + if ( + not fc.boundaries + and fc.num_buckets <= 1 + and self._data_config.sample_weight != input_0 + and sub_feature_type == fc.RawFeature + and fc.raw_input_dim == 1 + ): logging.info( - 'Not set boundaries or num_buckets or hash_bucket_size, %s will process as two dimension sequence raw feature' + ( + 'Not set boundaries or num_buckets or hash_bucket_size, ' + '%s will process as two dimension sequence raw feature' + ) % feature_name ) parsed_dict[feature_name] = tf.sparse_to_dense( parsed_dict[feature_name].indices, [tf.shape(parsed_dict[feature_name])[0], fc.sequence_length], - parsed_dict[feature_name].values + parsed_dict[feature_name].values, ) sample_num = tf.to_int64(tf.shape(parsed_dict[feature_name])[0]) indices_0 = tf.range(sample_num, dtype=tf.int64) @@ -822,29 +815,33 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( indices=indices, values=indices_1[:, 0], - dense_shape=[sample_num, fc.sequence_length] + dense_shape=[sample_num, fc.sequence_length], ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( indices=indices, values=tf.reshape(tmp_parsed, [-1]), - dense_shape=[sample_num, fc.sequence_length] + dense_shape=[sample_num, fc.sequence_length], ) elif ( - not fc.boundaries and fc.num_buckets <= 1 + not fc.boundaries + and fc.num_buckets <= 1 and self._data_config.sample_weight != input_0 - and sub_feature_type == fc.RawFeature and fc.raw_input_dim > 1 + and sub_feature_type == fc.RawFeature + and fc.raw_input_dim > 1 ): # for 3 dimension sequence feature input. logging.info( 'Not set boundaries or num_buckets or hash_bucket_size,' - ' %s will process as three dimension sequence raw feature' % - feature_name + ' %s will process as three dimension sequence raw feature' % feature_name ) parsed_dict[feature_name] = tf.sparse_to_dense( - parsed_dict[feature_name].indices, [ - tf.shape(parsed_dict[feature_name])[0], fc.sequence_length, - fc.raw_input_dim - ], parsed_dict[feature_name].values + parsed_dict[feature_name].indices, + [ + tf.shape(parsed_dict[feature_name])[0], + fc.sequence_length, + fc.raw_input_dim, + ], + parsed_dict[feature_name].values, ) sample_num = tf.to_int64(tf.shape(parsed_dict[feature_name])[0]) indices_0 = tf.range(sample_num, dtype=tf.int64) @@ -865,12 +862,12 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( indices=indices, values=indices_1[:, 0], - dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim] + dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim], ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( indices=indices, values=tf.reshape(parsed_dict[feature_name], [-1]), - dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim] + dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim], ) # self._appended_fields.append(input_0 + '_raw_proj_id') # self._appended_fields.append(input_0 + '_raw_proj_val') @@ -936,8 +933,7 @@ def _preprocess(self, field_dict): elif feature_type == fc.ComboFeature: self._parse_combo_feature(fc, parsed_dict, field_dict) else: - feature_name = fc.feature_name if fc.HasField('feature_name' - ) else fc.input_names[0] + feature_name = fc.feature_name if fc.HasField('feature_name') else fc.input_names[0] for input_id, input_name in enumerate(fc.input_names): if input_id > 0: key = feature_name + '_' + str(input_id) @@ -957,45 +953,52 @@ def _preprocess(self, field_dict): else: assert dtype is not None, 'must set user_define_fn_res_type' logging.info('apply py_func transform: %s' % udf_class) - field_dict[input_name] = tf.py_func( - udf, [field_dict[input_name]], Tout=get_tf_type(dtype) - ) + field_dict[input_name] = tf.py_func(udf, [field_dict[input_name]], Tout=get_tf_type(dtype)) field_dict[input_name].set_shape(tf.TensorShape([None])) if field_dict[input_name].dtype == tf.string: if self._label_dim[input_id] > 1: logging.info('will split labels[%d]=%s' % (input_id, input_name)) - check_list = [ - tf.py_func( - check_split, [ - field_dict[input_name], self._label_sep[input_id], - self._label_dim[input_id], input_name - ], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_split, + [ + field_dict[input_name], + self._label_sep[input_id], + self._label_dim[input_id], + input_name, + ], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): - label_dict[input_name] = tf.string_split( - field_dict[input_name], self._label_sep[input_id] - ).values - label_dict[input_name] = tf.reshape( - label_dict[input_name], [-1, self._label_dim[input_id]] - ) + label_dict[input_name] = tf.string_split(field_dict[input_name], self._label_sep[input_id]).values + label_dict[input_name] = tf.reshape(label_dict[input_name], [-1, self._label_dim[input_id]]) else: label_dict[input_name] = field_dict[input_name] - check_list = [ - tf.py_func( - check_string_to_number, [label_dict[input_name], input_name], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_string_to_number, + [label_dict[input_name], input_name], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): - label_dict[input_name] = tf.string_to_number( - label_dict[input_name], tf.float32, name=input_name - ) + label_dict[input_name] = tf.string_to_number(label_dict[input_name], tf.float32, name=input_name) else: assert field_dict[input_name].dtype in [ - tf.float32, tf.double, tf.int32, tf.int64 + tf.float32, + tf.double, + tf.int32, + tf.int64, ], 'invalid label dtype: %s' % str(field_dict[input_name].dtype) label_dict[input_name] = field_dict[input_name] @@ -1003,15 +1006,12 @@ def _preprocess(self, field_dict): for func_config in self._data_config.extra_label_func: lbl_name = func_config.label_name func_name = func_config.label_func - logging.info( - 'generating new label `%s` by transform: %s' % (lbl_name, func_name) - ) + logging.info('generating new label `%s` by transform: %s' % (lbl_name, func_name)) lbl_fn = load_by_path(func_name) label_dict[lbl_name] = lbl_fn(label_dict) if self._data_config.HasField('sample_weight'): - parsed_dict[constant.SAMPLE_WEIGHT - ] = field_dict[self._data_config.sample_weight] + parsed_dict[constant.SAMPLE_WEIGHT] = field_dict[self._data_config.sample_weight] if Input.DATA_OFFSET in field_dict: parsed_dict[Input.DATA_OFFSET] = field_dict[Input.DATA_OFFSET] @@ -1060,14 +1060,12 @@ def _lookup(args, pad=True): indices_1 = tf.range(0, n, dtype=tf.int64) indices = [ tf.expand_dims(indices_0, axis=1), - tf.expand_dims(indices_1, axis=1) + tf.expand_dims(indices_1, axis=1), ] indices = tf.concat(indices, axis=1) return tf.sparse.SparseTensor(indices, vals, [1, n]) - vals, masks, indices = tf.map_fn( - _lookup, [key_fields, map_fields], dtype=(tf.string, tf.bool, tf.int64) - ) + vals, masks, indices = tf.map_fn(_lookup, [key_fields, map_fields], dtype=(tf.string, tf.bool, tf.int64)) batch_size = tf.to_int64(tf.shape(vals)[0]) vals = tf.boolean_mask(vals, masks) indices_1 = tf.boolean_mask(indices, masks) @@ -1076,9 +1074,8 @@ def _lookup(args, pad=True): indices_0 = indices_0 + tf.zeros([1, max_sel_num], dtype=tf.int64) indices_0 = tf.boolean_mask(indices_0, masks) indices = tf.concat( - [tf.expand_dims(indices_0, axis=1), - tf.expand_dims(indices_1, axis=1)], - axis=1 + [tf.expand_dims(indices_0, axis=1), tf.expand_dims(indices_1, axis=1)], + axis=1, ) shapes = tf.stack([batch_size, tf.reduce_max(indices_1) + 1]) return tf.sparse.SparseTensor(indices, vals, shapes) @@ -1098,14 +1095,11 @@ def stop(self): def _safe_shard(self, dataset): if self._data_config.chief_redundant: - return dataset.shard( - max(self._task_num - 1, 1), max(self._task_index - 1, 0) - ) + return dataset.shard(max(self._task_num - 1, 1), max(self._task_index - 1, 0)) else: return dataset.shard(self._task_num, self._task_index) def create_input(self, export_config=None): - def _input_fn(mode=None, params=None, config=None): """Build input_fn for estimator. @@ -1123,8 +1117,9 @@ def _input_fn(mode=None, params=None, config=None): """ self._pre_build(mode, params) if mode in ( - tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT + tf.estimator.ModeKeys.TRAIN, + tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT, ): # build dataset from self._config.input_path self._mode = mode @@ -1140,9 +1135,7 @@ def _input_fn(mode=None, params=None, config=None): else: with conditional(place_on_cpu, ops.device('/CPU:0')): inputs, features = self.create_placeholders(export_config) - print( - 'built feature placeholders. features: {}'.format(features.keys()) - ) + print('built feature placeholders. features: {}'.format(features.keys())) return tf.estimator.export.ServingInputReceiver(features, inputs) _input_fn.input_creator = self diff --git a/easy_rec/python/input/kafka_dataset.py b/easy_rec/python/input/kafka_dataset.py index b8f39f83d..fcaa985da 100644 --- a/easy_rec/python/input/kafka_dataset.py +++ b/easy_rec/python/input/kafka_dataset.py @@ -23,9 +23,7 @@ try: from easy_rec.python.ops import gen_kafka_ops except ImportError: - logging.warning( - 'failed to import gen_kafka_ops: %s' % traceback.format_exc() - ) + logging.warning('failed to import gen_kafka_ops: %s' % traceback.format_exc()) class KafkaDataset(dataset_ops.Dataset): @@ -41,7 +39,7 @@ def __init__( config_global=None, config_topic=None, message_key=False, - message_offset=False + message_offset=False, ): """Create a KafkaReader. @@ -68,27 +66,15 @@ def __init__( message_key: If True, the kafka will output both message value and key. message_offset: If True, the kafka will output both message value and offset. """ - self._topics = ops.convert_to_tensor( - topics, dtype=dtypes.string, name='topics' - ) - self._servers = ops.convert_to_tensor( - servers, dtype=dtypes.string, name='servers' - ) - self._group = ops.convert_to_tensor( - group, dtype=dtypes.string, name='group' - ) + self._topics = ops.convert_to_tensor(topics, dtype=dtypes.string, name='topics') + self._servers = ops.convert_to_tensor(servers, dtype=dtypes.string, name='servers') + self._group = ops.convert_to_tensor(group, dtype=dtypes.string, name='group') self._eof = ops.convert_to_tensor(eof, dtype=dtypes.bool, name='eof') - self._timeout = ops.convert_to_tensor( - timeout, dtype=dtypes.int64, name='timeout' - ) + self._timeout = ops.convert_to_tensor(timeout, dtype=dtypes.int64, name='timeout') config_global = config_global if config_global else [] - self._config_global = ops.convert_to_tensor( - config_global, dtype=dtypes.string, name='config_global' - ) + self._config_global = ops.convert_to_tensor(config_global, dtype=dtypes.string, name='config_global') config_topic = config_topic if config_topic else [] - self._config_topic = ops.convert_to_tensor( - config_topic, dtype=dtypes.string, name='config_topic' - ) + self._config_topic = ops.convert_to_tensor(config_topic, dtype=dtypes.string, name='config_topic') self._message_key = message_key self._message_offset = message_offset super(KafkaDataset, self).__init__() @@ -115,28 +101,27 @@ def output_classes(self): return (ops.Tensor, ops.Tensor) elif self._message_key and self._message_offset: return (ops.Tensor, ops.Tensor, ops.Tensor) - return (ops.Tensor) + return ops.Tensor @property def output_shapes(self): if self._message_key ^ self._message_offset: - return ((tensor_shape.TensorShape([]), tensor_shape.TensorShape([]))) + return (tensor_shape.TensorShape([]), tensor_shape.TensorShape([])) elif self._message_key and self._message_offset: return ( - ( - tensor_shape.TensorShape([]), tensor_shape.TensorShape([]), - tensor_shape.TensorShape([]) - ) + tensor_shape.TensorShape([]), + tensor_shape.TensorShape([]), + tensor_shape.TensorShape([]), ) - return ((tensor_shape.TensorShape([]))) + return tensor_shape.TensorShape([]) @property def output_types(self): if self._message_key ^ self._message_offset: - return ((dtypes.string, dtypes.string)) + return (dtypes.string, dtypes.string) elif self._message_key and self._message_offset: - return ((dtypes.string, dtypes.string, dtypes.string)) - return ((dtypes.string)) + return (dtypes.string, dtypes.string, dtypes.string) + return dtypes.string def write_kafka_v2(message, topic, servers='localhost', name=None): @@ -152,6 +137,4 @@ def write_kafka_v2(message, topic, servers='localhost', name=None): Returns: A `Tensor` of type `string`. 0-D. """ - return gen_kafka_ops.io_write_kafka_v2( - message=message, topic=topic, servers=servers, name=name - ) + return gen_kafka_ops.io_write_kafka_v2(message=message, topic=topic, servers=servers, name=name) diff --git a/easy_rec/python/input/kafka_input.py b/easy_rec/python/input/kafka_input.py index d2bb60dbf..721654660 100644 --- a/easy_rec/python/input/kafka_input.py +++ b/easy_rec/python/input/kafka_input.py @@ -20,8 +20,7 @@ from kafka import KafkaConsumer, TopicPartition except ImportError: logging.warning( - 'kafka-python is not installed[%s]. You can install it by: pip install kafka-python' - % traceback.format_exc() + 'kafka-python is not installed[%s]. You can install it by: pip install kafka-python' % traceback.format_exc() ) if tf.__version__ >= '2.0': @@ -32,7 +31,6 @@ class KafkaInput(Input): - DATA_OFFSET = 'DATA_OFFSET' def __init__( @@ -43,11 +41,16 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(KafkaInput, self).__init__( - data_config, feature_config, '', task_index, task_num, check_mode, - pipeline_config + data_config, + feature_config, + '', + task_index, + task_num, + check_mode, + pipeline_config, ) self._kafka = kafka_config self._offset_dict = {} @@ -55,24 +58,18 @@ def __init__( consumer = KafkaConsumer( group_id='kafka_dataset_consumer', bootstrap_servers=[self._kafka.server], - api_version_auto_timeout_ms=60000 + api_version_auto_timeout_ms=60000, ) # in miliseconds partitions = consumer.partitions_for_topic(self._kafka.topic) self._num_partition = len(partitions) - logging.info( - 'all partitions[%d]: %s' % (self._num_partition, partitions) - ) + logging.info('all partitions[%d]: %s' % (self._num_partition, partitions)) # determine kafka offsets for each partition offset_type = self._kafka.WhichOneof('offset') if offset_type is not None: if offset_type == 'offset_time': ts = parse_time(self._kafka.offset_time) - input_map = { - TopicPartition(partition=part_id, topic=self._kafka.topic): - ts * 1000 - for part_id in partitions - } + input_map = {TopicPartition(partition=part_id, topic=self._kafka.topic): ts * 1000 for part_id in partitions} part_offsets = consumer.offsets_for_times(input_map) # part_offsets is a dictionary: # { @@ -82,10 +79,13 @@ def __init__( for part in part_offsets: self._offset_dict[part.partition] = part_offsets[part].offset logging.info( - 'Find offset by time, topic[%s], partition[%d], timestamp[%ss], offset[%d], offset_timestamp[%dms]' + ('Find offset by time, topic[%s], partition[%d], timestamp[%ss], ' 'offset[%d], offset_timestamp[%dms]') % ( - self._kafka.topic, part.partition, ts, - part_offsets[part].offset, part_offsets[part].timestamp + self._kafka.topic, + part.partition, + ts, + part_offsets[part].offset, + part_offsets[part].timestamp, ) ) elif offset_type == 'offset_info': @@ -112,8 +112,7 @@ def _preprocess(self, field_dict): def _parse_csv(self, line, message_key, message_offset): record_defaults = [ - self.get_type_defaults(t, v) - for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) for t, v in zip(self._input_field_types, self._input_field_defaults) ] fields = tf.decode_csv( @@ -121,7 +120,7 @@ def _parse_csv(self, line, message_key, message_offset): use_quote_delim=False, field_delim=self._data_config.separator, record_defaults=record_defaults, - name='decode_csv' + name='decode_csv', ) inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids} @@ -141,8 +140,7 @@ def _parse_offset(message_offset): self._task_offset_dict[k] = v return json.dumps(self._task_offset_dict) - inputs[Input.DATA_OFFSET - ] = tf.py_func(_parse_offset, [message_offset], tf.string) + inputs[Input.DATA_OFFSET] = tf.py_func(_parse_offset, [message_offset], tf.string) return inputs def restore(self, checkpoint_path): @@ -178,10 +176,10 @@ def _get_topics(self): topics.append('%s:%d:%d' % (self._kafka.topic, part_id, offset)) self._task_offset_dict[part_id] = offset logging.info('assigned topic partitions: %s' % (','.join(topics))) - assert len(topics - ) > 0, 'no partitions are assigned for this task(%d/%d)' % ( - self._task_index, self._task_num - ) + assert len(topics) > 0, 'no partitions are assigned for this task(%d/%d)' % ( + self._task_index, + self._task_num, + ) return topics def _build(self, mode, params): @@ -193,8 +191,11 @@ def _build(self, mode, params): logging.info( 'train kafka server: %s topic: %s task_num: %d task_index: %d topics: %s' % ( - train_kafka.server, train_kafka.topic, self._task_num, - self._task_index, task_topics + train_kafka.server, + train_kafka.topic, + self._task_num, + self._task_index, + task_topics, ) ) @@ -206,14 +207,14 @@ def _build(self, mode, params): config_global=list(self._kafka.config_global), config_topic=list(self._kafka.config_topic), message_key=True, - message_offset=True + message_offset=True, ) if self._data_config.shuffle: dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) else: eval_kafka = self._kafka @@ -222,8 +223,11 @@ def _build(self, mode, params): logging.info( 'eval kafka server: %s topic: %s task_num: %d task_index: %d topics: %s' % ( - eval_kafka.server, eval_kafka.topic, self._task_num, - self._task_index, task_topics + eval_kafka.server, + eval_kafka.topic, + self._task_num, + self._task_index, + task_topics, ) ) @@ -235,26 +239,20 @@ def _build(self, mode, params): config_global=list(self._kafka.config_global), config_topic=list(self._kafka.config_topic), message_key=True, - message_offset=True + message_offset=True, ) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map( - self._parse_csv, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(self._parse_csv, num_parallel_calls=num_parallel_calls) if self._data_config.ignore_error: dataset = dataset.apply(ignore_errors) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/load_parquet.py b/easy_rec/python/input/load_parquet.py index 8154669d6..71e7f6767 100644 --- a/easy_rec/python/input/load_parquet.py +++ b/easy_rec/python/input/load_parquet.py @@ -21,7 +21,7 @@ def start_data_proc( dense_fea_cfgs, reserve_fields, drop_remainder, - need_pack=True + need_pack=True, ): mp_ctxt = multiprocessing.get_context('spawn') proc_arr = [] @@ -29,11 +29,23 @@ def start_data_proc( proc = mp_ctxt.Process( target=load_data_proc, args=( - proc_id, file_que, data_que, proc_start_que, proc_stop_que, batch_size, - label_fields, sparse_fea_names, dense_fea_names, dense_fea_cfgs, - reserve_fields, drop_remainder, task_index, task_num, need_pack + proc_id, + file_que, + data_que, + proc_start_que, + proc_stop_que, + batch_size, + label_fields, + sparse_fea_names, + dense_fea_names, + dense_fea_cfgs, + reserve_fields, + drop_remainder, + task_index, + task_num, + need_pack, ), - name='task_%d_data_proc_%d' % (task_index, proc_id) + name='task_%d_data_proc_%d' % (task_index, proc_id), ) proc.daemon = True proc.start() @@ -97,9 +109,7 @@ def _pack_sparse_feas(data_dict, sparse_fea_names): def _pack_dense_feas(data_dict, dense_fea_names, dense_fea_cfgs): fea_val_arr = [] for fea_name, fea_cfg in zip(dense_fea_names, dense_fea_cfgs): - fea_val_arr.append( - data_dict[fea_name].reshape([-1, fea_cfg.raw_input_dim]) - ) + fea_val_arr.append(data_dict[fea_name].reshape([-1, fea_cfg.raw_input_dim])) del data_dict[fea_name] fea_vals = np.concatenate(fea_val_arr, axis=1) data_dict['dense_fea'] = fea_vals @@ -107,25 +117,26 @@ def _pack_dense_feas(data_dict, dense_fea_names, dense_fea_cfgs): def _reshape_dense_feas(data_dict, dense_fea_names, dense_fea_cfgs): for fea_name, fea_cfg in zip(dense_fea_names, dense_fea_cfgs): - data_dict[fea_name] = data_dict[fea_name].reshape( - [-1, fea_cfg.raw_input_dim] - ) + data_dict[fea_name] = data_dict[fea_name].reshape([-1, fea_cfg.raw_input_dim]) def _load_dense(input_data, field_names, sid, eid, dense_dict): for k in field_names: if isinstance(input_data[k][0], np.ndarray): np_dtype = type(input_data[k][sid][0]) - dense_dict[k] = np.array( - [x[0] for x in input_data[k][sid:eid]], dtype=np_dtype - ) + dense_dict[k] = np.array([x[0] for x in input_data[k][sid:eid]], dtype=np_dtype) else: dense_dict[k] = input_data[k][sid:eid].to_numpy() def _load_and_pad_dense( - input_data, field_names, sid, dense_dict, part_dense_dict, part_dense_dict_n, - batch_size + input_data, + field_names, + sid, + dense_dict, + part_dense_dict, + part_dense_dict_n, + batch_size, ): for k in field_names: if isinstance(input_data[k][0], np.ndarray): @@ -147,14 +158,23 @@ def _load_and_pad_dense( def load_data_proc( - proc_id, file_que, data_que, proc_start_que, proc_stop_que, batch_size, - label_fields, sparse_fea_names, dense_fea_names, dense_fea_cfgs, - reserve_fields, drop_remainder, task_index, task_num, need_pack + proc_id, + file_que, + data_que, + proc_start_que, + proc_stop_que, + batch_size, + label_fields, + sparse_fea_names, + dense_fea_names, + dense_fea_cfgs, + reserve_fields, + drop_remainder, + task_index, + task_num, + need_pack, ): - logging.info( - 'data proc %d start, proc_start_que=%s' % - (proc_id, proc_start_que.qsize()) - ) + logging.info('data proc %d start, proc_start_que=%s' % (proc_id, proc_start_que.qsize())) proc_start_que.get() effective_fields = sparse_fea_names + dense_fea_names all_fields = effective_fields @@ -164,9 +184,7 @@ def load_data_proc( for tmp in reserve_fields: if tmp not in all_fields: all_fields.append(tmp) - logging.info( - 'data proc %d start, file_que.qsize=%d' % (proc_id, file_que.qsize()) - ) + logging.info('data proc %d start, file_que.qsize=%d' % (proc_id, file_que.qsize())) num_files = 0 part_data_dict = {} @@ -208,10 +226,9 @@ def load_data_proc( else: all_lens = np.ones([len(val)], dtype=np.int32) all_vals = val.to_numpy() - assert np.sum(all_lens) == len( - all_vals - ), 'len(all_vals)=%d np.sum(all_lens)=%d' % ( - len(all_vals), np.sum(all_lens) + assert np.sum(all_lens) == len(all_vals), 'len(all_vals)=%d np.sum(all_lens)=%d' % ( + len(all_vals), + np.sum(all_lens), ) data_dict[k] = (all_lens, all_vals) @@ -241,22 +258,37 @@ def load_data_proc( if label_fields is not None and len(label_fields) > 0: _load_and_pad_dense( - input_data, label_fields, sid, data_dict, part_data_dict, - part_data_dict_n, batch_size + input_data, + label_fields, + sid, + data_dict, + part_data_dict, + part_data_dict_n, + batch_size, ) if reserve_fields is not None and len(reserve_fields) > 0: data_dict['reserve'] = {} part_data_dict_n['reserve'] = {} _load_and_pad_dense( - input_data, label_fields, sid, data_dict['reserve'], - part_data_dict['reserve'], part_data_dict_n['reserve'], batch_size + input_data, + label_fields, + sid, + data_dict['reserve'], + part_data_dict['reserve'], + part_data_dict_n['reserve'], + batch_size, ) if len(dense_fea_names) > 0: _load_and_pad_dense( - input_data, dense_fea_names, sid, data_dict, part_data_dict, - part_data_dict_n, batch_size + input_data, + dense_fea_names, + sid, + data_dict, + part_data_dict, + part_data_dict_n, + batch_size, ) if len(sparse_fea_names) > 0: @@ -314,21 +346,16 @@ def load_data_proc( else: if len(dense_fea_names) > 0: _reshape_dense_feas(part_data_dict, dense_fea_names, dense_fea_cfgs) - logging.info( - 'remainder batch: %s sample_num=%d' % - (','.join(part_data_dict.keys()), batch_len) - ) + logging.info('remainder batch: %s sample_num=%d' % (','.join(part_data_dict.keys()), batch_len)) _add_to_que(part_data_dict, data_que, proc_stop_que) total_batch_cnt += 1 else: - logging.warning( - 'drop remain %d samples as drop_remainder is set' % batch_len - ) + logging.warning('drop remain %d samples as drop_remainder is set' % batch_len) if is_good: is_good = _add_to_que(None, data_que, proc_stop_que) logging.info( - 'data_proc_id[%d]: is_good = %s, total_batch_cnt=%d, total_sample_cnt=%d' % - (proc_id, is_good, total_batch_cnt, total_sample_cnt) + 'data_proc_id[%d]: is_good = %s, total_batch_cnt=%d, total_sample_cnt=%d' + % (proc_id, is_good, total_batch_cnt, total_sample_cnt) ) data_que.close(wait_send_finish=is_good) diff --git a/easy_rec/python/input/odps_input.py b/easy_rec/python/input/odps_input.py index 67f10315b..b33b6afa2 100644 --- a/easy_rec/python/input/odps_input.py +++ b/easy_rec/python/input/odps_input.py @@ -12,7 +12,6 @@ class OdpsInput(Input): - def __init__( self, data_config, @@ -21,11 +20,16 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(OdpsInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) def _build(self, mode, params): @@ -33,27 +37,24 @@ def _build(self, mode, params): odps_util.check_input_field_and_types(self._data_config) selected_cols = ','.join(self._input_fields) - if self._data_config.chief_redundant and \ - mode == tf.estimator.ModeKeys.TRAIN: + if self._data_config.chief_redundant and mode == tf.estimator.ModeKeys.TRAIN: reader = tf.TableRecordReader( csv_delimiter=self._data_config.separator, selected_cols=selected_cols, slice_count=max(self._task_num - 1, 1), - slice_id=max(self._task_index - 1, 0) + slice_id=max(self._task_index - 1, 0), ) else: reader = tf.TableRecordReader( csv_delimiter=self._data_config.separator, selected_cols=selected_cols, slice_count=self._task_num, - slice_id=self._task_index + slice_id=self._task_index, ) if type(self._input_path) != list: self._input_path = self._input_path.split(',') - assert len( - self._input_path - ) > 0, 'match no files with %s' % self._input_path + assert len(self._input_path) > 0, 'match no files with %s' % self._input_path if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.pai_worker_queue: @@ -61,7 +62,7 @@ def _build(self, mode, params): self._input_path, num_epochs=self.num_epochs, shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num + num_slices=self._data_config.pai_worker_slice_num * self._task_num, ) work_queue.add_summary() file_queue = work_queue.input_producer() @@ -71,23 +72,20 @@ def _build(self, mode, params): self._input_path, num_epochs=self.num_epochs, capacity=1000, - shuffle=self._data_config.shuffle + shuffle=self._data_config.shuffle, ) else: - file_queue = tf.train.string_input_producer( - self._input_path, num_epochs=1, capacity=1000, shuffle=False - ) + file_queue = tf.train.string_input_producer(self._input_path, num_epochs=1, capacity=1000, shuffle=False) key, value = reader.read_up_to(file_queue, self._batch_size) record_defaults = [ - self.get_type_defaults(t, v) - for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) for t, v in zip(self._input_field_types, self._input_field_defaults) ] fields = tf.decode_csv( value, record_defaults=record_defaults, field_delim=self._data_config.separator, - name='decode_csv' + name='decode_csv', ) inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids} diff --git a/easy_rec/python/input/odps_input_v2.py b/easy_rec/python/input/odps_input_v2.py index bd6bac28e..c1ec5f515 100644 --- a/easy_rec/python/input/odps_input_v2.py +++ b/easy_rec/python/input/odps_input_v2.py @@ -14,7 +14,6 @@ class OdpsInputV2(Input): - def __init__( self, data_config, @@ -23,11 +22,16 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(OdpsInputV2, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) def _parse_table(self, *fields): @@ -40,43 +44,32 @@ def _parse_table(self, *fields): def _build(self, mode, params): if type(self._input_path) != list: self._input_path = self._input_path.split(',') - assert len( - self._input_path - ) > 0, 'match no files with %s' % self._input_path + assert len(self._input_path) > 0, 'match no files with %s' % self._input_path # check data_config are consistent with odps tables odps_util.check_input_field_and_types(self._data_config) selected_cols = ','.join(self._input_fields) record_defaults = [ - self.get_type_defaults(x, v) - for x, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(x, v) for x, v in zip(self._input_field_types, self._input_field_defaults) ] - if self._data_config.pai_worker_queue and \ - mode == tf.estimator.ModeKeys.TRAIN: - logging.info( - 'pai_worker_slice_num = %d' % self._data_config.pai_worker_slice_num - ) + if self._data_config.pai_worker_queue and mode == tf.estimator.ModeKeys.TRAIN: + logging.info('pai_worker_slice_num = %d' % self._data_config.pai_worker_slice_num) work_queue = pai.data.WorkQueue( self._input_path, num_epochs=self.num_epochs, shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num + num_slices=self._data_config.pai_worker_slice_num * self._task_num, ) que_paths = work_queue.input_dataset() - dataset = tf.data.TableRecordDataset( - que_paths, - record_defaults=record_defaults, - selected_cols=selected_cols - ) - elif self._data_config.chief_redundant and \ - mode == tf.estimator.ModeKeys.TRAIN: + dataset = tf.data.TableRecordDataset(que_paths, record_defaults=record_defaults, selected_cols=selected_cols) + elif self._data_config.chief_redundant and mode == tf.estimator.ModeKeys.TRAIN: dataset = tf.data.TableRecordDataset( self._input_path, record_defaults=record_defaults, selected_cols=selected_cols, slice_id=max(self._task_index - 1, 0), - slice_count=max(self._task_num - 1, 1) + slice_count=max(self._task_num - 1, 1), ) else: dataset = tf.data.TableRecordDataset( @@ -84,7 +77,7 @@ def _build(self, mode, params): record_defaults=record_defaults, selected_cols=selected_cols, slice_id=self._task_index, - slice_count=self._task_num + slice_count=self._task_num, ) if mode == tf.estimator.ModeKeys.TRAIN: @@ -92,7 +85,7 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: @@ -100,24 +93,19 @@ def _build(self, mode, params): dataset = dataset.batch(batch_size=self._data_config.batch_size) - dataset = dataset.map( - self._parse_table, - num_parallel_calls=self._data_config.num_parallel_calls - ) + dataset = dataset.map(self._parse_table, num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/odps_input_v3.py b/easy_rec/python/input/odps_input_v3.py index 93d273bf7..649a9f3c0 100644 --- a/easy_rec/python/input/odps_input_v3.py +++ b/easy_rec/python/input/odps_input_v3.py @@ -27,18 +27,23 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(OdpsInputV3, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) self._num_epoch = 0 if common_io is None: logging.error( - ''' + """ please install common_io pip install - https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.2%2Btunnel-py2.py3-none-any.whl''' + https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.2%2Btunnel-py2.py3-none-any.whl""" ) sys.exit(1) @@ -54,16 +59,13 @@ def _odps_read(self): self._num_epoch += 1 if type(self._input_path) != list: self._input_path = self._input_path.split(',') - assert len( - self._input_path - ) > 0, 'match no files with %s' % self._input_path + assert len(self._input_path) > 0, 'match no files with %s' % self._input_path # check data_config are consistent with odps tables odps_util.check_input_field_and_types(self._data_config) record_defaults = [ - self.get_type_defaults(x, v) - for x, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(x, v) for x, v in zip(self._input_field_types, self._input_field_defaults) ] selected_cols = ','.join(self._input_fields) @@ -72,19 +74,15 @@ def _odps_read(self): table_path, selected_cols=selected_cols, slice_id=self._task_index, - slice_count=self._task_num + slice_count=self._task_num, ) total_records_num = reader.get_row_count() batch_num = int(total_records_num / self._data_config.batch_size) res_num = total_records_num - batch_num * self._data_config.batch_size - batch_defaults = [ - [x] * self._data_config.batch_size for x in record_defaults - ] + batch_defaults = [[x] * self._data_config.batch_size for x in record_defaults] for batch_id in range(batch_num): batch_data_np = [x.copy() for x in batch_defaults] - for row_id, one_data in enumerate( - reader.read(self._data_config.batch_size) - ): + for row_id, one_data in enumerate(reader.read(self._data_config.batch_size)): for col_id in range(len(record_defaults)): if one_data[col_id] not in ['', 'NULL', None]: batch_data_np[col_id][row_id] = one_data[col_id] @@ -107,38 +105,31 @@ def _build(self, mode, params): list_shapes = tuple(list_shapes) # read odps tables - dataset = tf.data.Dataset.from_generator( - self._odps_read, output_types=list_type, output_shapes=list_shapes - ) + dataset = tf.data.Dataset.from_generator(self._odps_read, output_types=list_type, output_shapes=list_shapes) if mode == tf.estimator.ModeKeys.TRAIN: dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) - dataset = dataset.map( - self._parse_table, - num_parallel_calls=self._data_config.num_parallel_calls - ) + dataset = dataset.map(self._parse_table, num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/odps_rtp_input.py b/easy_rec/python/input/odps_rtp_input.py index 359938be5..70d047221 100644 --- a/easy_rec/python/input/odps_rtp_input.py +++ b/easy_rec/python/input/odps_rtp_input.py @@ -37,67 +37,58 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(OdpsRTPInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config - ) - logging.info( - 'input_fields: %s label_fields: %s' % - (','.join(self._input_fields), ','.join(self._label_fields)) + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) + logging.info('input_fields: %s label_fields: %s' % (','.join(self._input_fields), ','.join(self._label_fields))) def _parse_table(self, *fields): fields = list(fields) labels = fields[:-1] - selected_cols = self._data_config.selected_cols \ - if self._data_config.selected_cols else None + selected_cols = self._data_config.selected_cols if self._data_config.selected_cols else None non_feature_cols = self._label_fields if selected_cols: cols = [c.strip() for c in selected_cols.split(',')] non_feature_cols = cols[:-1] # only for features, labels and sample_weight excluded - record_types = [ - t for x, t in zip(self._input_fields, self._input_field_types) - if x not in non_feature_cols - ] + record_types = [t for x, t in zip(self._input_fields, self._input_field_types) if x not in non_feature_cols] record_defaults = [ - self.get_type_defaults(t, v) for x, t, v in zip( - self._input_fields, self._input_field_types, self._input_field_defaults - ) if x not in non_feature_cols + self.get_type_defaults(t, v) + for x, t, v in zip(self._input_fields, self._input_field_types, self._input_field_defaults) + if x not in non_feature_cols ] feature_num = len(record_types) # assume that the last field is the generated feature column - print( - 'field_delim = %s, feature_num = %d' % - (self._data_config.separator, feature_num) - ) - logging.info( - 'field_delim = %s, input_field_name = %d' % - (self._data_config.separator, len(record_types)) + print('field_delim = %s, feature_num = %d' % (self._data_config.separator, feature_num)) + logging.info('field_delim = %s, input_field_name = %d' % (self._data_config.separator, len(record_types))) + + check_list = ( + [ + tf.py_func( + check_split, + [fields[-1], self._data_config.separator, len(record_types)], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] ) - - check_list = [ - tf.py_func( - check_split, - [fields[-1], self._data_config.separator, - len(record_types)], - Tout=tf.bool - ) - ] if self._check_mode else [] with tf.control_dependencies(check_list): - fields = str_split_by_chr( - fields[-1], self._data_config.separator, skip_empty=False - ) + fields = str_split_by_chr(fields[-1], self._data_config.separator, skip_empty=False) tmp_fields = tf.reshape(fields.values, [-1, feature_num]) - fields = labels[len(self._label_fields):] + fields = labels[len(self._label_fields) :] for i in range(feature_num): - field = string_to_number( - tmp_fields[:, i], record_types[i], record_defaults[i], i - ) + field = string_to_number(tmp_fields[:, i], record_types[i], record_defaults[i], i) fields.append(field) field_keys = [x for x in self._input_fields if x not in self._label_fields] @@ -106,75 +97,69 @@ def _parse_table(self, *fields): for x in range(len(self._label_fields)): inputs[self._label_fields[x]] = labels[x] - print( - 'effective field num = %d, input_num = %d' % (len(fields), len(inputs)) - ) + print('effective field num = %d, input_num = %d' % (len(fields), len(inputs))) return inputs def _build(self, mode, params): if type(self._input_path) != list: self._input_path = self._input_path.split(',') - assert len( - self._input_path - ) > 0, 'match no files with %s' % self._input_path + assert len(self._input_path) > 0, 'match no files with %s' % self._input_path - selected_cols = self._data_config.selected_cols \ - if self._data_config.selected_cols else None + selected_cols = self._data_config.selected_cols if self._data_config.selected_cols else None if selected_cols: cols = [c.strip() for c in selected_cols.split(',')] record_defaults = [ - self.get_type_defaults(t, v) for x, t, v in zip( - self._input_fields, self._input_field_types, - self._input_field_defaults - ) if x in cols[:-1] + self.get_type_defaults(t, v) + for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) + if x in cols[:-1] ] - print( - 'selected_cols: %s; defaults num: %d' % - (','.join(cols), len(record_defaults)) - ) + print('selected_cols: %s; defaults num: %d' % (','.join(cols), len(record_defaults))) else: record_defaults = [ - self.get_type_defaults(t, v) for x, t, v in zip( - self._input_fields, self._input_field_types, - self._input_field_defaults - ) if x in self._label_fields + self.get_type_defaults(t, v) + for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) + if x in self._label_fields ] # the actual features are in one single column record_defaults.append( self._data_config.separator.join( [ - str(self.get_type_defaults(t, v)) for x, t, v in zip( - self._input_fields, self._input_field_types, - self._input_field_defaults - ) if x not in self._label_fields + str(self.get_type_defaults(t, v)) + for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) + if x not in self._label_fields ] ) ) - if self._data_config.pai_worker_queue and \ - mode == tf.estimator.ModeKeys.TRAIN: - logging.info( - 'pai_worker_slice_num = %d' % self._data_config.pai_worker_slice_num - ) + if self._data_config.pai_worker_queue and mode == tf.estimator.ModeKeys.TRAIN: + logging.info('pai_worker_slice_num = %d' % self._data_config.pai_worker_slice_num) work_queue = pai.data.WorkQueue( self._input_path, num_epochs=self.num_epochs, shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num + num_slices=self._data_config.pai_worker_slice_num * self._task_num, ) que_paths = work_queue.input_dataset() - dataset = tf.data.TableRecordDataset( - que_paths, - record_defaults=record_defaults, - selected_cols=selected_cols - ) + dataset = tf.data.TableRecordDataset(que_paths, record_defaults=record_defaults, selected_cols=selected_cols) else: dataset = tf.data.TableRecordDataset( self._input_path, record_defaults=record_defaults, selected_cols=selected_cols, slice_id=self._task_index, - slice_count=self._task_num + slice_count=self._task_num, ) if mode == tf.estimator.ModeKeys.TRAIN: @@ -182,7 +167,7 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: @@ -190,24 +175,19 @@ def _build(self, mode, params): dataset = dataset.batch(batch_size=self._data_config.batch_size) - dataset = dataset.map( - self._parse_table, - num_parallel_calls=self._data_config.num_parallel_calls - ) + dataset = dataset.map(self._parse_table, num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/odps_rtp_input_v2.py b/easy_rec/python/input/odps_rtp_input_v2.py index 8f070c3d0..ba988d9d1 100644 --- a/easy_rec/python/input/odps_rtp_input_v2.py +++ b/easy_rec/python/input/odps_rtp_input_v2.py @@ -40,11 +40,16 @@ def __init__( task_num=1, check_mode=False, fg_json_path=None, - pipeline_config=None + pipeline_config=None, ): super(OdpsRTPInputV2, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) if fg_json_path.startswith('!'): fg_json_path = fg_json_path[1:] @@ -89,15 +94,11 @@ def create_placeholders(self, *args, **kwargs): def create_multi_placeholders(self, *args, **kwargs): """Create serving multi-placeholders with rtp_fg.""" - raise NotImplementedError( - 'create_multi_placeholders is not supported for OdpsRTPInputV2' - ) + raise NotImplementedError('create_multi_placeholders is not supported for OdpsRTPInputV2') def check_rtp(self): if rtp_fg is None: - raise NotImplementedError( - 'OdpsRTPInputV2 cannot run without rtp_fg, which is not installed' - ) + raise NotImplementedError('OdpsRTPInputV2 cannot run without rtp_fg, which is not installed') def _pre_build(self, mode, params): try: diff --git a/easy_rec/python/input/parquet_input.py b/easy_rec/python/input/parquet_input.py index 3fdc84a3e..027ac6c6e 100644 --- a/easy_rec/python/input/parquet_input.py +++ b/easy_rec/python/input/parquet_input.py @@ -17,7 +17,6 @@ class ParquetInput(Input): - def __init__( self, data_config, @@ -27,11 +26,17 @@ def __init__( task_num=1, check_mode=False, pipeline_config=None, - **kwargs + **kwargs, ): super(ParquetInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config, **kwargs + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + **kwargs, ) self._need_pack = True if input_path is None: @@ -40,20 +45,12 @@ def __init__( self._input_files = [] for sub_path in input_path.strip().split(','): self._input_files.extend(tf.gfile.Glob(sub_path)) - logging.info( - 'parquet input_path=%s file_num=%d' % - (input_path, len(self._input_files)) - ) + logging.info('parquet input_path=%s file_num=%d' % (input_path, len(self._input_files))) mp_ctxt = multiprocessing.get_context('spawn') - self._data_que = queues.Queue( - name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size - ) + self._data_que = queues.Queue(name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size) file_num = len(self._input_files) - logging.info( - '[task_index=%d] total_file_num=%d task_num=%d' % - (task_index, file_num, task_num) - ) + logging.info('[task_index=%d] total_file_num=%d task_num=%d' % (task_index, file_num, task_num)) self._my_files = [] for file_id in range(file_num): @@ -61,9 +58,7 @@ def __init__( self._my_files.append(self._input_files[file_id]) # self._my_files = self._input_files - logging.info( - '[task_index=%d] task_file_num=%d' % (task_index, len(self._my_files)) - ) + logging.info('[task_index=%d] task_file_num=%d' % (task_index, len(self._my_files))) self._file_que = queues.Queue(name='file_que', ctx=mp_ctxt) self._num_proc = 8 @@ -108,9 +103,7 @@ def __init__( def _rebuild_que(self): mp_ctxt = multiprocessing.get_context('spawn') - self._data_que = queues.Queue( - name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size - ) + self._data_que = queues.Queue(name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size) self._file_que = queues.Queue(name='file_que', ctx=mp_ctxt) self._proc_start_que = queues.Queue(name='proc_start_que', ctx=mp_ctxt) self._proc_stop_que = queues.Queue(name='proc_stop_que', ctx=mp_ctxt) @@ -118,12 +111,9 @@ def _rebuild_que(self): def _sample_generator(self): if not self._proc_start: self._proc_start = True - for proc in (self._proc_arr): + for proc in self._proc_arr: self._proc_start_que.put(True) - logging.info( - 'task[%s] data_proc=%s is_alive=%s' % - (self._task_index, proc, proc.is_alive()) - ) + logging.info('task[%s] data_proc=%s is_alive=%s' % (self._task_index, proc, proc.is_alive())) done_proc_cnt = 0 fetch_timeout_cnt = 0 @@ -154,36 +144,28 @@ def _sample_generator(self): yield sample if fetch_good_cnt % 200 == 0: logging.info( - 'task[%d] fetch_batch_cnt=%d, fetch_timeout_cnt=%d, qsize=%d' % ( - self._task_index, fetch_good_cnt, fetch_timeout_cnt, - self._data_que.qsize() + 'task[%d] fetch_batch_cnt=%d, fetch_timeout_cnt=%d, qsize=%d' + % ( + self._task_index, + fetch_good_cnt, + fetch_timeout_cnt, + self._data_que.qsize(), ) ) except queue.Empty: fetch_timeout_cnt += 1 if done_proc_cnt >= len(self._proc_arr): - logging.info( - 'all sample finished, fetch_timeout_cnt=%d' % fetch_timeout_cnt - ) + logging.info('all sample finished, fetch_timeout_cnt=%d' % fetch_timeout_cnt) break except Exception as ex: - logging.warning( - 'task[%d] get from data_que exception: %s' % - (self._task_index, str(ex)) - ) + logging.warning('task[%d] get from data_que exception: %s' % (self._task_index, str(ex))) break - logging.info( - 'task[%d] sample_generator: total_batches=%d' % - (self._task_index, fetch_good_cnt) - ) + logging.info('task[%d] sample_generator: total_batches=%d' % (self._task_index, fetch_good_cnt)) def stop(self): if self._proc_arr is None or len(self._proc_arr) == 0: return - logging.info( - 'task[%d] will stop dataset procs, proc_num=%d' % - (self._task_index, len(self._proc_arr)) - ) + logging.info('task[%d] will stop dataset procs, proc_num=%d' % (self._task_index, len(self._proc_arr))) self._file_que.close() if self._proc_start: logging.info('try close data que') @@ -224,13 +206,17 @@ def _to_fea_dict(self, input_dict): if len(self._sparse_fea_names) > 0: if self._has_ev: - tmp_vals, tmp_lens = input_dict['sparse_fea'][1], input_dict[ - 'sparse_fea'][0] + tmp_vals, tmp_lens = ( + input_dict['sparse_fea'][1], + input_dict['sparse_fea'][0], + ) fea_dict['sparse_fea'] = (tmp_vals, tmp_lens) else: - tmp_vals, tmp_lens = input_dict['sparse_fea'][1], input_dict[ - 'sparse_fea'][0] + tmp_vals, tmp_lens = ( + input_dict['sparse_fea'][1], + input_dict['sparse_fea'][0], + ) num_buckets = -1 for fc in self._feature_configs: if fc.num_buckets > 0: @@ -238,7 +224,8 @@ def _to_fea_dict(self, input_dict): num_buckets = fc.num_buckets else: assert num_buckets == fc.num_buckets, 'all features must share the same buckets, but are %d and %s' % ( - num_buckets, str(fc) + num_buckets, + str(fc), ) fea_dict['sparse_fea'] = (tmp_vals % num_buckets, tmp_lens) @@ -266,20 +253,14 @@ def add_fea_type_and_shape(self, out_types, out_shapes): # second field: field values if len(self._sparse_fea_names) > 0: out_types['sparse_fea'] = (tf.int32, tf.int64) - out_shapes['sparse_fea'] = ( - tf.TensorShape([None]), tf.TensorShape([None]) - ) + out_shapes['sparse_fea'] = (tf.TensorShape([None]), tf.TensorShape([None])) if len(self._dense_fea_names) > 0: out_types['dense_fea'] = tf.float32 - out_shapes['dense_fea'] = tf.TensorShape( - [None, self._total_dense_fea_dim] - ) + out_shapes['dense_fea'] = tf.TensorShape([None, self._total_dense_fea_dim]) def _build(self, mode, params): if mode == tf.estimator.ModeKeys.TRAIN and self._data_config.num_epochs > 1: - logging.info( - 'will repeat train data for %d epochs' % self._data_config.num_epochs - ) + logging.info('will repeat train data for %d epochs' % self._data_config.num_epochs) my_files = self._my_files * self._data_config.num_epochs else: my_files = self._my_files @@ -308,7 +289,7 @@ def _build(self, mode, params): self._dense_fea_cfgs, self._reserve_fields, drop_remainder, - need_pack=self._need_pack + need_pack=self._need_pack, ) for input_file in my_files: @@ -317,9 +298,7 @@ def _build(self, mode, params): # add end signal for proc in self._proc_arr: self._file_que.put(None) - logging.info( - 'add input_files to file_que, qsize=%d' % self._file_que.qsize() - ) + logging.info('add input_files to file_que, qsize=%d' % self._file_que.qsize()) out_types = {} out_shapes = {} @@ -338,13 +317,9 @@ def _build(self, mode, params): self.add_fea_type_and_shape(out_types, out_shapes) - dataset = tf.data.Dataset.from_generator( - self._sample_generator, output_types=out_types, output_shapes=out_shapes - ) + dataset = tf.data.Dataset.from_generator(self._sample_generator, output_types=out_types, output_shapes=out_shapes) num_parallel_calls = self._data_config.num_parallel_calls - dataset = dataset.map( - self._to_fea_dict, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(self._to_fea_dict, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) # Note: Input._preprocess is currently not supported as all features @@ -353,9 +328,7 @@ def _build(self, mode, params): # map_func=self._preprocess, num_parallel_calls=num_parallel_calls) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) # initial test show that prefetch to gpu has no performance gain # dataset = dataset.apply(tf.data.experimental.prefetch_to_device('/gpu:0')) else: @@ -370,7 +343,7 @@ def _get_for_predictor(self, fea_dict): out_dict = { 'feature': { 'ragged_ids': fea_dict['feature']['sparse_fea'][0], - 'ragged_lens': fea_dict['feature']['sparse_fea'][1] + 'ragged_lens': fea_dict['feature']['sparse_fea'][1], } } if self._is_predictor and self._reserve_fields is not None: @@ -378,7 +351,6 @@ def _get_for_predictor(self, fea_dict): return out_dict def create_input(self, export_config=None): - def _input_fn(mode=None, params=None, config=None): """Build input_fn for estimator. @@ -395,8 +367,9 @@ def _input_fn(mode=None, params=None, config=None): tf.estimator.export.ServingInputReceiver instance """ if mode in ( - tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT + tf.estimator.ModeKeys.TRAIN, + tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT, ): # build dataset from self._config.input_path self._mode = mode @@ -405,24 +378,21 @@ def _input_fn(mode=None, params=None, config=None): elif mode is None: # serving_input_receiver_fn for export SavedModel inputs, features = {}, {} if len(self._sparse_fea_names) > 0: - ragged_ids = array_ops.placeholder( - tf.int64, [None], name='ragged_ids' - ) - ragged_lens = array_ops.placeholder( - tf.int32, [None], name='ragged_lens' - ) + ragged_ids = array_ops.placeholder(tf.int64, [None], name='ragged_ids') + ragged_lens = array_ops.placeholder(tf.int32, [None], name='ragged_lens') inputs = {'ragged_ids': ragged_ids, 'ragged_lens': ragged_lens} if self._has_ev: - features = {'ragged_ids': ragged_ids, 'ragged_lens': ragged_lens} + features = { + 'ragged_ids': ragged_ids, + 'ragged_lens': ragged_lens, + } else: features = { 'ragged_ids': ragged_ids % self._feature_configs[0].num_buckets, - 'ragged_lens': ragged_lens + 'ragged_lens': ragged_lens, } if len(self._dense_fea_names) > 0: - inputs['dense_fea'] = array_ops.placeholder( - tf.float32, [None, self._total_dense_fea_dim], name='dense_fea' - ) + inputs['dense_fea'] = array_ops.placeholder(tf.float32, [None, self._total_dense_fea_dim], name='dense_fea') features['dense_fea'] = inputs['dense_fea'] return tf.estimator.export.ServingInputReceiver(features, inputs) diff --git a/easy_rec/python/input/parquet_input_v2.py b/easy_rec/python/input/parquet_input_v2.py index af2c2414d..cf488dbb5 100644 --- a/easy_rec/python/input/parquet_input_v2.py +++ b/easy_rec/python/input/parquet_input_v2.py @@ -7,6 +7,7 @@ # import pandas as pd import tensorflow as tf from tensorflow.python.framework import dtypes, ops + # from tensorflow.python.ops import math_ops # from tensorflow.python.ops import logging_ops from tensorflow.python.ops import array_ops, string_ops @@ -18,7 +19,6 @@ class ParquetInputV2(ParquetInput): - def __init__( self, data_config, @@ -28,11 +28,17 @@ def __init__( task_num=1, check_mode=False, pipeline_config=None, - **kwargs + **kwargs, ): super(ParquetInputV2, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config, **kwargs + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + **kwargs, ) self._need_pack = False @@ -84,8 +90,7 @@ def _preprocess(self, inputs=None): features = {} placeholders = {} for fc in self._feature_configs: - feature_name = fc.feature_name if fc.feature_name != '' else fc.input_names[ - 0] + feature_name = fc.feature_name if fc.feature_name != '' else fc.input_names[0] feature_type = fc.feature_type if feature_type in [fc.IdFeature, fc.TagFeature]: input_name0 = fc.input_names[0] @@ -95,21 +100,15 @@ def _preprocess(self, inputs=None): if input_name0 in placeholders: input_lens, input_vals = placeholders[input_name0] else: - input_vals = array_ops.placeholder( - dtypes.int64, [None], name=input_name0 + '/ids' - ) - input_lens = array_ops.placeholder( - dtypes.int64, [None], name=input_name0 + '/lens' - ) + input_vals = array_ops.placeholder(dtypes.int64, [None], name=input_name0 + '/ids') + input_lens = array_ops.placeholder(dtypes.int64, [None], name=input_name0 + '/lens') placeholders[input_name0] = (input_lens, input_vals) if not self._has_ev: if fc.num_buckets > 0: input_vals = input_vals % fc.num_buckets else: input_vals = string_ops.as_string(input_vals) - features[feature_name] = tf.RaggedTensor.from_row_lengths( - values=input_vals, row_lengths=input_lens - ) + features[feature_name] = tf.RaggedTensor.from_row_lengths(values=input_vals, row_lengths=input_lens) elif feature_type in [fc.RawFeature]: input_name0 = fc.input_names[0] if inputs is not None: @@ -120,12 +119,12 @@ def _preprocess(self, inputs=None): else: if fc.raw_input_dim > 1: input_vals = array_ops.placeholder( - dtypes.float32, [None, fc.raw_input_dim], name=input_name0 + dtypes.float32, + [None, fc.raw_input_dim], + name=input_name0, ) else: - input_vals = array_ops.placeholder( - dtypes.float32, [None], name=input_name0 - ) + input_vals = array_ops.placeholder(dtypes.float32, [None], name=input_name0) placeholders[input_name0] = input_vals features[feature_name] = input_vals else: @@ -153,7 +152,6 @@ def _get_for_predictor(self, fea_dict): return fea_dict def create_input(self, export_config=None): - def _input_fn(mode=None, params=None, config=None): """Build input_fn for estimator. @@ -170,8 +168,9 @@ def _input_fn(mode=None, params=None, config=None): tf.estimator.export.ServingInputReceiver instance """ if mode in ( - tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT + tf.estimator.ModeKeys.TRAIN, + tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT, ): # build dataset from self._config.input_path self._mode = mode diff --git a/easy_rec/python/input/parquet_input_v3.py b/easy_rec/python/input/parquet_input_v3.py index f779c3249..b18dac429 100644 --- a/easy_rec/python/input/parquet_input_v3.py +++ b/easy_rec/python/input/parquet_input_v3.py @@ -8,20 +8,23 @@ from easy_rec.python.utils.input_utils import get_type_defaults try: - from tensorflow.python.data.experimental.ops import dataframe, parquet_dataset_ops, parquet_pybind # NOQA + from tensorflow.python.data.experimental.ops import ( # NOQA + dataframe, + parquet_dataset_ops, + parquet_pybind, + ) from tensorflow.python.ops import gen_ragged_conversion_ops from tensorflow.python.ops.work_queue import WorkQueue + _has_deep_rec = True except Exception: _has_deep_rec = False - pass if tf.__version__ >= '2.0': tf = tf.compat.v1 class ParquetInputV3(Input): - def __init__( self, data_config, @@ -31,26 +34,29 @@ def __init__( task_num=1, check_mode=False, pipeline_config=None, - **kwargs + **kwargs, ): if not _has_deep_rec: raise RuntimeError('You should install DeepRec first.') super(ParquetInputV3, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) self._ignore_val_dict = {} for f in data_config.input_fields: if f.HasField('ignore_val'): - self._ignore_val_dict[f.input_name - ] = get_type_defaults(f.input_type, f.ignore_val) + self._ignore_val_dict[f.input_name] = get_type_defaults(f.input_type, f.ignore_val) self._true_type_dict = {} for fc in self._feature_configs: if fc.feature_type in [fc.IdFeature, fc.TagFeature, fc.SequenceFeature]: - if fc.hash_bucket_size > 0 or len(fc.vocab_list - ) > 0 or fc.HasField('vocab_file'): + if fc.hash_bucket_size > 0 or len(fc.vocab_list) > 0 or fc.HasField('vocab_file'): self._true_type_dict[fc.input_names[0]] = tf.string else: self._true_type_dict[fc.input_names[0]] = tf.int64 @@ -75,14 +81,15 @@ def _ignore_and_cast(self, name, value): indices = tf.where(tf.equal(value.values, ignore_value)) value = tf.SparseTensor( tf.gather_nd(value.indices, indices), - tf.gather_nd(value.values, indices), value.dense_shape + tf.gather_nd(value.values, indices), + value.dense_shape, ) elif isinstance(value, tf.Tensor): indices = tf.where(tf.not_equal(value, ignore_value), name='indices') value = tf.SparseTensor( indices=indices, values=tf.gather_nd(value, indices), - dense_shape=tf.shape(value, out_type=tf.int64) + dense_shape=tf.shape(value, out_type=tf.int64), ) dtype = self._true_type_dict.get(name, None) if dtype: @@ -93,12 +100,11 @@ def _parse_dataframe_value(self, value): if len(value.nested_row_splits) == 0: return value.values value.values.set_shape([None]) - sparse_value = gen_ragged_conversion_ops.ragged_tensor_to_sparse( - value.nested_row_splits, value.values - ) + sparse_value = gen_ragged_conversion_ops.ragged_tensor_to_sparse(value.nested_row_splits, value.values) return tf.SparseTensor( - sparse_value.sparse_indices, sparse_value.sparse_values, - sparse_value.sparse_dense_shape + sparse_value.sparse_indices, + sparse_value.sparse_values, + sparse_value.sparse_dense_shape, ) def _parse_dataframe(self, df): @@ -123,10 +129,7 @@ def _build(self, mode, params): for sub_path in self._input_path.strip().split(','): input_files.extend(tf.gfile.Glob(sub_path)) file_num = len(input_files) - logging.info( - '[task_index=%d] total_file_num=%d task_num=%d' % - (self._task_index, file_num, self._task_num) - ) + logging.info('[task_index=%d] total_file_num=%d task_num=%d' % (self._task_index, file_num, self._task_num)) task_index = self._task_index task_num = self._task_num @@ -134,12 +137,11 @@ def _build(self, mode, params): task_index = max(self._task_index - 1, 0) task_num = max(self._task_num - 1, 1) - if self._data_config.pai_worker_queue and \ - mode == tf.estimator.ModeKeys.TRAIN: + if self._data_config.pai_worker_queue and mode == tf.estimator.ModeKeys.TRAIN: work_queue = WorkQueue( input_files, num_epochs=self.num_epochs, - shuffle=self._data_config.shuffle + shuffle=self._data_config.shuffle, ) my_files = work_queue.input_dataset() else: @@ -165,16 +167,13 @@ def _build(self, mode, params): if f.name in all_fields: selected_fields.append(f) - num_parallel_reads = min( - self._data_config.num_parallel_calls, - len(input_files) // task_num - ) + num_parallel_reads = min(self._data_config.num_parallel_calls, len(input_files) // task_num) dataset = parquet_dataset_ops.ParquetDataset( my_files, batch_size=self._batch_size, fields=selected_fields, drop_remainder=self._data_config.drop_remainder, - num_parallel_reads=num_parallel_reads + num_parallel_reads=num_parallel_reads, ) # partition_count=task_num, # partition_index=task_index) @@ -184,7 +183,7 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: @@ -192,22 +191,20 @@ def _build(self, mode, params): dataset = dataset.map( self._parse_dataframe, - num_parallel_calls=self._data_config.num_parallel_calls + num_parallel_calls=self._data_config.num_parallel_calls, ) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/rtp_input.py b/easy_rec/python/input/rtp_input.py index 1c677f414..d6da5539a 100644 --- a/easy_rec/python/input/rtp_input.py +++ b/easy_rec/python/input/rtp_input.py @@ -6,11 +6,13 @@ from easy_rec.python.input.input import Input from easy_rec.python.ops.gen_str_avx_op import str_split_by_chr +from easy_rec.python.utils.check_utils import ( # NOQA + check_split, + check_string_to_number, +) from easy_rec.python.utils.input_utils import string_to_number from easy_rec.python.utils.tf_utils import get_tf_type -from easy_rec.python.utils.check_utils import check_split, check_string_to_number # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -38,22 +40,22 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(RTPInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config - ) - logging.info( - 'input_fields: %s label_fields: %s' % - (','.join(self._input_fields), ','.join(self._label_fields)) + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) + logging.info('input_fields: %s label_fields: %s' % (','.join(self._input_fields), ','.join(self._label_fields))) self._rtp_separator = self._data_config.rtp_separator if not isinstance(self._rtp_separator, str): self._rtp_separator = self._rtp_separator.encode('utf-8') - self._selected_cols = [ - int(x) for x in self._data_config.selected_cols.split(',') - ] + self._selected_cols = [int(x) for x in self._data_config.selected_cols.split(',')] self._num_cols = -1 self._feature_col_id = self._selected_cols[-1] logging.info('rtp separator = %s' % self._rtp_separator) @@ -64,20 +66,27 @@ def _parse_csv(self, line): # the actual features are in one single column record_defaults[self._feature_col_id] = self._data_config.separator.join( [ - str(self.get_type_defaults(t, v)) for x, t, v in zip( - self._input_fields, self._input_field_types, - self._input_field_defaults - ) if x not in self._label_fields + str(self.get_type_defaults(t, v)) + for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) + if x not in self._label_fields ] ) - check_list = [ - tf.py_func( - check_split, [line, self._rtp_separator, - len(record_defaults)], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_split, + [line, self._rtp_separator, len(record_defaults)], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): fields = tf.string_split(line, self._rtp_separator, skip_empty=False) @@ -90,44 +99,38 @@ def _parse_csv(self, line): ftype = self._input_field_types[idx] tf_type = get_tf_type(ftype) if field.dtype in [tf.string]: - check_list = [ - tf.py_func(check_string_to_number, [field, fname], Tout=tf.bool) - ] if self._check_mode else [] + check_list = [tf.py_func(check_string_to_number, [field, fname], Tout=tf.bool)] if self._check_mode else [] with tf.control_dependencies(check_list): field = tf.string_to_number(field, tf_type) labels.append(field) # only for features, labels excluded - record_types = [ - t for x, t in zip(self._input_fields, self._input_field_types) - if x not in self._label_fields - ] + record_types = [t for x, t in zip(self._input_fields, self._input_field_types) if x not in self._label_fields] # assume that the last field is the generated feature column print('field_delim = %s' % self._data_config.separator) feature_str = fields[:, self._feature_col_id] - check_list = [ - tf.py_func( - check_split, - [feature_str, self._data_config.separator, - len(record_types)], - Tout=tf.bool - ) - ] if self._check_mode else [] + check_list = ( + [ + tf.py_func( + check_split, + [feature_str, self._data_config.separator, len(record_types)], + Tout=tf.bool, + ) + ] + if self._check_mode + else [] + ) with tf.control_dependencies(check_list): - fields = str_split_by_chr( - feature_str, self._data_config.separator, skip_empty=False - ) + fields = str_split_by_chr(feature_str, self._data_config.separator, skip_empty=False) tmp_fields = tf.reshape(fields.values, [-1, len(record_types)]) rtp_record_defaults = [ - str(self.get_type_defaults(t, v)) for x, t, v in zip( - self._input_fields, self._input_field_types, self._input_field_defaults - ) if x not in self._label_fields + str(self.get_type_defaults(t, v)) + for x, t, v in zip(self._input_fields, self._input_field_types, self._input_field_defaults) + if x not in self._label_fields ] fields = [] for i in range(len(record_types)): - field = string_to_number( - tmp_fields[:, i], record_types[i], rtp_record_defaults[i], i - ) + field = string_to_number(tmp_fields[:, i], record_types[i], rtp_record_defaults[i], i) fields.append(field) field_keys = [x for x in self._input_fields if x not in self._label_fields] @@ -152,9 +155,9 @@ def _build(self, mode, params): for line_str in fin: line_tok = line_str.strip().split(self._rtp_separator) if self._num_cols != -1: - assert self._num_cols == len(line_tok), \ - 'num selected cols is %d, not equal to %d, current line is: %s, please check rtp_separator and data.' % \ - (self._num_cols, len(line_tok), line_str) + assert self._num_cols == len(line_tok), ( + 'num selected cols is %d, not equal to %d, current line is: %s,' ' please check rtp_separator and data.' + ) % (self._num_cols, len(line_tok), line_str) self._num_cols = len(line_tok) num_lines += 1 if num_lines > 10: @@ -162,28 +165,29 @@ def _build(self, mode, params): logging.info('num selected cols = %d' % self._num_cols) record_defaults = [ - self.get_type_defaults(t, v) for x, t, v in zip( - self._input_fields, self._input_field_types, self._input_field_defaults - ) if x in self._label_fields + self.get_type_defaults(t, v) + for x, t, v in zip(self._input_fields, self._input_field_types, self._input_field_defaults) + if x in self._label_fields ] # the features are in one single column record_defaults.append( self._data_config.separator.join( [ - str(self.get_type_defaults(t, v)) for x, t, v in zip( - self._input_fields, self._input_field_types, - self._input_field_defaults - ) if x not in self._label_fields + str(self.get_type_defaults(t, v)) + for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) + if x not in self._label_fields ] ) ) num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info( - 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -199,7 +203,7 @@ def _build(self, mode, params): dataset = dataset.interleave( tf.data.TextLineDataset, cycle_length=parallel_num, - num_parallel_calls=parallel_num + num_parallel_calls=parallel_num, ) if not self._data_config.file_shard: @@ -209,35 +213,29 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info( - 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(batch_size=self._data_config.batch_size) - dataset = dataset.map( - self._parse_csv, num_parallel_calls=self._data_config.num_parallel_calls - ) + dataset = dataset.map(self._parse_csv, num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/rtp_input_v2.py b/easy_rec/python/input/rtp_input_v2.py index d427c3e0b..3ad32f2ce 100644 --- a/easy_rec/python/input/rtp_input_v2.py +++ b/easy_rec/python/input/rtp_input_v2.py @@ -25,11 +25,16 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(RTPInputV2, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) def _parse_rtp(self, lines): @@ -43,21 +48,17 @@ def _parse_one_line_tf(line): keys = field_vals[:, 0] vals = field_vals[:, 1] temp_vals = [ - str( - self.get_type_defaults( - self._input_field_types[i], self._input_field_defaults[i] - ) - ) for i in range(len(self._input_fields)) + str(self.get_type_defaults(self._input_field_types[i], self._input_field_defaults[i])) + for i in range(len(self._input_fields)) ] for i, key in enumerate(self._input_fields): msk = tf.equal(key, keys) val = tf.boolean_mask(vals, msk) - def_val = self.get_type_defaults( - self._input_field_types[i], self._input_field_defaults[i] - ) + def_val = self.get_type_defaults(self._input_field_types[i], self._input_field_defaults[i]) temp_vals[i] = tf.cond( - tf.reduce_any(msk), lambda: tf.reduce_join(val, separator=','), - lambda: tf.constant(str(def_val)) + tf.reduce_any(msk), + lambda: tf.reduce_join(val, separator=','), + lambda: tf.constant(str(def_val)), ) return temp_vals @@ -66,27 +67,20 @@ def _parse_one_line_tf(line): lines, tf_types, parallel_iterations=64, - name='parse_one_line_tf_map_fn' + name='parse_one_line_tf_map_fn', ) def _convert(x, target_type, name): if target_type in [DatasetConfig.FLOAT, DatasetConfig.DOUBLE]: - return tf.string_to_number( - x, tf.float32, name='convert_input_flt32/%s' % name - ) + return tf.string_to_number(x, tf.float32, name='convert_input_flt32/%s' % name) elif target_type == DatasetConfig.INT32: - return tf.string_to_number( - x, tf.int32, name='convert_input_int32/%s' % name - ) + return tf.string_to_number(x, tf.int32, name='convert_input_int32/%s' % name) elif target_type == DatasetConfig.INT64: - return tf.string_to_number( - x, tf.int64, name='convert_input_int64/%s' % name - ) + return tf.string_to_number(x, tf.int64, name='convert_input_int64/%s' % name) return x inputs = { - self._input_fields[x]: - _convert(fields[x], self._input_field_types[x], self._input_fields[x]) + self._input_fields[x]: _convert(fields[x], self._input_field_types[x], self._input_fields[x]) for x in self._effective_fids } @@ -104,9 +98,7 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info( - 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -122,7 +114,7 @@ def _build(self, mode, params): dataset = dataset.interleave( tf.data.TextLineDataset, cycle_length=parallel_num, - num_parallel_calls=parallel_num + num_parallel_calls=parallel_num, ) if not self._data_config.file_shard: @@ -132,31 +124,23 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info( - 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map( - self._parse_rtp, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(self._parse_rtp, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/tfrecord_input.py b/easy_rec/python/input/tfrecord_input.py index 010eec649..9d778fb99 100644 --- a/easy_rec/python/input/tfrecord_input.py +++ b/easy_rec/python/input/tfrecord_input.py @@ -12,7 +12,6 @@ class TFRecordInput(Input): - def __init__( self, data_config, @@ -21,22 +20,23 @@ def __init__( task_index=0, task_num=1, check_mode=False, - pipeline_config=None + pipeline_config=None, ): super(TFRecordInput, self).__init__( - data_config, feature_config, input_path, task_index, task_num, - check_mode, pipeline_config + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) self.feature_desc = {} - for x, t, d in zip( - self._input_fields, self._input_field_types, self._input_field_defaults - ): + for x, t, d in zip(self._input_fields, self._input_field_types, self._input_field_defaults): d = self.get_type_defaults(t, d) t = get_tf_type(t) - self.feature_desc[x] = tf.FixedLenFeature( - dtype=t, shape=1, default_value=d - ) + self.feature_desc[x] = tf.FixedLenFeature(dtype=t, shape=1, default_value=d) def _parse_tfrecord(self, example): try: @@ -56,9 +56,7 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls data_compression_type = self._data_config.data_compression_type if mode == tf.estimator.ModeKeys.TRAIN: - logging.info( - 'train files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) + logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.shuffle: # shuffle input files @@ -67,43 +65,32 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data. - TFRecordDataset(x, compression_type=data_compression_type), + lambda x: tf.data.TFRecordDataset(x, compression_type=data_compression_type), cycle_length=parallel_num, - num_parallel_calls=parallel_num + num_parallel_calls=parallel_num, ) dataset = dataset.shard(self._task_num, self._task_index) if self._data_config.shuffle: dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info( - 'eval files[%d]: %s' % (len(file_paths), ','.join(file_paths)) - ) - dataset = tf.data.TFRecordDataset( - file_paths, compression_type=data_compression_type - ) + logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + dataset = tf.data.TFRecordDataset(file_paths, compression_type=data_compression_type) dataset = dataset.repeat(1) - dataset = dataset.map( - self._parse_tfrecord, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(self._parse_tfrecord, num_parallel_calls=num_parallel_calls) dataset = dataset.batch(self._data_config.batch_size) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map( - map_func=self._preprocess, num_parallel_calls=num_parallel_calls - ) + dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map( - lambda x: (self._get_features(x), self._get_labels(x)) - ) + dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/layers/backbone.py b/easy_rec/python/layers/backbone.py index c82fd3454..404d3fd0e 100644 --- a/easy_rec/python/layers/backbone.py +++ b/easy_rec/python/layers/backbone.py @@ -20,6 +20,7 @@ class Package(object): """A sub DAG of tf ops for reuse.""" + __packages = {} @staticmethod @@ -59,15 +60,11 @@ def __init__(self, config, features, input_layer, l2_reg=None): layer = block.WhichOneof('layer') if layer in {'input_layer', 'raw_input', 'embedding_layer'}: if len(block.inputs) != 1: - raise ValueError( - 'input layer `%s` takes only one input' % block.name - ) + raise ValueError('input layer `%s` takes only one input' % block.name) one_input = block.inputs[0] name = one_input.WhichOneof('name') if name != 'feature_group_name': - raise KeyError( - '`feature_group_name` should be set for input layer: ' + block.name - ) + raise KeyError('`feature_group_name` should be set for input layer: ' + block.name) group = one_input.feature_group_name if not input_layer.has_group(group): raise KeyError('invalid feature group name: ' + group) @@ -85,25 +82,25 @@ def __init__(self, config, features, input_layer, l2_reg=None): self._name_to_layer[block.name] = input_fn else: if layer == 'input_layer': - input_fn = EnhancedInputLayer( - self._input_layer, self._features, group, reuse - ) + input_fn = EnhancedInputLayer(self._input_layer, self._features, group, reuse) input_feature_groups[group] = input_fn elif layer == 'raw_input': - input_fn = self._input_layer.get_raw_features( - self._features, group - ) + input_fn = self._input_layer.get_raw_features(self._features, group) input_feature_groups[group] = input_fn else: # embedding_layer - inputs, vocab, weights = self._input_layer.get_bucketized_features( - self._features, group - ) + ( + inputs, + vocab, + weights, + ) = self._input_layer.get_bucketized_features(self._features, group) block.embedding_layer.vocab_size = vocab params = Parameter.make_from_pb(block.embedding_layer) input_fn = EmbeddingLayer(params, block.name) input_feature_groups[group] = (inputs, vocab, weights) logging.info( - 'add an embedding layer %s with vocab size %d', block.name, vocab + 'add an embedding layer %s with vocab size %d', + block.name, + vocab, ) self._name_to_layer[block.name] = input_fn else: @@ -173,8 +170,7 @@ def __init__(self, config, features, input_layer, l2_reg=None): num_pkg_input += 1 else: raise KeyError( - 'invalid input name `%s`, must be the name of either a feature group or an another block' - % iname + 'invalid input name `%s`, must be the name of either a feature group or an another block' % iname ) num_groups = len(input_feature_groups) assert num_pkg_input > 0 or num_groups > 0, 'there must be at least one input layer/feature group' @@ -188,9 +184,7 @@ def __init__(self, config, features, input_layer, l2_reg=None): self._config.concat_blocks.extend(leaf) Package.__packages[self._config.name] = self - logging.info( - '%s layers: %s' % (config.name, ','.join(self._name_to_layer.keys())) - ) + logging.info('%s layers: %s' % (config.name, ','.join(self._name_to_layer.keys()))) def define_layers(self, layer, layer_cnf, name, reuse): if layer == 'keras_layer': @@ -241,9 +235,7 @@ def block_input(self, config, block_outputs, training=None, **kwargs): pkg_input = block_outputs[pkg_input_name] else: if pkg_input_name not in Package.__packages: - raise KeyError( - 'package name `%s` does not exists' % pkg_input_name - ) + raise KeyError('package name `%s` does not exists' % pkg_input_name) inner_package = Package.__packages[pkg_input_name] pkg_input = inner_package(training) if input_node.HasField('package_input_fn'): @@ -304,9 +296,7 @@ def call(self, is_training, **kwargs): output = self.block_input(config, block_outputs, is_training, **kwargs) for i, layer in enumerate(config.layers): name_i = '%s_l%d' % (block, i) - output = self.call_layer( - output, layer, name_i, is_training, **kwargs - ) + output = self.call_layer(output, layer, name_i, is_training, **kwargs) block_outputs[block] = output continue # just one of layer @@ -330,9 +320,7 @@ def call(self, is_training, **kwargs): block_outputs[block] = input_fn([inputs, weights], is_training) else: with tf.name_scope(block + '_input'): - inputs = self.block_input( - config, block_outputs, is_training, **kwargs - ) + inputs = self.block_input(config, block_outputs, is_training, **kwargs) output = self.call_layer(inputs, config, block, is_training, **kwargs) block_outputs[block] = output @@ -363,9 +351,7 @@ def call(self, is_training, **kwargs): def load_keras_layer(self, layer_conf, name, reuse=None): layer_cls, customize = load_keras_layer(layer_conf.class_name) if layer_cls is None: - raise ValueError( - 'Invalid keras layer class name: ' + layer_conf.class_name - ) + raise ValueError('Invalid keras layer class name: ' + layer_conf.class_name) param_type = layer_conf.WhichOneof('params') if customize: @@ -378,11 +364,13 @@ def load_keras_layer(self, layer_conf, name, reuse=None): has_reuse = True try: from funcsigs import signature + sig = signature(layer_cls.__init__) has_reuse = 'reuse' in sig.parameters.keys() except ImportError: try: from sklearn.externals.funcsigs import signature + sig = signature(layer_cls.__init__) has_reuse = 'reuse' in sig.parameters.keys() except ImportError: @@ -400,17 +388,12 @@ def load_keras_layer(self, layer_conf, name, reuse=None): assert param_type == 'st_params', 'internal keras layer only support st_params' try: kwargs = convert_to_dict(layer_conf.st_params) - logging.info( - 'call %s layer with params %r' % (layer_conf.class_name, kwargs) - ) + logging.info('call %s layer with params %r' % (layer_conf.class_name, kwargs)) layer = layer_cls(name=name, **kwargs) except TypeError as e: logging.warning(e) args = map(format_value, layer_conf.st_params.values()) - logging.info( - 'try to call %s layer with params %r' % - (layer_conf.class_name, args) - ) + logging.info('try to call %s layer with params %r' % (layer_conf.class_name, args)) layer = layer_cls(*args, name=name) return layer, customize @@ -553,15 +536,14 @@ def merge_inputs(inputs, axis=-1, msg=''): return inputs[0] from functools import reduce + if all(map(lambda x: type(x) == list, inputs)): # merge multiple lists into a list return reduce(lambda x, y: x + y, inputs) if any(map(lambda x: type(x) == list, inputs)): logging.warning('%s: try to merge inputs into list' % msg) - return reduce( - lambda x, y: x + y, [e if type(e) == list else [e] for e in inputs] - ) + return reduce(lambda x, y: x + y, [e if type(e) == list else [e] for e in inputs]) if axis != -1: logging.info('concat inputs %s axis=%d' % (msg, axis)) diff --git a/easy_rec/python/layers/capsule_layer.py b/easy_rec/python/layers/capsule_layer.py index a710a9e49..82cb0e46c 100644 --- a/easy_rec/python/layers/capsule_layer.py +++ b/easy_rec/python/layers/capsule_layer.py @@ -10,7 +10,6 @@ class CapsuleLayer: - def __init__(self, capsule_config, is_training): # max_seq_len: max behaviour sequence length(history length) self._max_seq_len = capsule_config.max_seq_len @@ -35,16 +34,14 @@ def squash(self, inputs): """Squash inputs over the last dimension.""" input_norm = tf.reduce_sum(tf.square(inputs), keep_dims=True, axis=-1) input_norm_eps = tf.maximum(input_norm, 1e-8) - scale_factor = tf.pow(input_norm_eps / (1 + input_norm_eps), self._squash_pow) * \ - self._scale_ratio / tf.sqrt(input_norm_eps) + scale_factor = ( + tf.pow(input_norm_eps / (1 + input_norm_eps), self._squash_pow) * self._scale_ratio / tf.sqrt(input_norm_eps) + ) tf.summary.histogram('capsule/squash_scale_factor', scale_factor) return scale_factor * inputs def _build_capsule_simi(self, high_capsules, capsule_num): - high_capsule_mask = tf.sequence_mask( - capsule_num, - tf.shape(high_capsules)[1] - ) + high_capsule_mask = tf.sequence_mask(capsule_num, tf.shape(high_capsules)[1]) high_capsules = high_capsules * tf.to_float(high_capsule_mask[:, :, None]) high_capsules = tf.nn.l2_normalize(high_capsules, axis=-1) sum_sqr = tf.square(tf.reduce_sum(high_capsules, axis=1)) @@ -55,8 +52,7 @@ def _build_capsule_simi(self, high_capsules, capsule_num): simi = tf.reduce_sum(simi, axis=1) / div is_multi = tf.to_float(capsule_num > 1) - avg_simi = tf.reduce_sum((simi + 1) * is_multi) / \ - (2.0 * tf.reduce_sum(is_multi)) + avg_simi = tf.reduce_sum((simi + 1) * is_multi) / (2.0 * tf.reduce_sum(is_multi)) return avg_simi def __call__(self, seq_feas, seq_lens): @@ -72,12 +68,15 @@ def __call__(self, seq_feas, seq_lens): # pad or clip to max_seq_len seq_feas = tf.cond( tf.greater(tf.shape(seq_feas)[1], self._max_seq_len), - lambda: seq_feas[:, :self._max_seq_len, :], lambda: tf.cond( - tf.less(tf.shape(seq_feas)[1], self._max_seq_len), lambda: tf.pad( + lambda: seq_feas[:, : self._max_seq_len, :], + lambda: tf.cond( + tf.less(tf.shape(seq_feas)[1], self._max_seq_len), + lambda: tf.pad( seq_feas, - [[0, 0], [0, self._max_seq_len - tf.shape(seq_feas)[1]], [0, 0]] - ), lambda: seq_feas - ) + [[0, 0], [0, self._max_seq_len - tf.shape(seq_feas)[1]], [0, 0]], + ), + lambda: seq_feas, + ), ) seq_lens = tf.minimum(seq_lens, self._max_seq_len) @@ -86,16 +85,16 @@ def __call__(self, seq_feas, seq_lens): if self._is_training: routing_logits = tf.truncated_normal( [batch_size, self._max_seq_len, self._max_k], - stddev=self._routing_logits_stddev + stddev=self._routing_logits_stddev, ) else: np.random.seed(28) routing_logits = tf.constant( np.random.uniform( high=self._routing_logits_stddev, - size=[self._max_seq_len, self._max_k] + size=[self._max_seq_len, self._max_k], ), - dtype=tf.float32 + dtype=tf.float32, ) routing_logits = tf.tile(routing_logits[None, :, :], [batch_size, 1, 1]) routing_logits = tf.stop_gradient(routing_logits) @@ -103,9 +102,7 @@ def __call__(self, seq_feas, seq_lens): low_fea_dim = seq_feas.get_shape()[-1] # map low capsule features to high capsule features: # low_fea_dim x high_dim(de) - bilinear_matrix = tf.get_variable( - dtype=tf.float32, shape=[low_fea_dim, self._high_dim], name='capsule/S' - ) + bilinear_matrix = tf.get_variable(dtype=tf.float32, shape=[low_fea_dim, self._high_dim], name='capsule/S') # map sequence feature to high dimensional space seq_feas_high = tf.tensordot(seq_feas, bilinear_matrix, axes=1) seq_feas_high_stop = tf.stop_gradient(seq_feas_high) @@ -115,13 +112,8 @@ def __call__(self, seq_feas, seq_lens): logging.info('will use constant number of capsules: %d' % self._max_k) num_high_capsules = tf.zeros_like(seq_lens, dtype=tf.int32) + self._max_k else: - logging.info( - 'will use log(seq_len) number of capsules, max_capsules: %d' % - self._max_k - ) - num_high_capsules = tf.maximum( - 1, tf.minimum(self._max_k, tf.to_int32(tf.log(tf.to_float(seq_lens)))) - ) + logging.info('will use log(seq_len) number of capsules, max_capsules: %d' % self._max_k) + num_high_capsules = tf.maximum(1, tf.minimum(self._max_k, tf.to_int32(tf.log(tf.to_float(seq_lens))))) # batch_size x max_seq_len(bs) mask = tf.sequence_mask(seq_lens, self._max_seq_len) @@ -150,25 +142,21 @@ def __call__(self, seq_feas, seq_lens): high_capsules = tf.einsum( 'bse, bsh->bhe', seq_feas_high_stop if iter_id + 1 < self._num_iters else seq_feas_high, - routing_logits + routing_logits, ) if iter_id + 1 == self._num_iters: - capsule_simi = self._build_capsule_simi( - high_capsules, num_high_capsules - ) + capsule_simi = self._build_capsule_simi(high_capsules, num_high_capsules) tf.summary.scalar('caspule/simi_%d' % iter_id, capsule_simi) tf.summary.scalar( 'capsule/before_squash', - tf.reduce_mean(tf.norm(high_capsules, axis=-1)) + tf.reduce_mean(tf.norm(high_capsules, axis=-1)), ) high_capsules = self.squash(high_capsules) tf.summary.scalar( 'capsule/after_squash', - tf.reduce_mean(tf.norm(high_capsules, axis=-1)) - ) - capsule_simi_final = self._build_capsule_simi( - high_capsules, num_high_capsules + tf.reduce_mean(tf.norm(high_capsules, axis=-1)), ) + capsule_simi_final = self._build_capsule_simi(high_capsules, num_high_capsules) tf.summary.scalar('caspule/simi_final', capsule_simi_final) break @@ -179,16 +167,10 @@ def __call__(self, seq_feas, seq_lens): # batch_size x max_seq_len x max_k(bse, bhe->bsh) if self._routing_logits_scale > 0: if iter_id == 0: - logging.info( - 'routing_logits_scale = %.2f' % self._routing_logits_scale - ) - routing_logits = tf.einsum( - 'bse, bhe->bsh', seq_feas_high_norm, high_capsules - ) * self._routing_logits_scale + logging.info('routing_logits_scale = %.2f' % self._routing_logits_scale) + routing_logits = tf.einsum('bse, bhe->bsh', seq_feas_high_norm, high_capsules) * self._routing_logits_scale else: - routing_logits = tf.einsum( - 'bse, bhe->bsh', seq_feas_high_stop, high_capsules - ) + routing_logits = tf.einsum('bse, bhe->bsh', seq_feas_high_stop, high_capsules) # zero paddings high_capsule_mask = tf.sequence_mask(num_high_capsules, self._max_k) diff --git a/easy_rec/python/layers/cmbf.py b/easy_rec/python/layers/cmbf.py index 96a7fcc7e..4ba8f886a 100644 --- a/easy_rec/python/layers/cmbf.py +++ b/easy_rec/python/layers/cmbf.py @@ -17,9 +17,7 @@ class CMBF(object): https://www.mdpi.com/1424-8220/21/16/5275 """ - def __init__( - self, model_config, feature_configs, features, cmbf_config, input_layer - ): + def __init__(self, model_config, feature_configs, features, cmbf_config, input_layer): self._model_config = cmbf_config has_feature = False @@ -33,9 +31,7 @@ def __init__( has_feature = True self._txt_seq_features = None if input_layer.has_group('text'): - self._txt_seq_features, _, _ = input_layer( - features, 'text', is_combine=False - ) + self._txt_seq_features, _, _ = input_layer(features, 'text', is_combine=False) has_feature = True self._other_features = None if input_layer.has_group('other'): # e.g. statistical feature @@ -51,21 +47,19 @@ def __init__( if fea_group.group_name == 'general': self._general_feature_num = len(fea_group.feature_names) general_feature_names = set(fea_group.feature_names) - assert self._general_feature_num == len(general_feature_names), ( - 'there are duplicate features in `general` feature group' - ) + assert self._general_feature_num == len( + general_feature_names + ), 'there are duplicate features in `general` feature group' elif fea_group.group_name == 'image': self._img_feature_num = len(fea_group.feature_names) img_feature_names = set(fea_group.feature_names) - assert self._img_feature_num == len(img_feature_names), ( - 'there are duplicate features in `image` feature group' - ) + assert self._img_feature_num == len(img_feature_names), 'there are duplicate features in `image` feature group' elif fea_group.group_name == 'text': txt_seq_feature_names = set(fea_group.feature_names) self._txt_feature_num = len(fea_group.feature_names) - assert self._txt_feature_num == len(txt_seq_feature_names), ( - 'there are duplicate features in `text` feature group' - ) + assert self._txt_feature_num == len( + txt_seq_feature_names + ), 'there are duplicate features in `text` feature group' max_seq_len = 0 txt_fea_emb_dim_list = [] @@ -83,30 +77,23 @@ def __init__( txt_fea_emb_dim_list.append(feature_config.embedding_dim) if feature_config.HasField('max_seq_len'): assert feature_config.max_seq_len > 0, ( - 'feature config `max_seq_len` must be greater than 0 for feature: ' - + fea_name + 'feature config `max_seq_len` must be greater than 0 for feature: ' + fea_name ) if feature_config.max_seq_len > max_seq_len: max_seq_len = feature_config.max_seq_len unique_dim_num = len(set(txt_fea_emb_dim_list)) - assert unique_dim_num <= 1 and len( - txt_fea_emb_dim_list - ) == self._txt_feature_num, ( - 'CMBF requires that all `text` feature dimensions must be consistent.' - ) + assert ( + unique_dim_num <= 1 and len(txt_fea_emb_dim_list) == self._txt_feature_num + ), 'CMBF requires that all `text` feature dimensions must be consistent.' unique_dim_num = len(set(general_emb_dim_list)) - assert unique_dim_num <= 1 and len( - general_emb_dim_list - ) == self._general_feature_num, ( - 'CMBF requires that all `general` feature dimensions must be consistent.' - ) + assert ( + unique_dim_num <= 1 and len(general_emb_dim_list) == self._general_feature_num + ), 'CMBF requires that all `general` feature dimensions must be consistent.' unique_dim_num = len(set(img_fea_emb_dim_list)) - assert unique_dim_num <= 1 and len( - img_fea_emb_dim_list - ) == self._img_feature_num, ( - 'CMBF requires that all `image` feature dimensions must be consistent.' - ) + assert ( + unique_dim_num <= 1 and len(img_fea_emb_dim_list) == self._img_feature_num + ), 'CMBF requires that all `image` feature dimensions must be consistent.' if cmbf_config.use_position_embeddings: assert cmbf_config.max_position_embeddings > 0, ( @@ -114,14 +101,14 @@ def __init__( 'It must be set when `use_position_embeddings` is true (default)' ) assert cmbf_config.max_position_embeddings >= max_seq_len, ( - 'model config `max_position_embeddings` must be greater than or equal to the maximum of all feature config ' - '`max_seq_len`, which is %d' % max_seq_len - ) + 'model config `max_position_embeddings` must be greater than' + ' or equal to the maximum of all feature config ' + '`max_seq_len`, which is %d' + ) % max_seq_len self._img_emb_size = img_fea_emb_dim_list[0] if img_fea_emb_dim_list else 0 self._txt_emb_size = txt_fea_emb_dim_list[0] if txt_fea_emb_dim_list else 0 - self._general_emb_size = general_emb_dim_list[ - 0] if general_emb_dim_list else 0 + self._general_emb_size = general_emb_dim_list[0] if general_emb_dim_list else 0 self._head_num = cmbf_config.multi_head_num self._img_head_num = cmbf_config.image_multi_head_num self._txt_head_num = cmbf_config.text_multi_head_num @@ -132,17 +119,13 @@ def __init__( self._txt_self_attention_layer_num = cmbf_config.text_self_attention_layer_num self._cross_modal_layer_num = cmbf_config.cross_modal_layer_num print( - 'txt_feature_num: {0}, img_feature_num: {1}, txt_seq_feature_num: {2}'. - format( - self._general_feature_num, self._img_feature_num, - len(self._txt_seq_features) if self._txt_seq_features else 0 - ) - ) - print( - 'txt_embedding_size: {0}, img_embedding_size: {1}'.format( - self._txt_emb_size, self._img_emb_size + 'txt_feature_num: {0}, img_feature_num: {1}, txt_seq_feature_num: {2}'.format( + self._general_feature_num, + self._img_feature_num, + len(self._txt_seq_features) if self._txt_seq_features else 0, ) ) + print('txt_embedding_size: {0}, img_embedding_size: {1}'.format(self._txt_emb_size, self._img_emb_size)) if self._img_features is not None: assert self._img_emb_size > 0, '`image` feature dimensions must be greater than 0, set by `raw_input_dim`' @@ -164,62 +147,38 @@ def image_self_attention_tower(self): hidden_size = self._model_config.multi_head_num * self._model_config.image_cross_head_size if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` - image_features = tf.reshape( - self._img_features, shape=[-1, self._img_emb_size] - ) - image_features = tf.layers.dense( - image_features, hidden_size, name='img_projection' - ) - image_features = tf.reshape( - image_features, shape=[-1, img_fea_num, hidden_size] - ) + image_features = tf.reshape(self._img_features, shape=[-1, self._img_emb_size]) + image_features = tf.layers.dense(image_features, hidden_size, name='img_projection') + image_features = tf.reshape(image_features, shape=[-1, img_fea_num, hidden_size]) return image_features hidden_size = self._img_head_size * self._img_head_num if img_fea_num > 1: # in case of video frames or ROIs (Region Of Interest) if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` - image_features = tf.reshape( - self._img_features, shape=[-1, self._img_emb_size] - ) - image_features = tf.layers.dense( - image_features, hidden_size, name='img_projection' - ) - image_features = tf.reshape( - image_features, shape=[-1, img_fea_num, hidden_size] - ) + image_features = tf.reshape(self._img_features, shape=[-1, self._img_emb_size]) + image_features = tf.layers.dense(image_features, hidden_size, name='img_projection') + image_features = tf.reshape(image_features, shape=[-1, img_fea_num, hidden_size]) elif img_fea_num == 1: if self._img_patch_num > 1: # image feature dimension: patch_num * emb_size img_fea_num = self._img_patch_num img_emb_size = self._img_emb_size // self._img_patch_num - assert img_emb_size * self._img_patch_num == self._img_emb_size, ( - 'image feature dimension must equal to `image_feature_slice_num * embedding_size_per_region`' - ) + assert ( + img_emb_size * self._img_patch_num == self._img_emb_size + ), 'image feature dimension must equal to `image_feature_slice_num * embedding_size_per_region`' self._img_emb_size = img_emb_size if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` - image_features = tf.reshape( - self._img_features, shape=[-1, self._img_emb_size] - ) - image_features = tf.layers.dense( - image_features, hidden_size, name='img_projection' - ) - image_features = tf.reshape( - image_features, shape=[-1, img_fea_num, hidden_size] - ) + image_features = tf.reshape(self._img_features, shape=[-1, self._img_emb_size]) + image_features = tf.layers.dense(image_features, hidden_size, name='img_projection') + image_features = tf.reshape(image_features, shape=[-1, img_fea_num, hidden_size]) else: img_fea_num = self._model_config.image_feature_dim if img_fea_num != self._img_emb_size: - image_features = tf.layers.dense( - image_features, img_fea_num, name='img_projection' - ) + image_features = tf.layers.dense(image_features, img_fea_num, name='img_projection') # convert each element of image feature to a feature vector - img_mapping_matrix = tf.get_variable( - 'img_map_matrix', [1, img_fea_num, hidden_size], dtype=tf.float32 - ) - image_features = tf.expand_dims( - image_features, -1 - ) * img_mapping_matrix + img_mapping_matrix = tf.get_variable('img_map_matrix', [1, img_fea_num, hidden_size], dtype=tf.float32) + image_features = tf.expand_dims(image_features, -1) * img_mapping_matrix img_attention_fea = multihead_cross_attention.transformer_encoder( image_features, @@ -228,9 +187,8 @@ def image_self_attention_tower(self): num_attention_heads=self._head_num, intermediate_size=hidden_size * 4, hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config. - attention_probs_dropout_prob, - name='image_self_attention' + attention_probs_dropout_prob=self._model_config.attention_probs_dropout_prob, + name='image_self_attention', ) # shape: [batch_size, image_seq_num/image_feature_dim, hidden_size] # print('img_attention_fea:', img_attention_fea.shape) return img_attention_fea @@ -245,22 +203,13 @@ def text_self_attention_tower(self): general_features = self._general_features if self._general_emb_size != hidden_size: # Run a linear projection of `hidden_size` - general_features = tf.reshape( - general_features, shape=[-1, self._general_emb_size] - ) - general_features = tf.layers.dense( - general_features, hidden_size, name='txt_projection' - ) - txt_features = tf.reshape( - general_features, shape=[-1, self._general_feature_num, hidden_size] - ) + general_features = tf.reshape(general_features, shape=[-1, self._general_emb_size]) + general_features = tf.layers.dense(general_features, hidden_size, name='txt_projection') + txt_features = tf.reshape(general_features, shape=[-1, self._general_feature_num, hidden_size]) all_txt_features.append(txt_features) batch_size = tf.shape(txt_features)[0] - mask = tf.ones( - shape=tf.stack([batch_size, self._general_feature_num]), - dtype=tf.int32 - ) + mask = tf.ones(shape=tf.stack([batch_size, self._general_feature_num]), dtype=tf.int32) input_masks.append(mask) input_mask = None @@ -277,28 +226,25 @@ def dynamic_mask(x, max_len): batch_size, max_seq_len, emb_size = get_shape_list(seq_fea, 3) if emb_size != hidden_size: seq_fea = tf.reshape(seq_fea, shape=[-1, emb_size]) - seq_fea = tf.layers.dense( - seq_fea, hidden_size, name='txt_seq_projection_%d' % i - ) + seq_fea = tf.layers.dense(seq_fea, hidden_size, name='txt_seq_projection_%d' % i) seq_fea = tf.reshape(seq_fea, shape=[-1, max_seq_len, hidden_size]) seq_fea = multihead_cross_attention.embedding_postprocessor( seq_fea, use_token_type=self._model_config.use_token_type, - token_type_ids=tf. - ones(shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * i, + token_type_ids=tf.ones(shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * i, token_type_vocab_size=token_type_vocab_size, reuse_token_type=tf.AUTO_REUSE, use_position_embeddings=self._model_config.use_position_embeddings, max_position_embeddings=self._model_config.max_position_embeddings, position_embedding_name='position_embeddings_%d' % i, - dropout_prob=self._model_config.text_seq_emb_dropout_prob + dropout_prob=self._model_config.text_seq_emb_dropout_prob, ) all_txt_features.append(seq_fea) input_mask = tf.map_fn( fn=lambda t: dynamic_mask(t, max_seq_len), - elems=tf.to_int32(seq_len) + elems=tf.to_int32(seq_len), ) input_masks.append(input_mask) @@ -319,9 +265,8 @@ def dynamic_mask(x, max_len): attention_mask=attention_mask, intermediate_size=hidden_size * 4, hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config. - attention_probs_dropout_prob, - name='text_self_attention' + attention_probs_dropout_prob=self._model_config.attention_probs_dropout_prob, + name='text_self_attention', ) # shape: [batch_size, txt_seq_length, hidden_size] print('txt_attention_fea:', txt_attention_fea.shape) return txt_attention_fea, input_mask, input_masks @@ -334,26 +279,21 @@ def merge_text_embedding(self, txt_embeddings, input_masks): text_seq_emb = [] if self._general_feature_num > 0: text_emb = tf.slice( - txt_embeddings, [0, 0, 0], - [shape[0], self._general_feature_num, shape[2]] + txt_embeddings, + [0, 0, 0], + [shape[0], self._general_feature_num, shape[2]], ) text_seq_emb.append(text_emb) begin = self._general_feature_num for i in range(len(text_seq_emb), len(input_masks)): size = tf.shape(input_masks[i])[1] - temp_emb = tf.slice( - txt_embeddings, [0, begin, 0], [shape[0], size, shape[2]] - ) + temp_emb = tf.slice(txt_embeddings, [0, begin, 0], [shape[0], size, shape[2]]) mask = tf.expand_dims(tf.to_float(input_masks[i]), -1) temp_emb = temp_emb * mask # avg pooling - emb_sum = tf.reduce_sum( - temp_emb, axis=1, keepdims=True - ) # shape: [batch_size, 1, hidden_size] - count = tf.reduce_sum( - mask, axis=1, keepdims=True - ) # shape: [batch_size, 1, 1] + emb_sum = tf.reduce_sum(temp_emb, axis=1, keepdims=True) # shape: [batch_size, 1, hidden_size] + count = tf.reduce_sum(mask, axis=1, keepdims=True) # shape: [batch_size, 1, 1] seq_emb = emb_sum / count # shape: [batch_size, 1, hidden_size] text_seq_emb.append(seq_emb) @@ -376,30 +316,29 @@ def __call__(self, is_training, *args, **kwargs): img_attention_fea = self.image_self_attention_tower() # shape: [batch_size, txt_seq_length, hidden_size] - txt_attention_fea, input_mask, input_masks = self.text_self_attention_tower( - ) + txt_attention_fea, input_mask, input_masks = self.text_self_attention_tower() all_fea = [] if None not in [img_attention_fea, txt_attention_fea]: - img_embeddings, txt_embeddings = multihead_cross_attention.cross_attention_tower( + ( + img_embeddings, + txt_embeddings, + ) = multihead_cross_attention.cross_attention_tower( img_attention_fea, txt_attention_fea, num_hidden_layers=self._cross_modal_layer_num, num_attention_heads=self._head_num, right_input_mask=input_mask, left_size_per_head=self._model_config.image_cross_head_size, - left_intermediate_size=4 * self._model_config.image_cross_head_size * - self._head_num, + left_intermediate_size=4 * self._model_config.image_cross_head_size * self._head_num, right_size_per_head=self._model_config.text_cross_head_size, - right_intermediate_size=4 * self._model_config.text_cross_head_size * - self._head_num, + right_intermediate_size=4 * self._model_config.text_cross_head_size * self._head_num, hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config. - attention_probs_dropout_prob + attention_probs_dropout_prob=self._model_config.attention_probs_dropout_prob, ) - # img_embeddings shape: [batch_size, image_(region_)num/image_feature_dim, multi_head_num * image_cross_head_size] + # img_emb shape: [batch_size, image_(region_)num/image_feature_dim, multi_head_num * image_cross_head_size] print('img_embeddings:', img_embeddings.shape) - # txt_embeddings shape: [batch_size, general_feature_num + max_txt_seq_len, multi_head_num * text_cross_head_size] + # txt_emb shape: [batch_size, general_feature_num + max_txt_seq_len, multi_head_num * text_cross_head_size] print('txt_embeddings:', txt_embeddings.shape) # shape: [batch_size, multi_head_num * image_cross_head_size] @@ -416,17 +355,17 @@ def __call__(self, is_training, *args, **kwargs): elif txt_attention_fea is not None: # only has text tower # shape: [batch_size, (general_feature_num + txt_seq_num) * multi_head_num * text_head_size] - txt_embeddings = self.merge_text_embedding( - txt_attention_fea, input_masks - ) + txt_embeddings = self.merge_text_embedding(txt_attention_fea, input_masks) all_fea = [txt_embeddings] if self._other_features is not None: if self._model_config.HasField('other_feature_dnn'): l2_reg = kwargs['l2_reg'] if 'l2_reg' in kwargs else 0 other_dnn_layer = dnn.DNN( - self._model_config.other_feature_dnn, l2_reg, 'other_dnn', - is_training + self._model_config.other_feature_dnn, + l2_reg, + 'other_dnn', + is_training, ) other_fea = other_dnn_layer(self._other_features) all_fea.append(other_fea) # e.g. statistical features diff --git a/easy_rec/python/layers/common_layers.py b/easy_rec/python/layers/common_layers.py index 7a25916bf..1138298da 100644 --- a/easy_rec/python/layers/common_layers.py +++ b/easy_rec/python/layers/common_layers.py @@ -19,7 +19,7 @@ def highway( scope='highway', dropout=0.0, init_gate_bias=-1.0, - reuse=None + reuse=None, ): if isinstance(activation, six.string_types): activation = get_activation(activation) @@ -37,11 +37,9 @@ def highway( activation=tf.sigmoid, bias_initializer=initializer, name='gate_%d' % i, - reuse=reuse - ) - H = tf.layers.dense( - x, size, activation=activation, name='activation_%d' % i, reuse=reuse + reuse=reuse, ) + H = tf.layers.dense(x, size, activation=activation, name='activation_%d' % i, reuse=reuse) if dropout > 0.0: H = tf.nn.dropout(H, 1.0 - dropout) x = H * T + x * (1.0 - T) @@ -53,7 +51,7 @@ def text_cnn( filter_sizes=(3, 4, 5), num_filters=(128, 64, 64), scope_name='textcnn', - reuse=False + reuse=False, ): # x: None * step_dim * embed_dim assert len(filter_sizes) == len(num_filters) @@ -73,15 +71,11 @@ def text_cnn( name='conv_layer', reuse=reuse, kernel_initializer=initializer, - padding='same' + padding='same', ) - pool = tf.reduce_max( - conv, axis=1 - ) # max pooling, shape: (batch_size, num_filters) + pool = tf.reduce_max(conv, axis=1) # max pooling, shape: (batch_size, num_filters) pooled_outputs.append(pool) - pool_flat = tf.concat( - pooled_outputs, 1 - ) # shape: (batch_size, num_filters * len(filter_sizes)) + pool_flat = tf.concat(pooled_outputs, 1) # shape: (batch_size, num_filters * len(filter_sizes)) return pool_flat @@ -92,7 +86,7 @@ def layer_norm(input_tensor, name=None, reuse=None): begin_norm_axis=-1, begin_params_axis=-1, reuse=reuse, - scope=name + scope=name, ) @@ -115,18 +109,14 @@ def __call__(self, config, is_training, **kwargs): return self.inputs if config.do_batch_norm and config.do_layer_norm: - raise ValueError( - 'can not do batch norm and layer norm for input layer at the same time' - ) + raise ValueError('can not do batch norm and layer norm for input layer at the same time') with tf.name_scope(self.name): return self.call(config, is_training) def build(self, config, training): self.built = True combine = not config.output_seq_and_normal_feature - self.inputs = self._input_layer( - self._feature_dict, self._group_name, is_combine=combine - ) + self.inputs = self._input_layer(self._feature_dict, self._group_name, is_combine=combine) if config.output_seq_and_normal_feature: seq_feature_and_len, _, target_features = self.inputs seq_len = seq_feature_and_len[0][1] @@ -136,9 +126,7 @@ def build(self, config, training): target_features = tf.concat(target_features, axis=-1) else: target_features = None - assert len( - seq_features - ) > 0, '[%s] sequence feature is empty' % self.name + assert len(seq_features) > 0, '[%s] sequence feature is empty' % self.name seq_features = tf.concat(seq_features, axis=-1) self.inputs = seq_features, seq_len, target_features self.reset(config, training) @@ -162,13 +150,9 @@ def call(self, config, training): keep_prob = 1.0 - config.feature_dropout_rate mask = self.bern.sample(num_features) elif do_bn: - features = tf.layers.batch_normalization( - features, training=training, reuse=self._reuse - ) + features = tf.layers.batch_normalization(features, training=training, reuse=self._reuse) elif do_ln: - features = layer_norm( - features, name=self._group_name + '_features', reuse=self._reuse - ) + features = layer_norm(features, name=self._group_name + '_features', reuse=self._reuse) output_feature_list = config.output_2d_tensor_and_feature_list output_feature_list = output_feature_list or config.only_output_feature_list @@ -179,9 +163,7 @@ def call(self, config, training): for i in range(num_features): fea = feature_list[i] if do_bn: - fea = tf.layers.batch_normalization( - fea, training=training, reuse=self._reuse - ) + fea = tf.layers.batch_normalization(fea, training=training, reuse=self._reuse) elif do_ln: ln_name = self._group_name + 'f_%d' % i fea = layer_norm(fea, name=ln_name, reuse=self._reuse) diff --git a/easy_rec/python/layers/dnn.py b/easy_rec/python/layers/dnn.py index 1d33e0628..01555822e 100644 --- a/easy_rec/python/layers/dnn.py +++ b/easy_rec/python/layers/dnn.py @@ -11,7 +11,6 @@ class DNN: - def __init__( self, dnn_config, @@ -19,7 +18,7 @@ def __init__( name='dnn', is_training=False, last_layer_no_activation=False, - last_layer_no_batch_norm=False + last_layer_no_batch_norm=False, ): """Initializes a `DNN` Layer. @@ -36,9 +35,7 @@ def __init__( self._name = name self._is_training = is_training logging.info('dnn activation function = %s' % self._config.activation) - self.activation = get_activation( - self._config.activation, training=is_training - ) + self.activation = get_activation(self._config.activation, training=is_training) self._last_layer_no_activation = last_layer_no_activation self._last_layer_no_batch_norm = last_layer_no_batch_norm @@ -62,33 +59,28 @@ def __call__(self, deep_fea, hidden_layer_feature_output=False): units=unit, kernel_regularizer=self._l2_reg, activation=None, - name='%s/dnn_%d' % (self._name, i) + name='%s/dnn_%d' % (self._name, i), ) - if self._config.use_bn and ( - (i + 1 < hidden_units_len) or not self._last_layer_no_batch_norm - ): + if self._config.use_bn and ((i + 1 < hidden_units_len) or not self._last_layer_no_batch_norm): deep_fea = tf.layers.batch_normalization( deep_fea, training=self._is_training, trainable=True, - name='%s/dnn_%d/bn' % (self._name, i) + name='%s/dnn_%d/bn' % (self._name, i), ) if (i + 1 < hidden_units_len) or not self._last_layer_no_activation: - deep_fea = self.activation( - deep_fea, name='%s/dnn_%d/act' % (self._name, i) - ) + deep_fea = self.activation(deep_fea, name='%s/dnn_%d/act' % (self._name, i)) if len(self.dropout_ratio) > 0 and self._is_training: - assert self.dropout_ratio[ - i] < 1, 'invalid dropout_ratio: %.3f' % self.dropout_ratio[i] + assert self.dropout_ratio[i] < 1, 'invalid dropout_ratio: %.3f' % self.dropout_ratio[i] deep_fea = tf.nn.dropout( deep_fea, keep_prob=1 - self.dropout_ratio[i], - name='%s/%d/dropout' % (self._name, i) + name='%s/%d/dropout' % (self._name, i), ) if hidden_layer_feature_output: hidden_feature_dict['hidden_layer' + str(i)] = deep_fea - if (i + 1 == hidden_units_len): + if i + 1 == hidden_units_len: hidden_feature_dict['hidden_layer_end'] = deep_fea return hidden_feature_dict else: diff --git a/easy_rec/python/layers/embed_input_layer.py b/easy_rec/python/layers/embed_input_layer.py index 77b7c97b8..8c325f687 100644 --- a/easy_rec/python/layers/embed_input_layer.py +++ b/easy_rec/python/layers/embed_input_layer.py @@ -7,17 +7,14 @@ class EmbedInputLayer(object): - def __init__(self, feature_groups_config, dump_dir=None): - self._feature_groups = { - x.group_name: FeatureGroup(x) - for x in feature_groups_config - } + self._feature_groups = {x.group_name: FeatureGroup(x) for x in feature_groups_config} self._dump_dir = dump_dir def __call__(self, features, group_name): - assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % \ - ','.join([x for x in self._feature_groups]) + assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ','.join( + [x for x in self._feature_groups] + ) feature_group = self._feature_groups[group_name] group_features = [] for feature_name in feature_group.feature_names: diff --git a/easy_rec/python/layers/fm.py b/easy_rec/python/layers/fm.py index 1929e00aa..342204c9a 100644 --- a/easy_rec/python/layers/fm.py +++ b/easy_rec/python/layers/fm.py @@ -8,7 +8,6 @@ class FM: - def __init__(self, name='fm'): """Initializes a `FM` Layer. diff --git a/easy_rec/python/layers/input_layer.py b/easy_rec/python/layers/input_layer.py index 092ce19c5..e1f145fdd 100644 --- a/easy_rec/python/layers/input_layer.py +++ b/easy_rec/python/layers/input_layer.py @@ -10,16 +10,20 @@ from easy_rec.python.compat import regularizers from easy_rec.python.compat.feature_column import feature_column +from easy_rec.python.compat.feature_column.feature_column_v2 import ( # NOQA + is_embedding_column, +) from easy_rec.python.feature_column.feature_column import FeatureColumnParser from easy_rec.python.feature_column.feature_group import FeatureGroup +from easy_rec.python.layers import ( # NOQA + sequence_feature_layer, + variational_dropout_layer, +) from easy_rec.python.layers.keras import TextCNN from easy_rec.python.layers.utils import Parameter from easy_rec.python.protos.feature_config_pb2 import WideOrDeep from easy_rec.python.utils import conditional, shape_utils -from easy_rec.python.compat.feature_column.feature_column_v2 import is_embedding_column # NOQA -from easy_rec.python.layers import sequence_feature_layer, variational_dropout_layer # NOQA - class InputLayer(object): """Input Layer for generate input features. @@ -37,31 +41,27 @@ def __init__( embedding_regularizer=None, kernel_regularizer=None, is_training=False, - is_predicting=False + is_predicting=False, ): - self._feature_groups = { - x.group_name: FeatureGroup(x) - for x in feature_groups_config - } + self._feature_groups = {x.group_name: FeatureGroup(x) for x in feature_groups_config} self.sequence_feature_layer = sequence_feature_layer.SequenceFeatureLayer( - feature_configs, feature_groups_config, ev_params, embedding_regularizer, - kernel_regularizer, is_training, is_predicting + feature_configs, + feature_groups_config, + ev_params, + embedding_regularizer, + kernel_regularizer, + is_training, + is_predicting, ) self._seq_feature_groups_config = [] for x in feature_groups_config: for y in x.sequence_features: self._seq_feature_groups_config.append(y) self._group_name_to_seq_features = { - x.group_name: x.sequence_features - for x in feature_groups_config if len(x.sequence_features) > 0 + x.group_name: x.sequence_features for x in feature_groups_config if len(x.sequence_features) > 0 } wide_and_deep_dict = self.get_wide_deep_dict() - self._fc_parser = FeatureColumnParser( - feature_configs, - wide_and_deep_dict, - wide_output_dim, - ev_params=ev_params - ) + self._fc_parser = FeatureColumnParser(feature_configs, wide_and_deep_dict, wide_output_dim, ev_params=ev_params) self._embedding_regularizer = embedding_regularizer self._kernel_regularizer = kernel_regularizer @@ -86,14 +86,11 @@ def get_combined_feature(self, features, group_name, is_dict=False): feature_name_to_output_tensors: dict, feature_name to feature_value, only present when is_dict is True """ feature_name_to_output_tensors = {} - negative_sampler = self._feature_groups[group_name - ]._config.negative_sampler + negative_sampler = self._feature_groups[group_name]._config.negative_sampler place_on_cpu = os.getenv('place_embedding_on_cpu') place_on_cpu = eval(place_on_cpu) if place_on_cpu else False - with conditional( - self._is_predicting and place_on_cpu, ops.device('/CPU:0') - ): + with conditional(self._is_predicting and place_on_cpu, ops.device('/CPU:0')): concat_features, group_features = self.single_call_input_layer( features, group_name, feature_name_to_output_tensors ) @@ -106,15 +103,13 @@ def get_combined_feature(self, features, group_name, is_dict=False): group_seq_arr, feature_name_to_output_tensors, negative_sampler=negative_sampler, - scope_name=group_name + scope_name=group_name, ) group_features.extend(all_seq_fea) for col, fea in zip(group_seq_arr, all_seq_fea): feature_name_to_output_tensors['seq_fea/' + col.group_name] = fea all_seq_fea = array_ops.concat(all_seq_fea, axis=-1) - concat_features = array_ops.concat( - [concat_features, all_seq_fea], axis=-1 - ) + concat_features = array_ops.concat([concat_features, all_seq_fea], axis=-1) if is_dict: return concat_features, group_features, feature_name_to_output_tensors else: @@ -132,7 +127,8 @@ def get_plain_feature(self, features, group_name): group_features: list of features """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups]) + group_name, + ','.join([x for x in self._feature_groups]), ) feature_group = self._feature_groups[group_name] @@ -145,7 +141,7 @@ def get_plain_feature(self, features, group_name): features, group_columns, cols_to_output_tensors=cols_to_output_tensors, - is_training=self._is_training + is_training=self._is_training, ) group_features = [cols_to_output_tensors[x] for x in group_columns] @@ -155,9 +151,7 @@ def get_plain_feature(self, features, group_name): embedding_reg_lst.append(val) if self._embedding_regularizer is not None and len(embedding_reg_lst) > 0: - regularizers.apply_regularization( - self._embedding_regularizer, weights_list=embedding_reg_lst - ) + regularizers.apply_regularization(self._embedding_regularizer, weights_list=embedding_reg_lst) return output_features, group_features def get_sequence_feature(self, features, group_name): @@ -173,13 +167,12 @@ def get_sequence_feature(self, features, group_name): 1d sequence length tensor. """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups]) + group_name, + ','.join([x for x in self._feature_groups]), ) if self._variational_dropout_config is not None: - raise ValueError( - 'variational dropout is not supported in not combined mode now.' - ) + raise ValueError('variational dropout is not supported in not combined mode now.') feature_group = self._feature_groups[group_name] _, group_seq_columns = feature_group.select_columns(self._fc_parser) @@ -188,21 +181,15 @@ def get_sequence_feature(self, features, group_name): builder = feature_column._LazyBuilder(features) seq_features = [] for fc in group_seq_columns: - with variable_scope.variable_scope( - 'input_layer/' + fc.categorical_column.name - ): + with variable_scope.variable_scope('input_layer/' + fc.categorical_column.name): tmp_embedding, tmp_seq_len = fc._get_sequence_dense_tensor(builder) if fc.max_seq_length > 0: - tmp_embedding, tmp_seq_len = shape_utils.truncate_sequence( - tmp_embedding, tmp_seq_len, fc.max_seq_length - ) + tmp_embedding, tmp_seq_len = shape_utils.truncate_sequence(tmp_embedding, tmp_seq_len, fc.max_seq_length) seq_features.append((tmp_embedding, tmp_seq_len)) embedding_reg_lst.append(tmp_embedding) if self._embedding_regularizer is not None and len(embedding_reg_lst) > 0: - regularizers.apply_regularization( - self._embedding_regularizer, weights_list=embedding_reg_lst - ) + regularizers.apply_regularization(self._embedding_regularizer, weights_list=embedding_reg_lst) return seq_features def get_raw_features(self, features, group_name): @@ -216,7 +203,8 @@ def get_raw_features(self, features, group_name): features: all raw features in list """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups]) + group_name, + ','.join([x for x in self._feature_groups]), ) feature_group = self._feature_groups[group_name] return [features[x] for x in feature_group.feature_names] @@ -232,7 +220,8 @@ def get_bucketized_features(self, features, group_name): features: all raw features in list, added feature offset """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups]) + group_name, + ','.join([x for x in self._feature_groups]), ) feature_group = self._feature_groups[group_name] offset = 0 @@ -279,7 +268,8 @@ def __call__(self, features, group_name, is_combine=True, is_dict=False): 1 dimension sequence length tensor. """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups]) + group_name, + ','.join([x for x in self._feature_groups]), ) if is_combine: return self.get_combined_feature(features, group_name, is_dict) @@ -287,18 +277,12 @@ def __call__(self, features, group_name, is_combine=True, is_dict=False): # return sequence feature in raw format instead of combine them place_on_cpu = os.getenv('place_embedding_on_cpu') place_on_cpu = eval(place_on_cpu) if place_on_cpu else False - with conditional( - self._is_predicting and place_on_cpu, ops.device('/CPU:0') - ): + with conditional(self._is_predicting and place_on_cpu, ops.device('/CPU:0')): seq_features = self.get_sequence_feature(features, group_name) - plain_features, feature_list = self.get_plain_feature( - features, group_name - ) + plain_features, feature_list = self.get_plain_feature(features, group_name) return seq_features, plain_features, feature_list - def single_call_input_layer( - self, features, group_name, feature_name_to_output_tensors=None - ): + def single_call_input_layer(self, features, group_name, feature_name_to_output_tensors=None): """Get features by group_name. Args: @@ -312,36 +296,31 @@ def single_call_input_layer( group_features: list of features """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups]) + group_name, + ','.join([x for x in self._feature_groups]), ) feature_group = self._feature_groups[group_name] - group_columns, group_seq_columns = feature_group.select_columns( - self._fc_parser - ) + group_columns, group_seq_columns = feature_group.select_columns(self._fc_parser) cols_to_output_tensors = OrderedDict() output_features = feature_column.input_layer( features, group_columns, cols_to_output_tensors=cols_to_output_tensors, feature_name_to_output_tensors=feature_name_to_output_tensors, - is_training=self._is_training + is_training=self._is_training, ) embedding_reg_lst = [] builder = feature_column._LazyBuilder(features) seq_features = [] for column in sorted(group_seq_columns, key=lambda x: x.name): - with variable_scope.variable_scope( - None, default_name=column._var_scope_name - ): + with variable_scope.variable_scope(None, default_name=column._var_scope_name): seq_feature, seq_len = column._get_sequence_dense_tensor(builder) embedding_reg_lst.append(seq_feature) sequence_combiner = column.sequence_combiner if sequence_combiner is None: - raise ValueError( - 'sequence_combiner is none, please set sequence_combiner or use TagFeature' - ) + raise ValueError('sequence_combiner is none, please set sequence_combiner or use TagFeature') if sequence_combiner.WhichOneof('combiner') == 'attention': attn_logits = tf.layers.dense( inputs=seq_feature, @@ -349,17 +328,13 @@ def single_call_input_layer( kernel_regularizer=self._kernel_regularizer, use_bias=False, activation=None, - name='attention' + name='attention', ) attn_logits = tf.squeeze(attn_logits, axis=-1) - attn_logits_padding = tf.ones_like(attn_logits) * (-2**32 + 1) + attn_logits_padding = tf.ones_like(attn_logits) * (-(2**32) + 1) seq_mask = tf.sequence_mask(seq_len) - attn_score = tf.nn.softmax( - tf.where(seq_mask, attn_logits, attn_logits_padding) - ) - seq_feature = tf.reduce_sum( - attn_score[:, :, tf.newaxis] * seq_feature, axis=1 - ) + attn_score = tf.nn.softmax(tf.where(seq_mask, attn_logits, attn_logits_padding)) + seq_feature = tf.reduce_sum(attn_score[:, :, tf.newaxis] * seq_feature, axis=1) seq_features.append(seq_feature) cols_to_output_tensors[column] = seq_feature elif sequence_combiner.WhichOneof('combiner') == 'text_cnn': @@ -371,40 +346,28 @@ def single_call_input_layer( else: raise NotImplementedError if self._variational_dropout_config is not None: - features_dimension = OrderedDict( - [ - (k.raw_name, int(v.shape[-1])) - for k, v in cols_to_output_tensors.items() - ] - ) - concat_features = array_ops.concat( - [output_features] + seq_features, axis=-1 - ) + features_dimension = OrderedDict([(k.raw_name, int(v.shape[-1])) for k, v in cols_to_output_tensors.items()]) + concat_features = array_ops.concat([output_features] + seq_features, axis=-1) variational_dropout = variational_dropout_layer.VariationalDropoutLayer( self._variational_dropout_config, features_dimension, self._is_training, - name=group_name + name=group_name, ) concat_features = variational_dropout(concat_features) - group_features = tf.split( - concat_features, list(features_dimension.values()), axis=-1 - ) + group_features = tf.split(concat_features, list(features_dimension.values()), axis=-1) else: - concat_features = array_ops.concat( - [output_features] + seq_features, axis=-1 - ) - group_features = [cols_to_output_tensors[x] for x in group_columns] + \ - [cols_to_output_tensors[x] for x in group_seq_columns] + concat_features = array_ops.concat([output_features] + seq_features, axis=-1) + group_features = [cols_to_output_tensors[x] for x in group_columns] + [ + cols_to_output_tensors[x] for x in group_seq_columns + ] if self._embedding_regularizer is not None: for fc, val in cols_to_output_tensors.items(): if is_embedding_column(fc): embedding_reg_lst.append(val) if embedding_reg_lst: - regularizers.apply_regularization( - self._embedding_regularizer, weights_list=embedding_reg_lst - ) + regularizers.apply_regularization(self._embedding_regularizer, weights_list=embedding_reg_lst) return concat_features, group_features def get_wide_deep_dict(self): diff --git a/easy_rec/python/layers/keras/__init__.py b/easy_rec/python/layers/keras/__init__.py index ab5e50324..64ed4cad5 100644 --- a/easy_rec/python/layers/keras/__init__.py +++ b/easy_rec/python/layers/keras/__init__.py @@ -2,6 +2,13 @@ from .auxiliary_loss import AuxiliaryLoss from .blocks import MLP, Gate, Highway, TextCNN from .bst import BST +from .custom_ops import ( # NOQA + EditDistance, + MappedDotProduct, + OverlapFeature, + SeqAugmentOps, + TextNormalize, +) from .data_augment import SeqAugment from .din import DIN from .embedding import EmbeddingLayer @@ -10,8 +17,10 @@ from .mask_net import MaskBlock, MaskNet from .multi_head_attention import MultiHeadAttention from .multi_task import AITMTower, MMoE +from .numerical_embedding import ( # NOQA + AutoDisEmbedding, + NaryDisEmbedding, + PeriodicEmbedding, +) from .ppnet import PPNet from .transformer import TextEncoder, TransformerBlock, TransformerEncoder - -from .custom_ops import EditDistance, MappedDotProduct, OverlapFeature, SeqAugmentOps, TextNormalize # NOQA -from .numerical_embedding import AutoDisEmbedding, NaryDisEmbedding, PeriodicEmbedding # NOQA diff --git a/easy_rec/python/layers/keras/activation.py b/easy_rec/python/layers/keras/activation.py index 27d5ec1a8..32c9f0e37 100644 --- a/easy_rec/python/layers/keras/activation.py +++ b/easy_rec/python/layers/keras/activation.py @@ -49,14 +49,12 @@ def __init__(self, axis=-1, epsilon=1e-9, **kwargs): super(Dice, self).__init__(**kwargs) def build(self, input_shape): - self.bn = BatchNormalization( - axis=self.axis, epsilon=self.epsilon, center=False, scale=False - ) + self.bn = BatchNormalization(axis=self.axis, epsilon=self.epsilon, center=False, scale=False) self.alphas = self.add_weight( - shape=(input_shape[-1], ), + shape=(input_shape[-1],), initializer=Zeros(), dtype=tf.float32, - name='dice_alpha' + name='dice_alpha', ) # name='alpha_'+self.name super(Dice, self).build(input_shape) # Be sure to call this somewhere! self.uses_learning_phase = True @@ -75,14 +73,15 @@ def compute_output_shape(self, input_shape): def updates(self): return self.bn.updates - def get_config(self, ): + def get_config( + self, + ): config = {'axis': self.axis, 'epsilon': self.epsilon} base_config = super(Dice, self).get_config() return dict(list(base_config.items()) + list(config.items())) class MaskedSoftmax(Layer): - def __init__(self, axis=-1, **kwargs): super(MaskedSoftmax, self).__init__(**kwargs) self.axis = axis @@ -109,8 +108,5 @@ def activation_layer(activation, name=None): elif issubclass(activation, Layer): act_layer = activation(name=name) else: - raise ValueError( - 'Invalid activation,found %s.You should use a str or a Activation Layer Class.' - % (activation) - ) + raise ValueError('Invalid activation,found %s.You should use a str or a Activation Layer Class.' % (activation)) return act_layer diff --git a/easy_rec/python/layers/keras/attention.py b/easy_rec/python/layers/keras/attention.py index fb5629beb..eb44e29d1 100644 --- a/easy_rec/python/layers/keras/attention.py +++ b/easy_rec/python/layers/keras/attention.py @@ -5,6 +5,7 @@ This file follows the terminology of https://arxiv.org/abs/1706.03762 Figure 2. Attention is formed by three tensors: Query, Key and Value. """ + import tensorflow as tf from tensorflow.python.keras.layers import Layer @@ -81,9 +82,7 @@ def __init__(self, params, name='attention', reuse=None, **kwargs): self.seed = params.get_or_default('seed', None) self.scale = None self.concat_score_weight = None - self._return_attention_scores = params.get_or_default( - 'return_attention_scores', False - ) + self._return_attention_scores = params.get_or_default('return_attention_scores', False) self.use_causal_mask = params.get_or_default('use_causal_mask', False) @property @@ -108,8 +107,7 @@ def build(self, input_shape): dtype=self.dtype, trainable=True, ) - super(Attention, - self).build(input_shape) # Be sure to call this somewhere! + super(Attention, self).build(input_shape) # Be sure to call this somewhere! def _calculate_scores(self, query, key): """Calculates attention scores as a query-key dot product. @@ -135,13 +133,9 @@ def _calculate_scores(self, query, key): # Reshape into [batch_size, 1, Tv, dim]. k_reshaped = tf.expand_dims(key, axis=-3) if self.scale is not None: - scores = self.concat_score_weight * tf.reduce_sum( - tf.tanh(self.scale * (q_reshaped + k_reshaped)), axis=-1 - ) + scores = self.concat_score_weight * tf.reduce_sum(tf.tanh(self.scale * (q_reshaped + k_reshaped)), axis=-1) else: - scores = self.concat_score_weight * tf.reduce_sum( - tf.tanh(q_reshaped + k_reshaped), axis=-1 - ) + scores = self.concat_score_weight * tf.reduce_sum(tf.tanh(q_reshaped + k_reshaped), axis=-1) return scores def _apply_scores(self, scores, value, scores_mask=None, training=False): @@ -217,12 +211,8 @@ def call(self, inputs, mask=None, training=False, **kwargs): q_mask = mask[0] if mask else None v_mask = mask[1] if mask else None scores = self._calculate_scores(query=q, key=k) - scores_mask = self._calculate_score_mask( - scores, v_mask, self.use_causal_mask - ) - result, attention_scores = self._apply_scores( - scores=scores, value=v, scores_mask=scores_mask, training=training - ) + scores_mask = self._calculate_score_mask(scores, v_mask, self.use_causal_mask) + result, attention_scores = self._apply_scores(scores=scores, value=v, scores_mask=scores_mask, training=training) if q_mask is not None: # Mask of shape [batch_size, Tq, 1]. q_mask = tf.expand_dims(q_mask, axis=-1) @@ -248,9 +238,7 @@ def _validate_inputs(self, inputs, mask=None): raise ValueError( '{class_name} layer must be called on a list of inputs, ' 'namely [query, value] or [query, value, key]. ' - 'Received: inputs={inputs}.'.format( - class_name=class_name, inputs=inputs - ) + 'Received: inputs={inputs}.'.format(class_name=class_name, inputs=inputs) ) if len(inputs) < 2 or len(inputs) > 3: raise ValueError( @@ -261,15 +249,13 @@ def _validate_inputs(self, inputs, mask=None): if mask is not None: if not isinstance(mask, list): raise ValueError( - '{class_name} layer mask must be a list, ' - 'namely [query_mask, value_mask]. Received: mask={mask}.'.format( + '{class_name} layer mask must be a list, ' 'namely [query_mask, value_mask]. Received: mask={mask}.'.format( class_name=class_name, mask=mask ) ) if len(mask) < 2 or len(mask) > 3: raise ValueError( - '{class_name} layer accepts mask list of length 2 or 3. ' - 'Received: inputs={inputs}, mask={mask}.'.format( + '{class_name} layer accepts mask list of length 2 or 3. ' 'Received: inputs={inputs}, mask={mask}.'.format( class_name=class_name, inputs=inputs, mask=mask ) ) diff --git a/easy_rec/python/layers/keras/auxiliary_loss.py b/easy_rec/python/layers/keras/auxiliary_loss.py index 8371bfdff..6ad614985 100644 --- a/easy_rec/python/layers/keras/auxiliary_loss.py +++ b/easy_rec/python/layers/keras/auxiliary_loss.py @@ -15,10 +15,7 @@ def __init__(self, params, name='auxiliary_loss', reuse=None, **kwargs): params.check_required('loss_type') self.loss_type = params.get_or_default('loss_type', None) self.loss_weight = params.get_or_default('loss_weight', 1.0) - logging.info( - 'init layer `%s` with loss type: %s and weight: %f' % - (self.name, self.loss_type, self.loss_weight) - ) + logging.info('init layer `%s` with loss type: %s and weight: %f' % (self.name, self.loss_type, self.loss_weight)) self.temperature = params.get_or_default('temperature', 0.1) def call(self, inputs, training=None, **kwargs): @@ -36,9 +33,7 @@ def call(self, inputs, training=None, **kwargs): loss_dict['%s_l2_loss' % self.name] = loss_value elif self.loss_type == 'info_nce': query, positive = inputs - loss = contrastive_loss.info_nce_loss( - query, positive, temperature=self.temperature - ) + loss = contrastive_loss.info_nce_loss(query, positive, temperature=self.temperature) loss_value = loss if self.loss_weight == 1.0 else loss * self.loss_weight loss_dict['%s_info_nce_loss' % self.name] = loss_value elif self.loss_type == 'nce_loss': diff --git a/easy_rec/python/layers/keras/blocks.py b/easy_rec/python/layers/keras/blocks.py index 85cc83dc2..a89e3898d 100644 --- a/easy_rec/python/layers/keras/blocks.py +++ b/easy_rec/python/layers/keras/blocks.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Convenience blocks for building models.""" + import logging import tensorflow as tf @@ -43,9 +44,18 @@ def __init__(self, params, name='mlp', reuse=None, **kwargs): units = list(params.hidden_units) logging.info( 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' - ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' % ( - name, units, dropout_rate, activation, use_bn, use_final_bn, - final_activation, use_bias, initializer, use_bn_after_act + ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' + % ( + name, + units, + dropout_rate, + activation, + use_bn, + use_final_bn, + final_activation, + use_bias, + initializer, + use_bn_after_act, ) ) assert len(units) > 0, 'MLP(%s) takes at least one hidden units' % name @@ -58,16 +68,30 @@ def __init__(self, params, name='mlp', reuse=None, **kwargs): name = 'layer_%d' % i drop_rate = dropout_rate[i] if i < num_dropout else 0.0 self.add_rich_layer( - num_units, use_bn, drop_rate, activation, initializer, use_bias, - use_bn_after_act, name, params.l2_regularizer + num_units, + use_bn, + drop_rate, + activation, + initializer, + use_bias, + use_bn_after_act, + name, + params.l2_regularizer, ) n = len(units) - 1 drop_rate = dropout_rate[n] if num_dropout > n else 0.0 name = 'layer_%d' % n self.add_rich_layer( - units[-1], use_final_bn, drop_rate, final_activation, initializer, - use_final_bias, use_bn_after_act, name, params.l2_regularizer + units[-1], + use_final_bn, + drop_rate, + final_activation, + initializer, + use_final_bias, + use_bn_after_act, + name, + params.l2_regularizer, ) def add_rich_layer( @@ -80,7 +104,7 @@ def add_rich_layer( use_bias, use_bn_after_activation, name, - l2_reg=None + l2_reg=None, ): act_layer = activation_layer(activation, name='%s/act' % name) if use_bn and not use_bn_after_activation: @@ -89,12 +113,10 @@ def add_rich_layer( use_bias=use_bias, kernel_initializer=initializer, kernel_regularizer=l2_reg, - name='%s/dense' % name + name='%s/dense' % name, ) self._sub_layers.append(dense) - bn = tf.keras.layers.BatchNormalization( - name='%s/bn' % name, trainable=True - ) + bn = tf.keras.layers.BatchNormalization(name='%s/bn' % name, trainable=True) self._sub_layers.append(bn) self._sub_layers.append(act_layer) else: @@ -103,7 +125,7 @@ def add_rich_layer( use_bias=use_bias, kernel_initializer=initializer, kernel_regularizer=l2_reg, - name='%s/dense' % name + name='%s/dense' % name, ) self._sub_layers.append(dense) self._sub_layers.append(act_layer) @@ -135,7 +157,6 @@ def call(self, x, training=None, **kwargs): class Highway(Layer): - def __init__(self, params, name='highway', reuse=None, **kwargs): super(Highway, self).__init__(name=name, **kwargs) self.emb_size = params.get_or_default('emb_size', None) @@ -144,9 +165,7 @@ def __init__(self, params, name='highway', reuse=None, **kwargs): self.dropout_rate = params.get_or_default('dropout_rate', 0.0) self.init_gate_bias = params.get_or_default('init_gate_bias', -3.0) self.act_layer = activation_layer(self.activation) - self.dropout_layer = Dropout( - self.dropout_rate - ) if self.dropout_rate > 0.0 else None + self.dropout_layer = Dropout(self.dropout_rate) if self.dropout_rate > 0.0 else None self.project_layer = None self.gate_bias_initializer = Constant(self.init_gate_bias) self.gates = [] # T @@ -159,13 +178,13 @@ def build(self, input_shape): if self.emb_size is not None and dim != self.emb_size: self.project_layer = Dense(self.emb_size, name='input_projection') dim = self.emb_size - self.carry_gate = Lambda(lambda x: 1.0 - x, output_shape=(dim, )) + self.carry_gate = Lambda(lambda x: 1.0 - x, output_shape=(dim,)) for i in range(self.num_layers): gate = Dense( units=dim, bias_initializer=self.gate_bias_initializer, activation='sigmoid', - name='gate_%d' % i + name='gate_%d' % i, ) self.gates.append(gate) self.transforms.append(Dense(units=dim)) @@ -199,9 +218,7 @@ def __init__(self, params, name='gate', reuse=None, **kwargs): self.top_mlp = None def call(self, inputs, training=None, **kwargs): - assert len( - inputs - ) > 1, 'input of Gate layer must be a list containing at least 2 elements' + assert len(inputs) > 1, 'input of Gate layer must be a list containing at least 2 elements' weights = inputs[self.weight_index] j = 0 for i, x in enumerate(inputs): @@ -229,19 +246,15 @@ def __init__(self, params, name='text_cnn', reuse=None, **kwargs): self.config = params.get_pb_config() self.pad_seq_length = self.config.pad_sequence_length if self.pad_seq_length <= 0: - logging.warning( - 'run text cnn with pad_sequence_length <= 0, the predict of model may be unstable' - ) + logging.warning('run text cnn with pad_sequence_length <= 0, the predict of model may be unstable') self.conv_layers = [] self.pool_layer = tf.keras.layers.GlobalMaxPool1D() self.concat_layer = tf.keras.layers.Concatenate(axis=-1) - for size, filters in zip( - self.config.filter_sizes, self.config.num_filters - ): + for size, filters in zip(self.config.filter_sizes, self.config.num_filters): conv = tf.keras.layers.Conv1D( filters=int(filters), kernel_size=int(size), - activation=self.config.activation + activation=self.config.activation, ) self.conv_layers.append(conv) if self.config.HasField('mlp'): @@ -258,9 +271,7 @@ def call(self, inputs, training=None, **kwargs): seq_emb, seq_len = inputs[:2] if self.pad_seq_length > 0: - seq_emb, seq_len = pad_or_truncate_sequence( - seq_emb, seq_len, self.pad_seq_length - ) + seq_emb, seq_len = pad_or_truncate_sequence(seq_emb, seq_len, self.pad_seq_length) pooled_outputs = [] for layer in self.conv_layers: conv = layer(seq_emb) diff --git a/easy_rec/python/layers/keras/bst.py b/easy_rec/python/layers/keras/bst.py index 4c6e92086..5920ffa10 100644 --- a/easy_rec/python/layers/keras/bst.py +++ b/easy_rec/python/layers/keras/bst.py @@ -12,7 +12,6 @@ class BST(Layer): - def __init__(self, params, name='bst', reuse=None, **kwargs): super(BST, self).__init__(name=name, **kwargs) self.reuse = reuse @@ -24,7 +23,7 @@ def encode(self, seq_input, max_position): seq_input, position_embedding_name=self.name, max_position_embeddings=max_position, - reuse_position_embedding=self.reuse + reuse_position_embedding=self.reuse, ) n = tf.count_nonzero(seq_input, axis=-1) @@ -47,13 +46,11 @@ def encode(self, seq_input, max_position): attention_probs_dropout_prob=self.config.attention_probs_dropout_prob, initializer_range=self.config.initializer_range, name=self.name + '/transformer', - reuse=self.reuse + reuse=self.reuse, ) # attention_fea shape: [batch_size, seq_length, hidden_size] if self.config.output_all_token_embeddings: - out_fea = tf.reshape( - attention_fea, [-1, max_position * self.config.hidden_size] - ) + out_fea = tf.reshape(attention_fea, [-1, max_position * self.config.hidden_size]) else: out_fea = attention_fea[:, 0, :] # target feature print('bst output shape:', out_fea.shape) @@ -70,24 +67,26 @@ def call(self, inputs, training=None, **kwargs): target = inputs[2] if len(inputs) > 2 else None max_position = self.config.max_position_embeddings # max_seq_len: the max sequence length in current mini-batch, all sequences are padded to this length - batch_size, cur_batch_max_seq_len, seq_embed_size = get_shape_list( - seq_input, 3 - ) + batch_size, cur_batch_max_seq_len, seq_embed_size = get_shape_list(seq_input, 3) valid_len = tf.assert_less_equal( cur_batch_max_seq_len, max_position, - message='sequence length is greater than `max_position_embeddings`:' + - str(max_position) + ' in feature group:' + self.name + - ', you should set `max_seq_len` in sequence feature configs' + message='sequence length is greater than `max_position_embeddings`:' + + str(max_position) + + ' in feature group:' + + self.name + + ', you should set `max_seq_len` in sequence feature configs', ) if self.config.output_all_token_embeddings: seq_input = tf.cond( - tf.constant(max_position) > cur_batch_max_seq_len, lambda: tf.pad( - seq_input, [ - [0, 0], [0, max_position - cur_batch_max_seq_len], [0, 0] - ], 'CONSTANT' - ), lambda: tf.slice(seq_input, [0, 0, 0], [-1, max_position, -1]) + tf.constant(max_position) > cur_batch_max_seq_len, + lambda: tf.pad( + seq_input, + [[0, 0], [0, max_position - cur_batch_max_seq_len], [0, 0]], + 'CONSTANT', + ), + lambda: tf.slice(seq_input, [0, 0, 0], [-1, max_position, -1]), ) if seq_embed_size != self.config.hidden_size: @@ -97,14 +96,15 @@ def call(self, inputs, training=None, **kwargs): activation=tf.nn.relu, kernel_regularizer=self.l2_reg, name=self.name + '/seq_project', - reuse=self.reuse + reuse=self.reuse, ) keep_target = self.config.target_item_position in ('head', 'tail') if target is not None and keep_target: target_size = target.shape.as_list()[-1] - assert seq_embed_size == target_size, 'the embedding size of sequence and target item is not equal' \ - ' in feature group:' + self.name + assert seq_embed_size == target_size, ( + 'the embedding size of sequence and target item is not equal' ' in feature group:' + self.name + ) if target_size != self.config.hidden_size: target = tf.layers.dense( target, @@ -112,7 +112,7 @@ def call(self, inputs, training=None, **kwargs): activation=tf.nn.relu, kernel_regularizer=self.l2_reg, name=self.name + '/target_project', - reuse=self.reuse + reuse=self.reuse, ) # target_feature: [batch_size, 1, embed_size] target = tf.expand_dims(target, 1) diff --git a/easy_rec/python/layers/keras/custom_ops.py b/easy_rec/python/layers/keras/custom_ops.py index feeacb4f7..2e04c6334 100644 --- a/easy_rec/python/layers/keras/custom_ops.py +++ b/easy_rec/python/layers/keras/custom_ops.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Convenience blocks for using custom ops.""" + import logging import os @@ -30,9 +31,7 @@ custom_ops = tf.load_op_library(custom_op_path) logging.info('load custom op from %s succeed' % custom_op_path) except Exception as ex: - logging.warning( - 'load custom op from %s failed: %s' % (custom_op_path, str(ex)) - ) + logging.warning('load custom op from %s failed: %s' % (custom_op_path, str(ex))) custom_ops = None # if tf.__version__ >= '2.0': @@ -49,27 +48,26 @@ def __init__(self, params, name='sequence_aug', reuse=None, **kwargs): self.seq_augment = custom_ops.my_seq_augment def call(self, inputs, training=None, **kwargs): - assert isinstance( - inputs, (list, tuple) - ), 'the inputs of SeqAugmentOps must be type of list/tuple' + assert isinstance(inputs, (list, tuple)), 'the inputs of SeqAugmentOps must be type of list/tuple' assert len(inputs) >= 2, 'SeqAugmentOps must have at least 2 inputs' seq_input, seq_len = inputs[:2] embedding_dim = int(seq_input.shape[-1]) with tf.variable_scope(self.name, reuse=self.reuse): - mask_emb = tf.get_variable( - 'mask', (embedding_dim, ), dtype=tf.float32, trainable=True - ) + mask_emb = tf.get_variable('mask', (embedding_dim,), dtype=tf.float32, trainable=True) seq_len = tf.to_int32(seq_len) with ops.device('/CPU:0'): aug_seq, aug_len = self.seq_augment( - seq_input, seq_len, mask_emb, self.seq_aug_params.crop_rate, - self.seq_aug_params.reorder_rate, self.seq_aug_params.mask_rate + seq_input, + seq_len, + mask_emb, + self.seq_aug_params.crop_rate, + self.seq_aug_params.reorder_rate, + self.seq_aug_params.mask_rate, ) return aug_seq, aug_len class TextNormalize(Layer): - def __init__(self, params, name='text_normalize', reuse=None, **kwargs): super(TextNormalize, self).__init__(name=name, **kwargs) self.txt_normalizer = custom_ops.text_normalize_op @@ -80,9 +78,7 @@ def call(self, inputs, training=None, **kwargs): inputs = inputs if type(inputs) in (tuple, list) else [inputs] with ops.device('/CPU:0'): result = [ - self.txt_normalizer( - txt, parameter=self.norm_parameter, remove_space=self.remove_space - ) for txt in inputs + self.txt_normalizer(txt, parameter=self.norm_parameter, remove_space=self.remove_space) for txt in inputs ] if len(result) == 1: return result[0] @@ -90,7 +86,6 @@ def call(self, inputs, training=None, **kwargs): class MappedDotProduct(Layer): - def __init__(self, params, name='mapped_dot_product', reuse=None, **kwargs): super(MappedDotProduct, self).__init__(name=name, **kwargs) self.mapped_dot_product = custom_ops.mapped_dot_product @@ -108,7 +103,7 @@ def __init__(self, params, name='mapped_dot_product', reuse=None, **kwargs): self.embedding_table = tf.get_variable( name='dot_product_emb_table', shape=[vocab_size, self.emb_dim], - dtype=tf.float32 + dtype=tf.float32, ) def call(self, inputs, training=None, **kwargs): @@ -119,17 +114,18 @@ def call(self, inputs, training=None, **kwargs): document=doc, feature_name=self.name, separator=self.separator, - default_value=self.default_value + default_value=self.default_value, ) tf.summary.scalar(self.name, tf.reduce_mean(feature)) if self.print_first_n: encode_q = tf.regex_replace(query, self.separator, ' ') encode_t = tf.regex_replace(query, self.separator, ' ') feature = tf.Print( - feature, [encode_q, encode_t, feature], + feature, + [encode_q, encode_t, feature], message=self.name, first_n=self.print_first_n, - summarize=self.summarize + summarize=self.summarize, ) if self.norm_fn is not None: fn = eval(self.norm_fn) @@ -137,10 +133,11 @@ def call(self, inputs, training=None, **kwargs): tf.summary.scalar('normalized_%s' % self.name, tf.reduce_mean(feature)) if self.print_first_n: feature = tf.Print( - feature, [feature], + feature, + [feature], message='normalized %s' % self.name, first_n=self.print_first_n, - summarize=self.summarize + summarize=self.summarize, ) if self.boundaries: feature = self.bucketize(feature, boundaries=self.boundaries) @@ -153,7 +150,6 @@ def call(self, inputs, training=None, **kwargs): class OverlapFeature(Layer): - def __init__(self, params, name='overlap_feature', reuse=None, **kwargs): super(OverlapFeature, self).__init__(name=name, **kwargs) self.overlap_feature = custom_ops.overlap_fg_op @@ -174,7 +170,7 @@ def __init__(self, params, name='overlap_feature', reuse=None, **kwargs): self.embedding_table = tf.get_variable( name='overlap_emb_table', shape=[vocab_size, self.emb_dim], - dtype=tf.float32 + dtype=tf.float32, ) def call(self, inputs, training=None, **kwargs): @@ -188,7 +184,7 @@ def call(self, inputs, training=None, **kwargs): default_value=self.default_value, boundaries=self.boundaries, methods=self.methods, - dtype=tf.int32 if self.boundaries else tf.float32 + dtype=tf.int32 if self.boundaries else tf.float32, ) for i, method in enumerate(self.methods): @@ -201,10 +197,11 @@ def call(self, inputs, training=None, **kwargs): encode_q = tf.regex_replace(query, self.separator, ' ') encode_t = tf.regex_replace(query, self.separator, ' ') feature = tf.Print( - feature, [encode_q, encode_t, feature], + feature, + [encode_q, encode_t, feature], message=self.name, first_n=self.print_first_n, - summarize=self.summarize + summarize=self.summarize, ) if self.norm_fn is not None: fn = eval(self.norm_fn) @@ -219,24 +216,17 @@ def call(self, inputs, training=None, **kwargs): # Compute offsets, add to every column indices offsets = tf.range(num_indices) * vocab_size # Shape: [3] offsets = tf.reshape(offsets, [1, num_indices]) # Shape: [1, 3] - offsets = tf.tile( - offsets, [batch_size, 1] - ) # Shape: [batch_size, num_indices] + offsets = tf.tile(offsets, [batch_size, 1]) # Shape: [batch_size, num_indices] shifted_indices = feature + offsets # Shape: [batch_size, num_indices] flat_feature_ids = tf.reshape(shifted_indices, [-1]) - one_hot_ids = tf.one_hot( - flat_feature_ids, depth=vocab_size * num_indices - ) + one_hot_ids = tf.one_hot(flat_feature_ids, depth=vocab_size * num_indices) feature_embeddings = tf.matmul(one_hot_ids, self.embedding_table) - feature_embeddings = tf.reshape( - feature_embeddings, [batch_size, num_indices * self.emb_dim] - ) + feature_embeddings = tf.reshape(feature_embeddings, [batch_size, num_indices * self.emb_dim]) return feature_embeddings return feature class EditDistance(Layer): - def __init__(self, params, name='edit_distance', reuse=None, **kwargs): super(EditDistance, self).__init__(name=name, **kwargs) self.edit_distance = custom_ops.my_edit_distance @@ -244,9 +234,7 @@ def __init__(self, params, name='edit_distance', reuse=None, **kwargs): self.emb_size = params.get_or_default('embedding_size', 512) emb_dim = params.get_or_default('embedding_dim', 4) with tf.variable_scope(self.name, reuse=reuse): - self.embedding_table = tf.get_variable( - 'embedding_table', [self.emb_size, emb_dim], tf.float32 - ) + self.embedding_table = tf.get_variable('embedding_table', [self.emb_size, emb_dim], tf.float32) def call(self, inputs, training=None, **kwargs): input1, input2 = inputs[:2] @@ -256,7 +244,7 @@ def call(self, inputs, training=None, **kwargs): input2, normalize=False, dtype=tf.int32, - encoding=self.txt_encoding + encoding=self.txt_encoding, ) ids = tf.clip_by_value(dist, 0, self.emb_size - 1) embed = tf.nn.embedding_lookup(self.embedding_table, ids) diff --git a/easy_rec/python/layers/keras/data_augment.py b/easy_rec/python/layers/keras/data_augment.py index a3e0114d4..d10c60999 100644 --- a/easy_rec/python/layers/keras/data_augment.py +++ b/easy_rec/python/layers/keras/data_augment.py @@ -30,16 +30,15 @@ def item_crop(aug_data, length, crop_rate): max_length = tf.cast(max_len, dtype=tf.int32) num_left = tf.cast(tf.math.floor(length1 * crop_rate), dtype=tf.int32) - crop_begin = tf.random.uniform( - [], minval=0, maxval=length - num_left, dtype=tf.int32 - ) + crop_begin = tf.random.uniform([], minval=0, maxval=length - num_left, dtype=tf.int32) zeros = tf.zeros_like(aug_data) - x = aug_data[crop_begin:crop_begin + num_left] - y = zeros[:max_length - num_left] + x = aug_data[crop_begin : crop_begin + num_left] + y = zeros[: max_length - num_left] cropped = tf.concat([x, y], axis=0) cropped_item_seq = tf.where( - crop_begin + num_left < max_length, cropped, - tf.concat([aug_data[crop_begin:], zeros[:crop_begin]], axis=0) + crop_begin + num_left < max_length, + cropped, + tf.concat([aug_data[crop_begin:], zeros[:crop_begin]], axis=0), ) return cropped_item_seq, num_left @@ -47,18 +46,14 @@ def item_crop(aug_data, length, crop_rate): def item_reorder(aug_data, length, reorder_rate): length1 = tf.cast(length, dtype=tf.float32) num_reorder = tf.cast(tf.math.floor(length1 * reorder_rate), dtype=tf.int32) - reorder_begin = tf.random.uniform( - [], minval=0, maxval=length - num_reorder, dtype=tf.int32 - ) + reorder_begin = tf.random.uniform([], minval=0, maxval=length - num_reorder, dtype=tf.int32) shuffle_index = tf.range(reorder_begin, reorder_begin + num_reorder) shuffle_index = tf.random.shuffle(shuffle_index) x = tf.range(get_shape_list(aug_data)[0]) left = tf.slice(x, [0], [reorder_begin]) right = tf.slice(x, [reorder_begin + num_reorder], [-1]) reordered_item_index = tf.concat([left, shuffle_index, right], axis=0) - reordered_item_seq = tf.scatter_nd( - tf.expand_dims(reordered_item_index, axis=1), aug_data, tf.shape(aug_data) - ) + reordered_item_seq = tf.scatter_nd(tf.expand_dims(reordered_item_index, axis=1), aug_data, tf.shape(aug_data)) return reordered_item_seq, length @@ -94,8 +89,9 @@ def reorder_fn(): return tf.cond(tf.equal(method, 0), trans_fn[0], trans_fn[1]) aug_seq, aug_len = tf.cond( - tf.equal(method, 0), crop_fn, - lambda: tf.cond(tf.equal(method, 1), mask_fn, reorder_fn) + tf.equal(method, 0), + crop_fn, + lambda: tf.cond(tf.equal(method, 1), mask_fn, reorder_fn), ) return aug_seq, aug_len @@ -105,7 +101,7 @@ def sequence_augment(seq_input, seq_len, mask, aug_param): aug_seq, aug_len = tf.map_fn( lambda elems: augment_fn(elems, aug_param, mask), elems=(seq_input, lengths), - dtype=(tf.float32, tf.int32) + dtype=(tf.float32, tf.int32), ) aug_seq = tf.reshape(aug_seq, tf.shape(seq_input)) @@ -126,11 +122,7 @@ def call(self, inputs, training=None, **kwargs): embedding_size = int(seq_input.shape[-1]) with tf.variable_scope(self.name, reuse=self.reuse): - mask_emb = tf.get_variable( - 'mask', [1, embedding_size], dtype=tf.float32, trainable=True - ) + mask_emb = tf.get_variable('mask', [1, embedding_size], dtype=tf.float32, trainable=True) - aug_seq, aug_len = sequence_augment( - seq_input, seq_len, mask_emb, self.seq_aug_params - ) + aug_seq, aug_len = sequence_augment(seq_input, seq_len, mask_emb, self.seq_aug_params) return aug_seq, aug_len diff --git a/easy_rec/python/layers/keras/din.py b/easy_rec/python/layers/keras/din.py index d3bded0bb..003c63f1a 100644 --- a/easy_rec/python/layers/keras/din.py +++ b/easy_rec/python/layers/keras/din.py @@ -11,7 +11,6 @@ class DIN(Layer): - def __init__(self, params, name='din', reuse=None, **kwargs): super(DIN, self).__init__(name=name, **kwargs) self.reuse = reuse @@ -31,8 +30,10 @@ def call(self, inputs, training=None, **kwargs): seq_emb_size = keys.shape.as_list()[-1] if query_emb_size != seq_emb_size: logging.info( - ' the embedding size of sequence [%d] and target item [%d] is not equal' - ' in feature group: %s', seq_emb_size, query_emb_size, self.name + ' the embedding size of sequence [%d] and target item [%d] is not equal' ' in feature group: %s', + seq_emb_size, + query_emb_size, + self.name, ) if query_emb_size < seq_emb_size: query = tf.pad(query, [[0, 0], [0, seq_emb_size - query_emb_size]]) @@ -41,15 +42,13 @@ def call(self, inputs, training=None, **kwargs): batch_size, max_seq_len, _ = get_shape_list(keys, 3) queries = tf.tile(tf.expand_dims(query, 1), [1, max_seq_len, 1]) - din_all = tf.concat( - [queries, keys, queries - keys, queries * keys], axis=-1 - ) + din_all = tf.concat([queries, keys, queries - keys, queries * keys], axis=-1) output = self.din_layer(din_all, training) # [B, L, 1] scores = tf.transpose(output, [0, 2, 1]) # [B, 1, L] seq_mask = tf.sequence_mask(seq_len, max_seq_len, dtype=tf.bool) seq_mask = tf.expand_dims(seq_mask, 1) - paddings = tf.ones_like(scores) * (-2**32 + 1) + paddings = tf.ones_like(scores) * (-(2**32) + 1) scores = tf.where(seq_mask, scores, paddings) # [B, 1, L] if self.config.attention_normalizer == 'softmax': scores = tf.nn.softmax(scores) # (B, 1, L) @@ -57,9 +56,7 @@ def call(self, inputs, training=None, **kwargs): scores = scores / (seq_emb_size**0.5) scores = tf.nn.sigmoid(scores) else: - raise ValueError( - 'unsupported attention normalizer: ' + self.config.attention_normalizer - ) + raise ValueError('unsupported attention normalizer: ' + self.config.attention_normalizer) if query_emb_size < seq_emb_size: keys = keys[:, :, :query_emb_size] # [B, L, E] diff --git a/easy_rec/python/layers/keras/einsum_dense.py b/easy_rec/python/layers/keras/einsum_dense.py index dc53a3317..e22f312c4 100644 --- a/easy_rec/python/layers/keras/einsum_dense.py +++ b/easy_rec/python/layers/keras/einsum_dense.py @@ -4,10 +4,14 @@ import string import tensorflow as tf +from tensorflow.python.keras import ( # NOQA + activations, + constraints, + initializers, + regularizers, +) from tensorflow.python.keras.layers import Layer -from tensorflow.python.keras import activations, constraints, initializers, regularizers # NOQA - class EinsumDense(Layer): """A layer that uses `einsum` as the backing computation. @@ -116,12 +120,12 @@ def __init__( kernel_constraint=None, bias_constraint=None, lora_rank=None, - **kwargs + **kwargs, ): super(EinsumDense, self).__init__(**kwargs) self.equation = equation if isinstance(output_shape, int): - self.partial_output_shape = (output_shape, ) + self.partial_output_shape = (output_shape,) else: self.partial_output_shape = tuple(output_shape) self.bias_axes = bias_axes @@ -184,9 +188,7 @@ def build(self, input_shape): @property def kernel(self): if not self.built: - raise AttributeError( - 'You must build the layer before accessing `kernel`.' - ) + raise AttributeError('You must build the layer before accessing `kernel`.') if self.lora_enabled: return self._kernel + tf.matmul(self.lora_kernel_a, self.lora_kernel_b) return self._kernel @@ -202,9 +204,7 @@ def call(self, inputs, training=None): x = self.activation(x) return x - def enable_lora( - self, rank, a_initializer='he_uniform', b_initializer='zeros' - ): + def enable_lora(self, rank, a_initializer='he_uniform', b_initializer='zeros'): if self.kernel_constraint: raise ValueError( 'Lora is incompatible with kernel constraints. ' @@ -214,14 +214,11 @@ def enable_lora( if not self.built: raise ValueError("Cannot enable lora on a layer that isn't yet built.") if self.lora_enabled: - raise ValueError( - 'lora is already enabled. ' - 'This can only be done once per layer.' - ) + raise ValueError('lora is already enabled. ' 'This can only be done once per layer.') self._tracker.unlock() self.lora_kernel_a = self.add_weight( name='lora_kernel_a', - shape=(self.kernel.shape[:-1] + (rank, )), + shape=(self.kernel.shape[:-1] + (rank,)), initializer=initializers.get(a_initializer), regularizer=self.kernel_regularizer, ) @@ -277,8 +274,7 @@ def get_config(self): 'bias_initializer': initializers.serialize(self.bias_initializer), 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), 'bias_regularizer': regularizers.serialize(self.bias_regularizer), - 'activity_regularizer': - regularizers.serialize(self.activity_regularizer), + 'activity_regularizer': regularizers.serialize(self.activity_regularizer), 'kernel_constraint': constraints.serialize(self.kernel_constraint), 'bias_constraint': constraints.serialize(self.bias_constraint), } @@ -312,9 +308,7 @@ def _check_load_own_variables(self, store): '`def get_build_config(self)`. ' 'The method `build_from_config()` is meant ' 'to create the state ' - 'of the layer (i.e. its variables) upon deserialization.'.format( - name=self.name, num_keys=len(store.keys()) - ) + 'of the layer (i.e. its variables) upon deserialization.'.format(name=self.name, num_keys=len(store.keys())) ) raise ValueError( "Layer '{name}' expected {num_var} variables, but received " @@ -323,7 +317,7 @@ def _check_load_own_variables(self, store): name=self.name, num_var=len(store.keys()), num_key=len(store.keys()), - names=[v.name for v in all_vars] + names=[v.name for v in all_vars], ) ) @@ -338,43 +332,27 @@ def _analyze_einsum_string(equation, bias_axes, input_shape, output_shape): dot_replaced_string = re.sub(r'\.\.\.', '0', equation) # This is the case where no ellipses are present in the string. - split_string = re.match( - '([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', dot_replaced_string - ) + split_string = re.match('([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', dot_replaced_string) if split_string: - return _analyze_split_string( - split_string, bias_axes, input_shape, output_shape - ) + return _analyze_split_string(split_string, bias_axes, input_shape, output_shape) # This is the case where ellipses are present on the left. - split_string = re.match( - '0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', dot_replaced_string - ) + split_string = re.match('0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', dot_replaced_string) if split_string: - return _analyze_split_string( - split_string, bias_axes, input_shape, output_shape, left_elided=True - ) + return _analyze_split_string(split_string, bias_axes, input_shape, output_shape, left_elided=True) # This is the case where ellipses are present on the right. - split_string = re.match( - '([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', dot_replaced_string - ) + split_string = re.match('([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', dot_replaced_string) if split_string: - return _analyze_split_string( - split_string, bias_axes, input_shape, output_shape - ) + return _analyze_split_string(split_string, bias_axes, input_shape, output_shape) raise ValueError( "Invalid einsum equation '{equation}'. Equations must be in the form " - '[X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format( - equation=equation - ) + '[X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format(equation=equation) ) -def _analyze_split_string( - split_string, bias_axes, input_shape, output_shape, left_elided=False -): +def _analyze_split_string(split_string, bias_axes, input_shape, output_shape, left_elided=False): """Analyze an pre-split einsum string to find the weight shape.""" input_spec = split_string.group(1) weight_spec = split_string.group(2) @@ -399,10 +377,7 @@ def _analyze_split_string( if left_elided: # If we have beginning dimensions elided, we need to use negative # indexing to determine where in the input dimension our values are. - input_dim_map = { - dim: (i + elided) - len(input_shape) - for i, dim in enumerate(input_spec) - } + input_dim_map = {dim: (i + elided) - len(input_shape) for i, dim in enumerate(input_spec)} # Because we've constructed the full output shape already, we don't need # to do negative indexing. output_dim_map = {dim: (i + elided) for i, dim in enumerate(output_spec)} @@ -414,17 +389,14 @@ def _analyze_split_string( input_shape_at_dim = input_shape[input_dim_map[dim]] if dim in output_dim_map: output_shape_at_dim = output_shape[output_dim_map[dim]] - if ( - output_shape_at_dim is not None - and output_shape_at_dim != input_shape_at_dim - ): + if output_shape_at_dim is not None and output_shape_at_dim != input_shape_at_dim: raise ValueError( 'Input shape and output shape do not match at shared ' "dimension '{dim}'. Input shape is {input_shape_at_dim}, " 'and output shape is {output_shape}.'.format( dim=dim, input_shape_at_dim=input_shape_at_dim, - output_shape=output_shape[output_dim_map[dim]] + output_shape=output_shape[output_dim_map[dim]], ) ) @@ -449,23 +421,17 @@ def _analyze_split_string( "Weight dimension '{dim}' did not have a match in either " "the input spec '{input_spec}' or the output " "spec '{output_spec}'. For this layer, the weight must " - 'be fully specified.'.format( - dim=dim, input_spec=input_spec, output_spec=output_spec - ) + 'be fully specified.'.format(dim=dim, input_spec=input_spec, output_spec=output_spec) ) if bias_axes is not None: num_left_elided = elided if left_elided else 0 - idx_map = { - char: output_shape[i + num_left_elided] - for i, char in enumerate(output_spec) - } + idx_map = {char: output_shape[i + num_left_elided] for i, char in enumerate(output_spec)} for char in bias_axes: if char not in output_spec: raise ValueError( - "Bias dimension '{char}' was requested, but is not part " - "of the output spec '{output_spec}'".format( + "Bias dimension '{char}' was requested, but is not part " "of the output spec '{output_spec}'".format( char=char, output_spec=output_spec ) ) @@ -473,9 +439,7 @@ def _analyze_split_string( first_bias_location = min([output_spec.find(char) for char in bias_axes]) bias_output_spec = output_spec[first_bias_location:] - bias_shape = [ - idx_map[char] if char in bias_axes else 1 for char in bias_output_spec - ] + bias_shape = [idx_map[char] if char in bias_axes else 1 for char in bias_output_spec] if not left_elided: for _ in range(elided): @@ -487,15 +451,12 @@ def _analyze_split_string( def _analyze_quantization_info(equation, input_shape): - def get_specs(equation, input_shape): possible_labels = string.ascii_letters dot_replaced_string = re.sub(r'\.\.\.', '0', equation) # This is the case where no ellipses are present in the string. - split_string = re.match( - '([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', dot_replaced_string - ) + split_string = re.match('([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', dot_replaced_string) if split_string is not None: input_spec = split_string.group(1) weight_spec = split_string.group(2) @@ -503,18 +464,13 @@ def get_specs(equation, input_shape): return input_spec, weight_spec, output_spec # This is the case where ellipses are present on the left. - split_string = re.match( - '0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', dot_replaced_string - ) + split_string = re.match('0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', dot_replaced_string) if split_string is not None: input_spec = split_string.group(1) weight_spec = split_string.group(2) output_spec = split_string.group(3) elided = len(input_shape) - len(input_spec) - possible_labels = sorted( - set(possible_labels) - set(input_spec) - set(weight_spec) - - set(output_spec) - ) + possible_labels = sorted(set(possible_labels) - set(input_spec) - set(weight_spec) - set(output_spec)) # Pad labels on the left to `input_spec` and `output_spec` for i in range(elided): input_spec = possible_labels[i] + input_spec @@ -522,18 +478,13 @@ def get_specs(equation, input_shape): return input_spec, weight_spec, output_spec # This is the case where ellipses are present on the right. - split_string = re.match( - '([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', dot_replaced_string - ) + split_string = re.match('([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', dot_replaced_string) if split_string is not None: input_spec = split_string.group(1) weight_spec = split_string.group(2) output_spec = split_string.group(3) elided = len(input_shape) - len(input_spec) - possible_labels = sorted( - set(possible_labels) - set(input_spec) - set(weight_spec) - - set(output_spec) - ) + possible_labels = sorted(set(possible_labels) - set(input_spec) - set(weight_spec) - set(output_spec)) # Pad labels on the right to `input_spec` and `output_spec` for i in range(elided): input_spec = input_spec + possible_labels[i] @@ -542,9 +493,7 @@ def get_specs(equation, input_shape): raise ValueError( "Invalid einsum equation '{equation}'. Equations must be in the " - 'form [X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format( - equation=equation - ) + 'form [X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format(equation=equation) ) input_spec, weight_spec, output_spec = get_specs(equation, input_shape) @@ -603,10 +552,7 @@ def get_specs(equation, input_shape): custom_gradient_equation = '{output_spec},{weight_spec}->{input_spec}'.format( output_spec=output_spec, input_spec=input_spec, weight_spec=weight_spec ) - weight_reverse_transpose_axes = [ - i for (_, - i) in sorted((v, i) for (i, v) in enumerate(weight_transpose_axes)) - ] + weight_reverse_transpose_axes = [i for (_, i) in sorted((v, i) for (i, v) in enumerate(weight_transpose_axes))] return ( input_reduced_axes, weight_reduced_axes, diff --git a/easy_rec/python/layers/keras/embedding.py b/easy_rec/python/layers/keras/embedding.py index 578e7fcf5..661036f82 100644 --- a/easy_rec/python/layers/keras/embedding.py +++ b/easy_rec/python/layers/keras/embedding.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Fused embedding layer.""" + import tensorflow as tf from tensorflow.python.keras.layers import Embedding, Layer @@ -23,7 +24,6 @@ def _combine(embeddings, weights, comb_fn): class EmbeddingLayer(Layer): - def __init__(self, params, name='embedding_layer', reuse=None, **kwargs): super(EmbeddingLayer, self).__init__(name=name, **kwargs) params.check_required(['vocab_size', 'embedding_dim']) @@ -71,10 +71,12 @@ def call(self, inputs, training=None, **kwargs): batch_size = tf.shape(inputs[i])[0] embeddings[i] = tf.cond( tf.equal(tf.size(embeddings[i]), 0), - lambda: tf.zeros([batch_size, self.embed_dim]), lambda: _combine( + lambda: tf.zeros([batch_size, self.embed_dim]), + lambda: _combine( tf.reshape(embeddings[i], [batch_size, -1, self.embed_dim]), - weights[i], self.combine_fn - ) + weights[i], + self.combine_fn, + ), ) if self.do_concat: embeddings = tf.concat(embeddings, axis=-1) diff --git a/easy_rec/python/layers/keras/fibinet.py b/easy_rec/python/layers/keras/fibinet.py index d58f4d154..72cf64383 100644 --- a/easy_rec/python/layers/keras/fibinet.py +++ b/easy_rec/python/layers/keras/fibinet.py @@ -43,9 +43,7 @@ def build(self, input_shape): for shape in input_shape: assert shape.ndims == 2, 'field embeddings must be rank 2 tensors' dim = int(shape[-1]) - assert dim >= g and dim % g == 0, 'field embedding dimension %d must be divisible by %d' % ( - dim, g - ) + assert dim >= g and dim % g == 0, 'field embedding dimension %d must be divisible by %d' % (dim, g) emb_size += dim r = self.config.reduction_ratio @@ -55,11 +53,9 @@ def build(self, input_shape): units=reduction_size, activation='relu', kernel_initializer='he_normal', - name='W1' - ) - self.excite_layer = Dense( - units=emb_size, kernel_initializer='glorot_normal', name='W2' + name='W1', ) + self.excite_layer = Dense(units=emb_size, kernel_initializer='glorot_normal', name='W2') super(SENet, self).build(input_shape) # Be sure to call this somewhere! def call(self, inputs, **kwargs): @@ -67,9 +63,7 @@ def call(self, inputs, **kwargs): # Squeeze # embedding dimension 必须能被 g 整除 - group_embs = [ - tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs - ] + group_embs = [tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs] squeezed = [] for emb in group_embs: @@ -97,9 +91,7 @@ def call(self, inputs, **kwargs): def _full_interaction(v_i, v_j): # [bs, 1, dim] x [bs, dim, 1] = [bs, 1] - interaction = tf.matmul( - tf.expand_dims(v_i, axis=1), tf.expand_dims(v_j, axis=-1) - ) + interaction = tf.matmul(tf.expand_dims(v_i, axis=1), tf.expand_dims(v_j, axis=-1)) return tf.squeeze(interaction, axis=1) @@ -134,9 +126,7 @@ def __init__(self, params, name='bilinear', reuse=None, **kwargs): self.output_size = params.num_output_units self.bilinear_type = params.get_or_default('type', 'interaction').lower() if self.bilinear_type not in ['all', 'each', 'interaction']: - raise NotImplementedError( - "bilinear_type only support: ['all', 'each', 'interaction']" - ) + raise NotImplementedError("bilinear_type only support: ['all', 'each', 'interaction']") if bilinear_plus: self.func = _full_interaction else: @@ -157,22 +147,17 @@ def build(self, input_shape): if shape[-1] != _dim: equal_dim = False if not equal_dim and self.bilinear_type != 'interaction': - raise ValueError( - 'all embedding dimensions must be same when not use bilinear type: interaction' - ) + raise ValueError('all embedding dimensions must be same when not use bilinear type: interaction') dim = int(_dim) if self.bilinear_type == 'all': self.dot_layer = Dense(dim, name='all') elif self.bilinear_type == 'each': - self.dot_layers = [ - Dense(dim, name='each_%d' % i) for i in range(field_num - 1) - ] + self.dot_layers = [Dense(dim, name='each_%d' % i) for i in range(field_num - 1)] else: # interaction self.dot_layers = [ - Dense( - units=int(input_shape[j][-1]), name='interaction_%d_%d' % (i, j) - ) for i, j in itertools.combinations(range(field_num), 2) + Dense(units=int(input_shape[j][-1]), name='interaction_%d_%d' % (i, j)) + for i, j in itertools.combinations(range(field_num), 2) ] super(BiLinear, self).build(input_shape) # Be sure to call this somewhere! @@ -187,23 +172,14 @@ def call(self, inputs, **kwargs): # - where k_i is the embedding size of the ith field if self.bilinear_type == 'all': v_dot = [self.dot_layer(v_i) for v_i in embeddings[:-1]] - p = [ - self.func(v_dot[i], embeddings[j]) - for i, j in itertools.combinations(range(field_num), 2) - ] + p = [self.func(v_dot[i], embeddings[j]) for i, j in itertools.combinations(range(field_num), 2)] elif self.bilinear_type == 'each': - v_dot = [ - self.dot_layers[i](v_i) for i, v_i in enumerate(embeddings[:-1]) - ] - p = [ - self.func(v_dot[i], embeddings[j]) - for i, j in itertools.combinations(range(field_num), 2) - ] + v_dot = [self.dot_layers[i](v_i) for i, v_i in enumerate(embeddings[:-1])] + p = [self.func(v_dot[i], embeddings[j]) for i, j in itertools.combinations(range(field_num), 2)] else: # interaction p = [ - self.func( - self.dot_layers[i * field_num + j](embeddings[i]), embeddings[j] - ) for i, j in itertools.combinations(range(field_num), 2) + self.func(self.dot_layers[i * field_num + j](embeddings[i]), embeddings[j]) + for i, j in itertools.combinations(range(field_num), 2) ] return self.output_layer(tf.concat(p, axis=-1)) @@ -223,15 +199,11 @@ def __init__(self, params, name='fibinet', reuse=None, **kwargs): self._config = params.get_pb_config() se_params = Parameter.make_from_pb(self._config.senet) - self.senet_layer = SENet( - se_params, name=self.name + '/senet', reuse=self.reuse - ) + self.senet_layer = SENet(se_params, name=self.name + '/senet', reuse=self.reuse) if self._config.HasField('bilinear'): bi_params = Parameter.make_from_pb(self._config.bilinear) - self.bilinear_layer = BiLinear( - bi_params, name=self.name + '/bilinear', reuse=self.reuse - ) + self.bilinear_layer = BiLinear(bi_params, name=self.name + '/bilinear', reuse=self.reuse) if self._config.HasField('mlp'): p = Parameter.make_from_pb(self._config.mlp) diff --git a/easy_rec/python/layers/keras/interaction.py b/easy_rec/python/layers/keras/interaction.py index d8098f6c0..b3d53ade7 100644 --- a/easy_rec/python/layers/keras/interaction.py +++ b/easy_rec/python/layers/keras/interaction.py @@ -92,10 +92,7 @@ def call(self, inputs, **kwargs): try: concat_features = tf.stack(inputs, axis=1) except (ValueError, tf.errors.InvalidArgumentError) as e: - raise ValueError( - 'Input tensors` dimensions must be equal, original' - 'error message: {}'.format(e) - ) + raise ValueError('Input tensors` dimensions must be equal, original' 'error message: {}'.format(e)) else: assert inputs.shape.ndims == 3, 'input of dot func must be a 3D tensor or a list of 2D tensors' concat_features = inputs @@ -122,7 +119,7 @@ def call(self, inputs, **kwargs): activations = tf.where( condition=tf.cast(upper_tri_mask, tf.bool), x=tf.zeros_like(xactions), - y=xactions + y=xactions, ) out_dim = num_features * num_features else: @@ -198,9 +195,7 @@ def __init__(self, params, name='cross', reuse=None, **kwargs): preactivation = params.get_or_default('preactivation', None) preact = get_activation(preactivation) self._preactivation = tf.keras.activations.get(preact) - kernel_initializer = params.get_or_default( - 'kernel_initializer', 'truncated_normal' - ) + kernel_initializer = params.get_or_default('kernel_initializer', 'truncated_normal') self._kernel_initializer = tf.keras.initializers.get(kernel_initializer) bias_initializer = params.get_or_default('bias_initializer', 'zeros') self._bias_initializer = tf.keras.initializers.get(bias_initializer) @@ -212,11 +207,7 @@ def __init__(self, params, name='cross', reuse=None, **kwargs): self._supports_masking = True if self._diag_scale < 0: # pytype: disable=unsupported-operands - raise ValueError( - '`diag_scale` should be non-negative. Got `diag_scale` = {}'.format( - self._diag_scale - ) - ) + raise ValueError('`diag_scale` should be non-negative. Got `diag_scale` = {}'.format(self._diag_scale)) def build(self, input_shape): last_dim = input_shape[0][-1] @@ -276,9 +267,7 @@ def call(self, inputs, **kwargs): if x0.shape[-1] != x.shape[-1]: raise ValueError( '`x0` and `x` dimension mismatch! Got `x0` dimension {}, and x ' - 'dimension {}. This case is not supported yet.'.format( - x0.shape[-1], x.shape[-1] - ) + 'dimension {}. This case is not supported yet.'.format(x0.shape[-1], x.shape[-1]) ) if self._projection_dim is None: @@ -295,22 +284,14 @@ def call(self, inputs, **kwargs): def get_config(self): config = { - 'projection_dim': - self._projection_dim, - 'diag_scale': - self._diag_scale, - 'use_bias': - self._use_bias, - 'preactivation': - tf.keras.activations.serialize(self._preactivation), - 'kernel_initializer': - tf.keras.initializers.serialize(self._kernel_initializer), - 'bias_initializer': - tf.keras.initializers.serialize(self._bias_initializer), - 'kernel_regularizer': - tf.keras.regularizers.serialize(self._kernel_regularizer), - 'bias_regularizer': - tf.keras.regularizers.serialize(self._bias_regularizer), + 'projection_dim': self._projection_dim, + 'diag_scale': self._diag_scale, + 'use_bias': self._use_bias, + 'preactivation': tf.keras.activations.serialize(self._preactivation), + 'kernel_initializer': tf.keras.initializers.serialize(self._kernel_initializer), + 'bias_initializer': tf.keras.initializers.serialize(self._bias_initializer), + 'kernel_regularizer': tf.keras.regularizers.serialize(self._kernel_regularizer), + 'bias_regularizer': tf.keras.regularizers.serialize(self._bias_regularizer), } base_config = super(Cross, self).get_config() return dict(list(base_config.items()) + list(config.items())) @@ -331,13 +312,11 @@ class CIN(tf.keras.layers.Layer): def __init__(self, params, name='cin', reuse=None, **kwargs): super(CIN, self).__init__(name=name, **kwargs) self._name = name - self._hidden_feature_sizes = list( - params.get_or_default('hidden_feature_sizes', []) - ) + self._hidden_feature_sizes = list(params.get_or_default('hidden_feature_sizes', [])) - assert isinstance(self._hidden_feature_sizes, list) and len( - self._hidden_feature_sizes - ) > 0, 'parameter hidden_feature_sizes must be a list of int with length greater than 0' + assert ( + isinstance(self._hidden_feature_sizes, list) and len(self._hidden_feature_sizes) > 0 + ), 'parameter hidden_feature_sizes must be a list of int with length greater than 0' kernel_regularizer = params.get_or_default('kernel_regularizer', None) self._kernel_regularizer = tf.keras.regularizers.get(kernel_regularizer) @@ -346,26 +325,24 @@ def __init__(self, params, name='cin', reuse=None, **kwargs): def build(self, input_shape): if len(input_shape) != 3: - raise ValueError( - 'Unexpected inputs dimensions %d, expect to be 3 dimensions' % - (len(input_shape)) - ) + raise ValueError('Unexpected inputs dimensions %d, expect to be 3 dimensions' % (len(input_shape))) - hidden_feature_sizes = [input_shape[1] - ] + [h for h in self._hidden_feature_sizes] + hidden_feature_sizes = [input_shape[1]] + [h for h in self._hidden_feature_sizes] tfv1 = tf.compat.v1 if tf.__version__ >= '2.0' else tf with tfv1.variable_scope(self._name): self.kernel_list = [ tfv1.get_variable( name='cin_kernel_%d' % i, shape=[ - hidden_feature_sizes[i + 1], hidden_feature_sizes[i], - hidden_feature_sizes[0] + hidden_feature_sizes[i + 1], + hidden_feature_sizes[i], + hidden_feature_sizes[0], ], initializer=tf.initializers.he_normal(), regularizer=self._kernel_regularizer, - trainable=True - ) for i in range(len(self._hidden_feature_sizes)) + trainable=True, + ) + for i in range(len(self._hidden_feature_sizes)) ] self.bias_list = [ tfv1.get_variable( @@ -373,8 +350,9 @@ def build(self, input_shape): shape=[hidden_feature_sizes[i + 1]], initializer=tf.keras.initializers.Zeros, regularizer=self._bias_regularizer, - trainable=True - ) for i in range(len(self._hidden_feature_sizes)) + trainable=True, + ) + for i in range(len(self._hidden_feature_sizes)) ] super(CIN, self).build(input_shape) @@ -401,29 +379,23 @@ def call(self, input, **kwargs): intermediate_tensor = tf.multiply(x_0_expanded, x_i_expanded) intermediate_tensor_expanded = tf.expand_dims(intermediate_tensor, 1) - intermediate_tensor_expanded = tf.tile( - intermediate_tensor_expanded, [1, hk, 1, 1, 1] - ) + intermediate_tensor_expanded = tf.tile(intermediate_tensor_expanded, [1, hk, 1, 1, 1]) feature_map_elementwise = tf.multiply( intermediate_tensor_expanded, - tf.expand_dims(tf.expand_dims(self.kernel_list[i], -1), 0) - ) - feature_map = tf.reduce_sum( - tf.reduce_sum(feature_map_elementwise, axis=3), axis=2 + tf.expand_dims(tf.expand_dims(self.kernel_list[i], -1), 0), ) + feature_map = tf.reduce_sum(tf.reduce_sum(feature_map_elementwise, axis=3), axis=2) feature_map = tf.add( feature_map, - tf.expand_dims(tf.expand_dims(self.bias_list[i], axis=-1), axis=0) + tf.expand_dims(tf.expand_dims(self.bias_list[i], axis=-1), axis=0), ) feature_map = tf.nn.relu(feature_map) x_i = feature_map pooled_feature_map_list.append(tf.reduce_sum(feature_map, axis=-1)) - return tf.concat( - pooled_feature_map_list, axis=-1 - ) # shape = (b, h1 + ... + hk) + return tf.concat(pooled_feature_map_list, axis=-1) # shape = (b, h1 + ... + hk) def get_config(self): pass diff --git a/easy_rec/python/layers/keras/layer_norm.py b/easy_rec/python/layers/keras/layer_norm.py index 9415ce977..51a1d8441 100644 --- a/easy_rec/python/layers/keras/layer_norm.py +++ b/easy_rec/python/layers/keras/layer_norm.py @@ -1,4 +1,5 @@ """Layer Normalization layer.""" + import tensorflow as tf from tensorflow.python.keras import constraints, initializers, regularizers from tensorflow.python.keras.layers import Layer @@ -18,11 +19,7 @@ def validate_axis(axis, input_shape): input_shape = tf.TensorShape(input_shape) rank = input_shape.ndims if not rank: - raise ValueError( - 'Input has undefined rank. Received: input_shape={input_shape}'.format( - input_shape=input_shape - ) - ) + raise ValueError('Input has undefined rank. Received: input_shape={input_shape}'.format(input_shape=input_shape)) # Convert axis to list and resolve negatives if isinstance(axis, int): @@ -39,9 +36,7 @@ def validate_axis(axis, input_shape): raise ValueError( 'Invalid value for `axis` argument. ' 'Expected 0 <= axis < inputs.rank (with ' - 'inputs.rank={rank}). Received: axis={axis}'.format( - rank=rank, axis=tuple(axis) - ) + 'inputs.rank={rank}). Received: axis={axis}'.format(rank=rank, axis=tuple(axis)) ) if len(axis) != len(set(axis)): raise ValueError('Duplicate axis: {axis}'.format(axis=tuple(axis))) @@ -183,7 +178,7 @@ def __init__( gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, - **kwargs + **kwargs, ): super(LayerNormalization, self).__init__(**kwargs) if isinstance(axis, (list, tuple)): @@ -191,10 +186,7 @@ def __init__( elif isinstance(axis, int): self.axis = axis else: - raise TypeError( - 'Expected an int or a list/tuple of ints for the ' - "argument 'axis', but received: %r" % axis - ) + raise TypeError('Expected an int or a list/tuple of ints for the ' "argument 'axis', but received: %r" % axis) self.epsilon = epsilon self.center = center @@ -266,8 +258,7 @@ def build(self, input_shape): self.beta = None self._fused = self._fused_can_be_used(rank) - super(LayerNormalization, - self).build(input_shape) # Be sure to call this somewhere! + super(LayerNormalization, self).build(input_shape) # Be sure to call this somewhere! def call(self, inputs): # Compute the axes along which to reduce the mean / variance @@ -281,15 +272,13 @@ def call(self, inputs): broadcast_shape[dim] = input_shape.dims[dim].value def _broadcast(v): - if ( - v is not None and len(v.shape) != ndims and self.axis != [ndims - 1] - ): + if v is not None and len(v.shape) != ndims and self.axis != [ndims - 1]: return tf.reshape(v, broadcast_shape) return v if not self._fused: input_dtype = inputs.dtype - if (input_dtype in ('float16', 'bfloat16') and self.dtype == 'float32'): + if input_dtype in ('float16', 'bfloat16') and self.dtype == 'float32': # If mixed precision is used, cast inputs to float32 so that # this is at least as numerically stable as the fused version. inputs = tf.cast(inputs, 'float32') @@ -315,8 +304,8 @@ def _broadcast(v): axis = sorted(self.axis) tensor_shape = tf.shape(inputs) - pre_dim = tf.reduce_prod(tensor_shape[:axis[0]]) - in_dim = tf.reduce_prod(tensor_shape[axis[0]:]) + pre_dim = tf.reduce_prod(tensor_shape[: axis[0]]) + in_dim = tf.reduce_prod(tensor_shape[axis[0] :]) squeezed_shape = [1, pre_dim, in_dim, 1] # This fused operation requires reshaped inputs to be NCHW. data_format = 'NCHW' diff --git a/easy_rec/python/layers/keras/mask_net.py b/easy_rec/python/layers/keras/mask_net.py index 7ae12f20e..77a1375c9 100644 --- a/easy_rec/python/layers/keras/mask_net.py +++ b/easy_rec/python/layers/keras/mask_net.py @@ -44,16 +44,14 @@ def build(self, input_shape): elif self.config.HasField('aggregation_size') is not None: aggregation_size = self.config.aggregation_size else: - raise ValueError( - 'Need one of reduction factor or aggregation size for MaskBlock.' - ) + raise ValueError('Need one of reduction factor or aggregation size for MaskBlock.') self.aggr_layer = Dense( aggregation_size, activation='relu', kernel_initializer='he_uniform', kernel_regularizer=self.l2_reg, - name='aggregation' + name='aggregation', ) self.weight_layer = Dense(input_dim, name='weights') if self._projection_dim is not None: @@ -62,25 +60,19 @@ def build(self, input_shape): self._projection_dim, kernel_regularizer=self.l2_reg, use_bias=False, - name='project' + name='project', ) if self.config.input_layer_norm: # 推荐在调用MaskBlock之前做好 layer norm，否则每一次调用都需要对input做ln if tf.__version__ >= '2.0': - self.input_layer_norm = tf.keras.layers.LayerNormalization( - name='input_ln' - ) + self.input_layer_norm = tf.keras.layers.LayerNormalization(name='input_ln') else: self.input_layer_norm = LayerNormalization(name='input_ln') if self.config.HasField('output_size'): - self.output_layer = Dense( - self.config.output_size, use_bias=False, name='output' - ) + self.output_layer = Dense(self.config.output_size, use_bias=False, name='output') if tf.__version__ >= '2.0': - self.output_layer_norm = tf.keras.layers.LayerNormalization( - name='output_ln' - ) + self.output_layer_norm = tf.keras.layers.LayerNormalization(name='output_ln') else: self.output_layer_norm = LayerNormalization(name='output_ln') super(MaskBlock, self).build(input_shape) @@ -138,9 +130,7 @@ def __init__(self, params, name='mask_net', reuse=None, **kwargs): if self.config.input_layer_norm: if tf.__version__ >= '2.0': - self.input_layer_norm = tf.keras.layers.LayerNormalization( - name='input_ln' - ) + self.input_layer_norm = tf.keras.layers.LayerNormalization(name='input_ln') else: self.input_layer_norm = LayerNormalization(name='input_ln') @@ -149,9 +139,7 @@ def call(self, inputs, training=None, **kwargs): inputs = self.input_layer_norm(inputs) if self.config.use_parallel: - mask_outputs = [ - mask_layer((inputs, inputs)) for mask_layer in self.mask_layers - ] + mask_outputs = [mask_layer((inputs, inputs)) for mask_layer in self.mask_layers] all_mask_outputs = tf.concat(mask_outputs, axis=1) if self.mlp is not None: output = self.mlp(all_mask_outputs, training=training) diff --git a/easy_rec/python/layers/keras/multi_head_attention.py b/easy_rec/python/layers/keras/multi_head_attention.py index 33386dd8a..7971eb714 100644 --- a/easy_rec/python/layers/keras/multi_head_attention.py +++ b/easy_rec/python/layers/keras/multi_head_attention.py @@ -89,9 +89,7 @@ class MultiHeadAttention(Layer): attention axes. """ - def __init__( - self, params, name='multi_head_attention', reuse=None, **kwargs - ): + def __init__(self, params, name='multi_head_attention', reuse=None, **kwargs): super(MultiHeadAttention, self).__init__(name=name, **kwargs) self.supports_masking = True self._num_heads = params.num_heads @@ -103,32 +101,16 @@ def __init__( self._dropout = params.get_or_default('dropout', 0.0) self._use_bias = params.get_or_default('use_bias', True) self._output_shape = params.get_or_default('output_shape', None) - self._kernel_initializer = initializers.get( - params.get_or_default('kernel_initializer', 'glorot_uniform') - ) - self._bias_initializer = initializers.get( - params.get_or_default('bias_initializer', 'zeros') - ) - self._kernel_regularizer = regularizers.get( - params.get_or_default('kernel_regularizer', None) - ) - self._bias_regularizer = regularizers.get( - params.get_or_default('bias_regularizer', None) - ) - self._activity_regularizer = regularizers.get( - params.get_or_default('activity_regularizer', None) - ) - self._kernel_constraint = constraints.get( - params.get_or_default('kernel_constraint', None) - ) - self._bias_constraint = constraints.get( - params.get_or_default('bias_constraint', None) - ) + self._kernel_initializer = initializers.get(params.get_or_default('kernel_initializer', 'glorot_uniform')) + self._bias_initializer = initializers.get(params.get_or_default('bias_initializer', 'zeros')) + self._kernel_regularizer = regularizers.get(params.get_or_default('kernel_regularizer', None)) + self._bias_regularizer = regularizers.get(params.get_or_default('bias_regularizer', None)) + self._activity_regularizer = regularizers.get(params.get_or_default('activity_regularizer', None)) + self._kernel_constraint = constraints.get(params.get_or_default('kernel_constraint', None)) + self._bias_constraint = constraints.get(params.get_or_default('bias_constraint', None)) self._attention_axes = params.get_or_default('attention_axes', None) self._use_causal_mask = params.get_or_default('use_causal_mask', False) - self._return_attention_scores = params.get_or_default( - 'return_attention_scores', False - ) + self._return_attention_scores = params.get_or_default('return_attention_scores', False) @property def num_heads(self): @@ -172,8 +154,7 @@ def get_config(self): 'bias_initializer': initializers.serialize(self._bias_initializer), 'kernel_regularizer': regularizers.serialize(self._kernel_regularizer), 'bias_regularizer': regularizers.serialize(self._bias_regularizer), - 'activity_regularizer': - regularizers.serialize(self._activity_regularizer), + 'activity_regularizer': regularizers.serialize(self._activity_regularizer), 'kernel_constraint': constraints.serialize(self._kernel_constraint), 'bias_constraint': constraints.serialize(self._bias_constraint), } @@ -194,43 +175,31 @@ def build(self, input_shape): query_rank = len(query_shape) value_rank = len(value_shape) key_rank = len(key_shape) - einsum_equation, bias_axes, output_rank = _build_proj_equation( - query_rank - 1, bound_dims=1, output_dims=2 - ) + einsum_equation, bias_axes, output_rank = _build_proj_equation(query_rank - 1, bound_dims=1, output_dims=2) self._query_dense = EinsumDense( einsum_equation, - output_shape=_get_output_shape( - output_rank - 1, [self._num_heads, self._key_dim] - ), + output_shape=_get_output_shape(output_rank - 1, [self._num_heads, self._key_dim]), bias_axes=bias_axes if self._use_bias else None, name='query', - **self._get_common_kwargs_for_sublayer() + **self._get_common_kwargs_for_sublayer(), ) self._query_dense.build(query_shape) - einsum_equation, bias_axes, output_rank = _build_proj_equation( - key_rank - 1, bound_dims=1, output_dims=2 - ) + einsum_equation, bias_axes, output_rank = _build_proj_equation(key_rank - 1, bound_dims=1, output_dims=2) self._key_dense = EinsumDense( einsum_equation, - output_shape=_get_output_shape( - output_rank - 1, [self._num_heads, self._key_dim] - ), + output_shape=_get_output_shape(output_rank - 1, [self._num_heads, self._key_dim]), bias_axes=bias_axes if self._use_bias else None, name='key', - **self._get_common_kwargs_for_sublayer() + **self._get_common_kwargs_for_sublayer(), ) self._key_dense.build(key_shape) - einsum_equation, bias_axes, output_rank = _build_proj_equation( - value_rank - 1, bound_dims=1, output_dims=2 - ) + einsum_equation, bias_axes, output_rank = _build_proj_equation(value_rank - 1, bound_dims=1, output_dims=2) self._value_dense = EinsumDense( einsum_equation, - output_shape=_get_output_shape( - output_rank - 1, [self._num_heads, self._value_dim] - ), + output_shape=_get_output_shape(output_rank - 1, [self._num_heads, self._value_dim]), bias_axes=bias_axes if self._use_bias else None, name='value', - **self._get_common_kwargs_for_sublayer() + **self._get_common_kwargs_for_sublayer(), ) self._value_dense.build(value_shape) # Builds the attention computations for multi-head dot product @@ -242,9 +211,7 @@ def build(self, input_shape): self._get_common_kwargs_for_sublayer(), 'attention_output', ) - output_dense_input_shape = list( - self._query_dense.compute_output_shape(query_shape) - ) + output_dense_input_shape = list(self._query_dense.compute_output_shape(query_shape)) output_dense_input_shape[-1] = self._value_dim self._output_dense.build(tuple(output_dense_input_shape)) self.built = True @@ -278,12 +245,8 @@ def _get_common_kwargs_for_sublayer(self): # Create new clone of kernel/bias initializer, so that we don't reuse # the initializer instance, which could lead to same init value since # initializer is stateless. - kernel_initializer = self._kernel_initializer.__class__.from_config( - self._kernel_initializer.get_config() - ) - bias_initializer = self._bias_initializer.__class__.from_config( - self._bias_initializer.get_config() - ) + kernel_initializer = self._kernel_initializer.__class__.from_config(self._kernel_initializer.get_config()) + bias_initializer = self._bias_initializer.__class__.from_config(self._bias_initializer.get_config()) common_kwargs['kernel_initializer'] = kernel_initializer common_kwargs['bias_initializer'] = bias_initializer return common_kwargs @@ -315,7 +278,7 @@ def _make_output_dense(self, query_shape, common_kwargs, name=None): output_shape=_get_output_shape(output_rank - 1, output_shape), bias_axes=bias_axes if self._use_bias else None, name=name, - **common_kwargs + **common_kwargs, ) def _build_attention(self, rank): @@ -337,12 +300,8 @@ def _build_attention(self, rank): self._combine_equation, attn_scores_rank, ) = _build_attention_equation(rank, attn_axes=self._attention_axes) - norm_axes = tuple( - range(attn_scores_rank - len(self._attention_axes), attn_scores_rank) - ) - self._softmax = Softmax( - axis=norm_axes - ) if tf.__version__ >= '2.0' else MaskedSoftmax(axis=norm_axes) + norm_axes = tuple(range(attn_scores_rank - len(self._attention_axes), attn_scores_rank)) + self._softmax = Softmax(axis=norm_axes) if tf.__version__ >= '2.0' else MaskedSoftmax(axis=norm_axes) self._dropout_layer = Dropout(rate=self._dropout) self._inverse_sqrt_key_dim = 1.0 / math.sqrt(float(self._key_dim)) @@ -355,14 +314,10 @@ def _masked_softmax(self, attention_scores, attention_mask=None): # key_attention_dims>) mask_expansion_axis = -len(self._attention_axes) * 2 - 1 for _ in range(len(attention_scores.shape) - len(attention_mask.shape)): - attention_mask = tf.expand_dims( - attention_mask, axis=mask_expansion_axis - ) + attention_mask = tf.expand_dims(attention_mask, axis=mask_expansion_axis) return self._softmax(attention_scores, mask=attention_mask) - def _compute_attention( - self, query, key, value, attention_mask=None, training=None - ): + def _compute_attention(self, query, key, value, attention_mask=None, training=None): """Applies Dot-product attention with query, key, value tensors. This function defines the computation inside `call` with projected @@ -387,9 +342,7 @@ def _compute_attention( # Note: Applying scalar multiply at the smaller end of einsum improves # XLA performance, but may introduce slight numeric differences in # the Transformer attention head. - query = tf.multiply( - query, tf.cast(self._inverse_sqrt_key_dim, query.dtype) - ) + query = tf.multiply(query, tf.cast(self._inverse_sqrt_key_dim, query.dtype)) # Take the dot product between "query" and "key" to get the raw # attention scores. @@ -400,22 +353,16 @@ def _compute_attention( # This is actually dropping out entire tokens to attend to, which might # seem a bit unusual, but is taken from the original Transformer paper. if self.dropout: - final_attn_scores = self._dropout_layer( - attention_scores, training=training - ) + final_attn_scores = self._dropout_layer(attention_scores, training=training) else: final_attn_scores = attention_scores # `context_layer` = [B, T, N, H] - attention_output = tf.einsum( - self._combine_equation, final_attn_scores, value - ) + attention_output = tf.einsum(self._combine_equation, final_attn_scores, value) return attention_output, attention_scores def call(self, inputs, mask=None, training=None, **kwargs): - assert isinstance( - inputs, (tuple, list) - ), 'inputs of MultiHeadAttention must be a list' + assert isinstance(inputs, (tuple, list)), 'inputs of MultiHeadAttention must be a list' query, value, key = (list(inputs) + [None] * 2)[:3] if key is None: key = value @@ -448,9 +395,7 @@ def call(self, inputs, mask=None, training=None, **kwargs): # `value` = [B, S, N, H] value = self._value_dense(value) - attention_output, attention_scores = self._compute_attention( - query, key, value, attention_mask, training - ) + attention_output, attention_scores = self._compute_attention(query, key, value, attention_mask, training) attention_output = self._output_dense(attention_output) if self._return_attention_scores: return attention_output, attention_scores @@ -516,10 +461,7 @@ def _compute_attention_mask( auto_mask = mask if auto_mask is None else auto_mask & mask if auto_mask is not None: # merge attention_mask & automatic mask, to shape [B, T, S] - attention_mask = ( - auto_mask if attention_mask is None else - tf.cast(attention_mask, tf.bool) & auto_mask - ) + attention_mask = auto_mask if attention_mask is None else tf.cast(attention_mask, tf.bool) & auto_mask return attention_mask def _compute_causal_mask(self, query, value=None): @@ -566,12 +508,11 @@ def compute_output_shape(self, input_shape): raise ValueError( 'The last dimension of `query_shape` and `value_shape` ' 'must be equal, but are {query_last_dim}, {value_last_dim}. ' - 'Received: query_shape={query_shape}, value_shape={value_shape}'. - format( + 'Received: query_shape={query_shape}, value_shape={value_shape}'.format( query_shape=query_shape, value_shape=value_shape, query_last_dim=query_shape[-1], - value_last_dim=value_shape[-1] + value_last_dim=value_shape[-1], ) ) @@ -579,9 +520,7 @@ def compute_output_shape(self, input_shape): raise ValueError( 'All dimensions of `value` and `key`, except the last one, ' 'must be equal. Received: value_shape={value_shape} and ' - 'key_shape={key_shape}'.format( - key_shape=key_shape, value_shape=value_shape - ) + 'key_shape={key_shape}'.format(key_shape=key_shape, value_shape=value_shape) ) if self._output_shape: @@ -630,7 +569,7 @@ def _build_attention_equation(rank, attn_axes): for i in range(rank): target_notation += _index_to_einsum_variable(i) # `batch_dims` includes the head dim. - batch_dims = tuple(np.delete(range(rank), attn_axes + (rank - 1, ))) + batch_dims = tuple(np.delete(range(rank), attn_axes + (rank - 1,))) letter_offset = rank source_notation = '' for i in range(rank): @@ -641,9 +580,9 @@ def _build_attention_equation(rank, attn_axes): letter_offset += 1 product_notation = ''.join( - [target_notation[i] - for i in batch_dims] + [target_notation[i] for i in attn_axes] + - [source_notation[i] for i in attn_axes] + [target_notation[i] for i in batch_dims] + + [target_notation[i] for i in attn_axes] + + [source_notation[i] for i in attn_axes] ) dot_product_equation = '%s,%s->%s' % ( source_notation, diff --git a/easy_rec/python/layers/keras/multi_task.py b/easy_rec/python/layers/keras/multi_task.py index 133d53558..6433e7183 100644 --- a/easy_rec/python/layers/keras/multi_task.py +++ b/easy_rec/python/layers/keras/multi_task.py @@ -27,10 +27,7 @@ def __init__(self, params, name='MMoE', reuse=None, **kwargs): expert_params = Parameter.make_from_pb(params.expert_mlp) expert_params.l2_regularizer = params.l2_regularizer self._has_experts = True - self._experts = [ - MLP(expert_params, 'expert_%d' % i, reuse=reuse) - for i in range(self._num_expert) - ] + self._experts = [MLP(expert_params, 'expert_%d' % i, reuse=reuse) for i in range(self._num_expert)] else: self._has_experts = False @@ -40,7 +37,7 @@ def __init__(self, params, name='MMoE', reuse=None, **kwargs): self._num_expert, activation='softmax', name='gate_%d' % task_id, - kernel_regularizer=params.l2_regularizer + kernel_regularizer=params.l2_regularizer, ) self._gates.append(dense) @@ -49,9 +46,7 @@ def call(self, inputs, training=None, **kwargs): logging.warning('num_expert of MMoE layer `%s` is 0' % self.name) return inputs if self._has_experts: - expert_fea_list = [ - expert(inputs, training=training) for expert in self._experts - ] + expert_fea_list = [expert(inputs, training=training) for expert in self._experts] else: expert_fea_list = inputs experts_fea = tf.stack(expert_fea_list, axis=1) diff --git a/easy_rec/python/layers/keras/numerical_embedding.py b/easy_rec/python/layers/keras/numerical_embedding.py index 031878595..42cb28738 100644 --- a/easy_rec/python/layers/keras/numerical_embedding.py +++ b/easy_rec/python/layers/keras/numerical_embedding.py @@ -34,9 +34,7 @@ custom_ops = tf.load_op_library(custom_op_path) logging.info('load custom op from %s succeed' % custom_op_path) except Exception as ex: - logging.warning( - 'load custom op from %s failed: %s' % (custom_op_path, str(ex)) - ) + logging.warning('load custom op from %s failed: %s' % (custom_op_path, str(ex))) custom_ops = None @@ -69,9 +67,7 @@ class NLinear(Layer): assert m(x).shape == (batch_size, n_features, d_embedding_out) """ - def __init__( - self, n_tokens, d_in, d_out, bias=True, name='nd_linear', **kwargs - ): + def __init__(self, n_tokens, d_in, d_out, bias=True, name='nd_linear', **kwargs): """Init with input shapes. Args: @@ -82,29 +78,18 @@ def __init__( name: layer name """ super(NLinear, self).__init__(name=name, **kwargs) - self.weight = self.add_weight( - 'weights', [1, n_tokens, d_in, d_out], dtype=tf.float32 - ) + self.weight = self.add_weight('weights', [1, n_tokens, d_in, d_out], dtype=tf.float32) if bias: initializer = tf.constant_initializer(0.0) - self.bias = self.add_weight( - 'bias', [1, n_tokens, d_out], - dtype=tf.float32, - initializer=initializer - ) + self.bias = self.add_weight('bias', [1, n_tokens, d_out], dtype=tf.float32, initializer=initializer) else: self.bias = None def call(self, x, **kwargs): if x.shape.ndims != 3: - raise ValueError( - 'The input must have three dimensions (batch_size, n_tokens, d_embedding)' - ) + raise ValueError('The input must have three dimensions (batch_size, n_tokens, d_embedding)') if x.shape[2] != self.weight.shape[2]: - raise ValueError( - 'invalid input embedding dimension %d, expect %d' % - (int(x.shape[2]), int(self.weight.shape[2])) - ) + raise ValueError('invalid input embedding dimension %d, expect %d' % (int(x.shape[2]), int(self.weight.shape[2]))) x = x[..., None] * self.weight # [B, N, D, D_out] x = tf.reduce_sum(x, axis=-2) # [B, N, D_out] @@ -146,9 +131,7 @@ def __init__(self, params, name='periodic_embedding', reuse=None, **kwargs): self.initializer = tf.random_normal_initializer(stddev=sigma) self.add_linear_layer = params.get_or_default('add_linear_layer', True) self.linear_activation = params.get_or_default('linear_activation', 'relu') - self.output_tensor_list = params.get_or_default( - 'output_tensor_list', False - ) + self.output_tensor_list = params.get_or_default('output_tensor_list', False) self.output_3d_tensor = params.get_or_default('output_3d_tensor', False) def build(self, input_shape): @@ -164,14 +147,14 @@ def build(self, input_shape): 'coefficients', shape=[1, self.num_features, emb_dim], partitioner=partitioner, - initializer=self.initializer + initializer=self.initializer, ) if self.add_linear_layer: self.linear = NLinear( self.num_features, self.embedding_dim, self.embedding_dim, - name='nd_linear' + name='nd_linear', ) super(PeriodicEmbedding, self).build(input_shape) @@ -209,9 +192,7 @@ def __init__(self, params, name='auto_dis_embedding', reuse=None, **kwargs): self.num_bins = int(params.num_bins) self.temperature = params.temperature self.keep_prob = params.get_or_default('keep_prob', 0.8) - self.output_tensor_list = params.get_or_default( - 'output_tensor_list', False - ) + self.output_tensor_list = params.get_or_default('output_tensor_list', False) self.output_3d_tensor = params.get_or_default('output_3d_tensor', False) def build(self, input_shape): @@ -225,17 +206,17 @@ def build(self, input_shape): self.meta_emb = self.add_weight( 'meta_embedding', shape=[self.num_features, self.num_bins, self.emb_dim], - partitioner=partitioner + partitioner=partitioner, ) self.proj_w = self.add_weight( 'project_w', shape=[1, self.num_features, self.num_bins], - partitioner=partitioner + partitioner=partitioner, ) self.proj_mat = self.add_weight( 'project_mat', shape=[self.num_features, self.num_bins, self.num_bins], - partitioner=partitioner + partitioner=partitioner, ) super(AutoDisEmbedding, self).build(input_shape) @@ -255,9 +236,7 @@ def call(self, inputs, **kwargs): # emb = tf.matmul(x_hat[:, :, None, :], meta_emb) # [B, N, 1, D] # emb = tf.squeeze(emb, axis=2) # [B, N, D] emb = tf.einsum('bnk,nkd->bnd', x_hat, self.meta_emb) - output = tf.reshape( - emb, [-1, self.emb_dim * self.num_features] - ) # [B, N*D] + output = tf.reshape(emb, [-1, self.emb_dim * self.num_features]) # [B, N*D] if self.output_tensor_list: return output, tf.unstack(emb, axis=1) @@ -286,15 +265,16 @@ def __init__(self, params, name='nary_dis_embedding', reuse=None, **kwargs): self.multiplier = params.get_or_default('multiplier', 1.0) self.intra_ary_pooling = params.get_or_default('intra_ary_pooling', 'sum') self.output_3d_tensor = params.get_or_default('output_3d_tensor', False) - self.output_tensor_list = params.get_or_default( - 'output_tensor_list', False - ) + self.output_tensor_list = params.get_or_default('output_tensor_list', False) logging.info( - '{} carries: {}, lengths: {}, vocab_size: {}, intra_ary: {}, replicas: {}, multiplier: {}' - .format( - self.name, ','.join(map(str, self.carries)), - ','.join(map(str, self.lengths)), self.vocab_size, - self.intra_ary_pooling, self.num_replicas, self.multiplier + '{} carries: {}, lengths: {}, vocab_size: {}, intra_ary: {}, replicas: {}, multiplier: {}'.format( + self.name, + ','.join(map(str, self.carries)), + ','.join(map(str, self.lengths)), + self.vocab_size, + self.intra_ary_pooling, + self.num_replicas, + self.multiplier, ) ) @@ -304,9 +284,7 @@ def max_length(carry): return (math.floor(bits) + 1) * carry def build(self, input_shape): - assert isinstance( - input_shape, tf.TensorShape - ), 'NaryDisEmbedding only takes 1 input' + assert isinstance(input_shape, tf.TensorShape), 'NaryDisEmbedding only takes 1 input' self.num_features = int(input_shape[-1]) logging.info('%s has %d input features', self.name, self.num_features) vocab_size = self.num_features * self.vocab_size @@ -315,9 +293,7 @@ def build(self, input_shape): partitioner = None if num_ps > 0: partitioner = tf.fixed_size_partitioner(num_shards=num_ps) - self.embedding_table = self.add_weight( - 'embed_table', shape=[vocab_size, emb_dim], partitioner=partitioner - ) + self.embedding_table = self.add_weight('embed_table', shape=[vocab_size, emb_dim], partitioner=partitioner) super(NaryDisEmbedding, self).build(input_shape) def call(self, inputs, **kwargs): @@ -352,9 +328,7 @@ def call(self, inputs, **kwargs): segment_ids = repeat(tf.range(total_length), repeats=splits) embedding = tf.math.segment_mean(embedding, segment_ids) else: - raise ValueError( - 'Unsupported intra ary pooling method %s' % self.intra_ary_pooling - ) + raise ValueError('Unsupported intra ary pooling method %s' % self.intra_ary_pooling) # B: batch size # N: num features # C: num carries diff --git a/easy_rec/python/layers/keras/ppnet.py b/easy_rec/python/layers/keras/ppnet.py index 29c063954..0d9f73227 100644 --- a/easy_rec/python/layers/keras/ppnet.py +++ b/easy_rec/python/layers/keras/ppnet.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Convenience blocks for building models.""" + import logging import tensorflow as tf @@ -13,10 +14,7 @@ class GateNN(tf.keras.layers.Layer): - - def __init__( - self, params, output_units=None, name='gate_nn', reuse=None, **kwargs - ): + def __init__(self, params, output_units=None, name='gate_nn', reuse=None, **kwargs): super(GateNN, self).__init__(name=name, **kwargs) output_dim = output_units if output_units is not None else params.output_dim hidden_dim = params.get_or_default('hidden_dim', output_dim) @@ -26,11 +24,7 @@ def __init__( dropout_rate = params.get_or_default('dropout_rate', 0.0) self._sub_layers = [] - dense = tf.keras.layers.Dense( - units=hidden_dim, - use_bias=not do_batch_norm, - kernel_initializer=initializer - ) + dense = tf.keras.layers.Dense(units=hidden_dim, use_bias=not do_batch_norm, kernel_initializer=initializer) self._sub_layers.append(dense) if do_batch_norm: @@ -51,7 +45,7 @@ def __init__( activation='sigmoid', use_bias=not do_batch_norm, kernel_initializer=initializer, - name='weight' + name='weight', ) self._sub_layers.append(dense) self._sub_layers.append(lambda x: x * 2) @@ -100,9 +94,18 @@ def __init__(self, params, name='ppnet', reuse=None, **kwargs): units = list(params.hidden_units) logging.info( 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' - ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' % ( - name, units, dropout_rate, activation, use_bn, use_final_bn, - final_activation, use_bias, initializer, use_bn_after_act + ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' + % ( + name, + units, + dropout_rate, + activation, + use_bn, + use_final_bn, + final_activation, + use_bias, + initializer, + use_bn_after_act, ) ) assert len(units) > 0, 'MLP(%s) takes at least one hidden units' % name @@ -117,24 +120,34 @@ def __init__(self, params, name='ppnet', reuse=None, **kwargs): name = 'layer_%d' % i drop_rate = dropout_rate[i] if i < num_dropout else 0.0 self.add_rich_layer( - num_units, use_bn, drop_rate, activation, initializer, use_bias, - use_bn_after_act, name, params.l2_regularizer - ) - self._sub_layers.append( - GateNN(gate_params, num_units, 'gate_%d' % (i + 1)) + num_units, + use_bn, + drop_rate, + activation, + initializer, + use_bias, + use_bn_after_act, + name, + params.l2_regularizer, ) + self._sub_layers.append(GateNN(gate_params, num_units, 'gate_%d' % (i + 1))) n = len(units) - 1 drop_rate = dropout_rate[n] if num_dropout > n else 0.0 name = 'layer_%d' % n self.add_rich_layer( - units[-1], use_final_bn, drop_rate, final_activation, initializer, - use_final_bias, use_bn_after_act, name, params.l2_regularizer + units[-1], + use_final_bn, + drop_rate, + final_activation, + initializer, + use_final_bias, + use_bn_after_act, + name, + params.l2_regularizer, ) if mode == 'lazy': - self._sub_layers.append( - GateNN(gate_params, units[-1], 'gate_%d' % (n + 1)) - ) + self._sub_layers.append(GateNN(gate_params, units[-1], 'gate_%d' % (n + 1))) def add_rich_layer( self, @@ -146,7 +159,7 @@ def add_rich_layer( use_bias, use_bn_after_activation, name, - l2_reg=None + l2_reg=None, ): act_layer = activation_layer(activation, name='%s/act' % name) if use_bn and not use_bn_after_activation: @@ -155,12 +168,10 @@ def add_rich_layer( use_bias=use_bias, kernel_initializer=initializer, kernel_regularizer=l2_reg, - name='%s/dense' % name + name='%s/dense' % name, ) self._sub_layers.append(dense) - bn = tf.keras.layers.BatchNormalization( - name='%s/bn' % name, trainable=True - ) + bn = tf.keras.layers.BatchNormalization(name='%s/bn' % name, trainable=True) self._sub_layers.append(bn) self._sub_layers.append(act_layer) else: @@ -169,7 +180,7 @@ def add_rich_layer( use_bias=use_bias, kernel_initializer=initializer, kernel_regularizer=l2_reg, - name='%s/dense' % name + name='%s/dense' % name, ) self._sub_layers.append(dense) self._sub_layers.append(act_layer) diff --git a/easy_rec/python/layers/keras/transformer.py b/easy_rec/python/layers/keras/transformer.py index a30eb45f3..32f0027bc 100644 --- a/easy_rec/python/layers/keras/transformer.py +++ b/easy_rec/python/layers/keras/transformer.py @@ -27,9 +27,7 @@ def __init__(self, params, name='transformer_block', reuse=None, **kwargs): mha_cfg = seq_encoder_pb2.MultiHeadAttention() mha_cfg.num_heads = num_heads mha_cfg.key_dim = d_model // num_heads - mha_cfg.dropout = params.get_or_default( - 'attention_probs_dropout_prob', 0.0 - ) + mha_cfg.dropout = params.get_or_default('attention_probs_dropout_prob', 0.0) mha_cfg.return_attention_scores = False args = Parameter.make_from_pb(mha_cfg) self.mha = MultiHeadAttention(args, 'multi_head_attn') @@ -66,14 +64,11 @@ def positional_encoding(length, depth): depths = np.arange(depth)[np.newaxis, :] / depth # (1, depth) angle_rates = 1 / (10000**depths) # (1, depth) angle_rads = positions * angle_rates # (pos, depth) - pos_encoding = np.concatenate( - [np.sin(angle_rads), np.cos(angle_rads)], axis=-1 - ) + pos_encoding = np.concatenate([np.sin(angle_rads), np.cos(angle_rads)], axis=-1) return tf.cast(pos_encoding, dtype=tf.float32) class PositionalEmbedding(Layer): - def __init__(self, vocab_size, d_model, max_position, name='pos_embedding'): super(PositionalEmbedding, self).__init__(name=name) self.d_model = d_model @@ -105,17 +100,11 @@ def __init__(self, params, name='transformer_encoder', reuse=None, **kwargs): max_position = params.get_or_default('max_position_embeddings', 512) num_layers = params.get_or_default('num_hidden_layers', 1) vocab_size = params.vocab_size - logging.info( - 'vocab size of TransformerEncoder(%s) is %d', name, vocab_size - ) - self.output_all = params.get_or_default( - 'output_all_token_embeddings', True - ) + logging.info('vocab size of TransformerEncoder(%s) is %d', name, vocab_size) + self.output_all = params.get_or_default('output_all_token_embeddings', True) self.pos_encoding = PositionalEmbedding(vocab_size, d_model, max_position) self.dropout = Dropout(dropout_rate) - self.enc_layers = [ - TransformerBlock(params, 'layer_%d' % i) for i in range(num_layers) - ] + self.enc_layers = [TransformerBlock(params, 'layer_%d' % i) for i in range(num_layers)] self._vocab_size = vocab_size self._max_position = max_position @@ -139,7 +128,6 @@ def call(self, inputs, training=None, **kwargs): class TextEncoder(Layer): - def __init__(self, params, name='text_encoder', reuse=None, **kwargs): super(TextEncoder, self).__init__(name=name, **kwargs) self.separator = params.get_or_default('separator', ' ') @@ -154,7 +142,7 @@ def __init__(self, params, name='text_encoder', reuse=None, **kwargs): self.vocab = tf.feature_column.categorical_column_with_vocabulary_file( 'tokens', vocabulary_file=vocab_file, - default_value=self.default_token_id + default_value=self.default_token_id, ) logging.info('vocab file of TextEncoder(%s) is %s', name, vocab_file) trans_params.vocab_size = self.vocab.vocabulary_size @@ -176,25 +164,23 @@ def call(self, inputs, training=None, **kwargs): if self.vocab is not None: features = {'tokens': tokens} token_ids = self.vocab._transform_feature(features) - token_ids = tf.sparse.to_dense( - token_ids, default_value=self.default_token_id, name='token_ids' - ) + token_ids = tf.sparse.to_dense(token_ids, default_value=self.default_token_id, name='token_ids') length = tf.shape(token_ids)[-1] token_ids = tf.cond( - tf.less_equal(length, self.encoder.max_position), lambda: token_ids, - lambda: tf.slice(token_ids, [0, 0], [-1, self.encoder.max_position]) + tf.less_equal(length, self.encoder.max_position), + lambda: token_ids, + lambda: tf.slice(token_ids, [0, 0], [-1, self.encoder.max_position]), ) mask = tf.not_equal(token_ids, self.default_token_id, name='mask') else: tokens = tf.sparse.to_dense(tokens, default_value='') length = tf.shape(tokens)[-1] tokens = tf.cond( - tf.less_equal(length, self.encoder.max_position), lambda: tokens, - lambda: tf.slice(tokens, [0, 0], [-1, self.encoder.max_position]) - ) - token_ids = tf.string_to_hash_bucket_fast( - tokens, self.encoder.vocab_size, name='token_ids' + tf.less_equal(length, self.encoder.max_position), + lambda: tokens, + lambda: tf.slice(tokens, [0, 0], [-1, self.encoder.max_position]), ) + token_ids = tf.string_to_hash_bucket_fast(tokens, self.encoder.vocab_size, name='token_ids') mask = tf.not_equal(tokens, '', name='mask') encoding = self.encoder([token_ids, mask], training=training) diff --git a/easy_rec/python/layers/layer_norm.py b/easy_rec/python/layers/layer_norm.py index c758ec06d..441648681 100644 --- a/easy_rec/python/layers/layer_norm.py +++ b/easy_rec/python/layers/layer_norm.py @@ -17,14 +17,16 @@ def __init__(self, hidden_size, params={}): def build(self, _): self.scale = tf.get_variable( - 'layer_norm_scale', [self.hidden_size], + 'layer_norm_scale', + [self.hidden_size], initializer=tf.keras.initializers.Ones(), - dtype=tf.float32 + dtype=tf.float32, ) self.bias = tf.get_variable( - 'layer_norm_bias', [self.hidden_size], + 'layer_norm_bias', + [self.hidden_size], initializer=tf.keras.initializers.Zeros(), - dtype=tf.float32 + dtype=tf.float32, ) self.built = True diff --git a/easy_rec/python/layers/mmoe.py b/easy_rec/python/layers/mmoe.py index 42e96a64c..fcaaa9342 100644 --- a/easy_rec/python/layers/mmoe.py +++ b/easy_rec/python/layers/mmoe.py @@ -11,7 +11,6 @@ class MMOE: - def __init__( self, expert_dnn_config, @@ -19,7 +18,7 @@ def __init__( num_task, num_expert=None, name='mmoe', - is_training=False + is_training=False, ): """Initializes a `DNN` Layer. @@ -37,8 +36,9 @@ def __init__( self._expert_dnn_configs = expert_dnn_config self._num_expert = len(expert_dnn_config) else: - assert num_expert is not None and num_expert > 0, \ - 'param `num_expert` must be large than zero, when expert_dnn_config is not a list' + assert ( + num_expert is not None and num_expert > 0 + ), 'param `num_expert` must be large than zero, when expert_dnn_config is not a list' self._expert_dnn_configs = [expert_dnn_config] * num_expert self._num_expert = num_expert logging.info('num_expert: {0}'.format(self._num_expert)) @@ -57,7 +57,7 @@ def gate(self, unit, deep_fea, name): inputs=deep_fea, units=unit, kernel_regularizer=self._l2_reg, - name='%s/dnn' % name + name='%s/dnn' % name, ) fea = tf.nn.softmax(fea, axis=1) return fea @@ -70,7 +70,7 @@ def __call__(self, deep_fea): expert_dnn_config, self._l2_reg, name='%s/expert_%d' % (self._name, expert_id), - is_training=self._is_training + is_training=self._is_training, ) expert_fea = expert_dnn(deep_fea) expert_fea_list.append(expert_fea) @@ -78,9 +78,7 @@ def __call__(self, deep_fea): task_input_list = [] for task_id in range(self._num_task): - gate = self.gate( - self._num_expert, deep_fea, name='%s/gate_%d' % (self._name, task_id) - ) + gate = self.gate(self._num_expert, deep_fea, name='%s/gate_%d' % (self._name, task_id)) gate = tf.expand_dims(gate, -1) task_input = tf.multiply(experts_fea, gate) task_input = tf.reduce_sum(task_input, axis=1) diff --git a/easy_rec/python/layers/multihead_attention.py b/easy_rec/python/layers/multihead_attention.py index b68429164..09b8fea4f 100644 --- a/easy_rec/python/layers/multihead_attention.py +++ b/easy_rec/python/layers/multihead_attention.py @@ -7,7 +7,6 @@ class MultiHeadAttention: - def __init__(self, head_num, head_size, l2_reg, use_res=False, name=''): """Initializes a `MultiHeadAttention` Layer. @@ -37,17 +36,11 @@ def _split_multihead_qkv(self, q, k, v): k: Key matrix of shape [bs, head_num, feature_num, head_size]. v: Value matrix of shape [bs, head_num, feature_num, head_size]. """ - reshaped_q = tf.reshape( - q, shape=[-1, q.shape[1], self._head_num, self._head_size] - ) + reshaped_q = tf.reshape(q, shape=[-1, q.shape[1], self._head_num, self._head_size]) q = tf.transpose(reshaped_q, perm=[0, 2, 1, 3]) - reshaped_k = tf.reshape( - k, shape=[-1, k.shape[1], self._head_num, self._head_size] - ) + reshaped_k = tf.reshape(k, shape=[-1, k.shape[1], self._head_num, self._head_size]) k = tf.transpose(reshaped_k, perm=[0, 2, 1, 3]) - reshaped_v = tf.reshape( - v, shape=[-1, v.shape[1], self._head_num, self._head_size] - ) + reshaped_v = tf.reshape(v, shape=[-1, v.shape[1], self._head_num, self._head_size]) v = tf.transpose(reshaped_v, perm=[0, 2, 1, 3]) return q, k, v @@ -64,8 +57,7 @@ def _scaled_dot_product_attention(self, q, k, v): k: Key matrix of shape [bs, head_num, feature_num, head_size]. v: Value matrix of shape [bs, head_num, feature_num, head_size]. """ - product = tf.linalg.matmul(a=q, b=k, - transpose_b=True) / (self._head_size**-0.5) + product = tf.linalg.matmul(a=q, b=k, transpose_b=True) / (self._head_size**-0.5) weights = tf.nn.softmax(product) out = tf.linalg.matmul(weights, v) return out @@ -88,21 +80,21 @@ def _compute_qkv(self, q, k, v): self._head_num * self._head_size, use_bias=False, kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'query') + name='%s/%s/dnn' % (self._name, 'query'), ) k = tf.layers.dense( k, self._head_num * self._head_size, use_bias=False, kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'key') + name='%s/%s/dnn' % (self._name, 'key'), ) v = tf.layers.dense( v, self._head_num * self._head_size, use_bias=False, kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'value') + name='%s/%s/dnn' % (self._name, 'value'), ) return q, k, v @@ -129,8 +121,9 @@ def _multi_head_attention(self, attention_input): out: The output of multi head attention layer, has a shape of [bs, feature_num, head_num * head_size]. """ if isinstance(attention_input, list): - assert len(attention_input) == 3 or len(attention_input) == 1, \ - 'If the input of multi_head_attention is a list, the length must be 1 or 3.' + assert ( + len(attention_input) == 3 or len(attention_input) == 1 + ), 'If the input of multi_head_attention is a list, the length must be 1 or 3.' if len(attention_input) == 3: ori_q = attention_input[0] @@ -156,7 +149,7 @@ def _multi_head_attention(self, attention_input): out.shape[2], use_bias=False, kernel_regularizer=self._l2_reg, - name='%s/dnn' % (self._name) + name='%s/dnn' % (self._name), ) res_out = tf.nn.relu(out + W_0_x) return res_out diff --git a/easy_rec/python/layers/multihead_cross_attention.py b/easy_rec/python/layers/multihead_cross_attention.py index a616ab346..6aeff64cf 100644 --- a/easy_rec/python/layers/multihead_cross_attention.py +++ b/easy_rec/python/layers/multihead_cross_attention.py @@ -52,7 +52,7 @@ def attention_layer( batch_size=None, from_seq_length=None, to_seq_length=None, - reuse=None + reuse=None, ): """Performs multi-headed attention from `from_tensor` to `to_tensor`. @@ -108,12 +108,8 @@ def attention_layer( ValueError: Any of the arguments or tensor shapes are invalid. """ - def transpose_for_scores( - input_tensor, batch_size, num_attention_heads, seq_length, width - ): - output_tensor = tf.reshape( - input_tensor, [batch_size, seq_length, num_attention_heads, width] - ) + def transpose_for_scores(input_tensor, batch_size, num_attention_heads, seq_length, width): + output_tensor = tf.reshape(input_tensor, [batch_size, seq_length, num_attention_heads, width]) output_tensor = tf.transpose(output_tensor, [0, 2, 1, 3]) return output_tensor @@ -122,18 +118,14 @@ def transpose_for_scores( to_shape = get_shape_list(to_tensor, expected_rank=[2, 3]) if len(from_shape) != len(to_shape): - raise ValueError( - 'The rank of `from_tensor` must match the rank of `to_tensor`.' - ) + raise ValueError('The rank of `from_tensor` must match the rank of `to_tensor`.') if len(from_shape) == 3: batch_size = from_shape[0] from_seq_length = from_shape[1] to_seq_length = to_shape[1] elif len(from_shape) == 2: - if ( - batch_size is None or from_seq_length is None or to_seq_length is None - ): + if batch_size is None or from_seq_length is None or to_seq_length is None: raise ValueError( 'When passing in rank 2 tensors to attention_layer, the values ' 'for `batch_size`, `from_seq_length`, and `to_seq_length` ' @@ -157,7 +149,7 @@ def transpose_for_scores( activation=query_act, name='query', kernel_initializer=create_initializer(initializer_range), - reuse=reuse + reuse=reuse, ) # `key_layer` = [B*T, N*H] @@ -167,7 +159,7 @@ def transpose_for_scores( activation=key_act, name='key', kernel_initializer=create_initializer(initializer_range), - reuse=reuse + reuse=reuse, ) # `value_layer` = [B*T, N*H] @@ -177,27 +169,20 @@ def transpose_for_scores( activation=value_act, name='value', kernel_initializer=create_initializer(initializer_range), - reuse=reuse + reuse=reuse, ) # `query_layer` = [B, N, F, H] - query_layer = transpose_for_scores( - query_layer, batch_size, num_attention_heads, from_seq_length, - size_per_head - ) + query_layer = transpose_for_scores(query_layer, batch_size, num_attention_heads, from_seq_length, size_per_head) # `key_layer` = [B, N, T, H] - key_layer = transpose_for_scores( - key_layer, batch_size, num_attention_heads, to_seq_length, size_per_head - ) + key_layer = transpose_for_scores(key_layer, batch_size, num_attention_heads, to_seq_length, size_per_head) # Take the dot product between "query" and "key" to get the raw # attention scores. # `attention_scores` = [B, N, F, T] attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) - attention_scores = tf.multiply( - attention_scores, 1.0 / math.sqrt(float(size_per_head)) - ) + attention_scores = tf.multiply(attention_scores, 1.0 / math.sqrt(float(size_per_head))) if attention_mask is not None: # `attention_mask` = [B, 1, F, T] @@ -221,10 +206,7 @@ def transpose_for_scores( attention_probs = dropout(attention_probs, attention_probs_dropout_prob) # `value_layer` = [B, T, N, H] - value_layer = tf.reshape( - value_layer, - [batch_size, to_seq_length, num_attention_heads, size_per_head] - ) + value_layer = tf.reshape(value_layer, [batch_size, to_seq_length, num_attention_heads, size_per_head]) # `value_layer` = [B, N, T, H] value_layer = tf.transpose(value_layer, [0, 2, 1, 3]) @@ -239,13 +221,13 @@ def transpose_for_scores( # `context_layer` = [B*F, N*H] context_layer = tf.reshape( context_layer, - [batch_size * from_seq_length, num_attention_heads * size_per_head] + [batch_size * from_seq_length, num_attention_heads * size_per_head], ) else: # `context_layer` = [B, F, N*H] context_layer = tf.reshape( context_layer, - [batch_size, from_seq_length, num_attention_heads * size_per_head] + [batch_size, from_seq_length, num_attention_heads * size_per_head], ) return context_layer @@ -263,7 +245,7 @@ def transformer_encoder( attention_probs_dropout_prob=0.1, initializer_range=0.02, reuse=None, - name='transformer' + name='transformer', ): """Multi-headed, multi-layer Transformer from "Attention is All You Need". @@ -312,10 +294,7 @@ def transformer_encoder( # The Transformer performs sum residuals on all layers so the input needs # to be the same as the hidden size. if input_width != hidden_size: - raise ValueError( - 'The width of the input tensor (%d) != hidden size (%d)' % - (input_width, hidden_size) - ) + raise ValueError('The width of the input tensor (%d) != hidden size (%d)' % (input_width, hidden_size)) # We keep the representation as a 2D tensor to avoid re-shaping it back and # forth from a 3D tensor to a 2D tensor. Re-shapes are normally free on @@ -342,7 +321,7 @@ def transformer_encoder( batch_size=batch_size, from_seq_length=seq_length, to_seq_length=seq_length, - reuse=reuse + reuse=reuse, ) # Run a linear projection of `hidden_size` then add a residual @@ -351,7 +330,7 @@ def transformer_encoder( attention_output = tf.layers.dense( attention_output, hidden_size, - kernel_initializer=create_initializer(initializer_range) + kernel_initializer=create_initializer(initializer_range), ) attention_output = dropout(attention_output, hidden_dropout_prob) attention_output = layer_norm(attention_output + layer_input) @@ -362,7 +341,7 @@ def transformer_encoder( attention_output, intermediate_size, activation=intermediate_act_fn, - kernel_initializer=create_initializer(initializer_range) + kernel_initializer=create_initializer(initializer_range), ) # Down-project back to `hidden_size` then add the residual. @@ -370,7 +349,7 @@ def transformer_encoder( layer_output = tf.layers.dense( intermediate_output, hidden_size, - kernel_initializer=create_initializer(initializer_range) + kernel_initializer=create_initializer(initializer_range), ) layer_output = dropout(layer_output, hidden_dropout_prob) layer_output = layer_norm(layer_output + attention_output) @@ -392,7 +371,7 @@ def cross_attention_block( hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, initializer_range=0.02, - name='' + name='', ): """Multi-headed cross attention block. @@ -447,7 +426,7 @@ def cross_attention_block( do_return_2d_tensor=True, batch_size=batch_size, from_seq_length=from_seq_length, - to_seq_length=to_seq_length + to_seq_length=to_seq_length, ) with tf.variable_scope('self'): @@ -463,14 +442,12 @@ def cross_attention_block( do_return_2d_tensor=True, batch_size=batch_size, from_seq_length=from_seq_length, - to_seq_length=from_seq_length + to_seq_length=from_seq_length, ) with tf.variable_scope('output'): attention_output = dropout(self_attention_output, hidden_dropout_prob) - attention_output = layer_norm( - attention_output + cross_attention_output - ) + attention_output = layer_norm(attention_output + cross_attention_output) # The activation is only applied to the "intermediate" hidden layer. with tf.variable_scope('intermediate'): @@ -478,7 +455,7 @@ def cross_attention_block( attention_output, intermediate_size, activation=tf.nn.relu, - kernel_initializer=create_initializer(initializer_range) + kernel_initializer=create_initializer(initializer_range), ) # Down-project back to `hidden_size` then add the residual. @@ -486,16 +463,13 @@ def cross_attention_block( layer_output = tf.layers.dense( intermediate_output, num_attention_heads * size_per_head, - kernel_initializer=create_initializer(initializer_range) + kernel_initializer=create_initializer(initializer_range), ) layer_output = dropout(layer_output, hidden_dropout_prob) # [batch_size * from_seq_length, num_attention_heads * size_per_head] layer_output = layer_norm(layer_output + attention_output) - final_output = reshape_from_matrix( - layer_output, - [batch_size, from_seq_length, num_attention_heads * size_per_head] - ) + final_output = reshape_from_matrix(layer_output, [batch_size, from_seq_length, num_attention_heads * size_per_head]) return final_output # [batch_size, from_seq_length, num_attention_heads * size_per_head] @@ -513,7 +487,7 @@ def cross_attention_tower( hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, initializer_range=0.02, - name='' + name='', ): """Multi-headed, multi layer cross attention block. @@ -555,27 +529,19 @@ def cross_attention_tower( left_attention_mask = None if left_input_mask is not None: - left_attention_mask = create_attention_mask_from_input_mask( - left_tensor, left_attention_mask - ) + left_attention_mask = create_attention_mask_from_input_mask(left_tensor, left_attention_mask) left_2_right_attention_mask = None if right_input_mask is not None: - left_2_right_attention_mask = create_attention_mask_from_input_mask( - left_tensor, right_input_mask - ) + left_2_right_attention_mask = create_attention_mask_from_input_mask(left_tensor, right_input_mask) right_attention_mask = None if right_input_mask is not None: - right_attention_mask = create_attention_mask_from_input_mask( - right_tensor, right_input_mask - ) + right_attention_mask = create_attention_mask_from_input_mask(right_tensor, right_input_mask) right_2_left_attention_mask = None if left_input_mask is not None: - right_2_left_attention_mask = create_attention_mask_from_input_mask( - right_tensor, left_input_mask - ) + right_2_left_attention_mask = create_attention_mask_from_input_mask(right_tensor, left_input_mask) prev_left_output = left_tensor prev_right_output = right_tensor @@ -592,7 +558,7 @@ def cross_attention_tower( self_attention_mask=left_attention_mask, attention_probs_dropout_prob=attention_probs_dropout_prob, initializer_range=initializer_range, - name='%sleft_to_right_' % name + name='%sleft_to_right_' % name, ) right_output = cross_attention_block( prev_right_output, @@ -606,7 +572,7 @@ def cross_attention_tower( self_attention_mask=right_attention_mask, attention_probs_dropout_prob=attention_probs_dropout_prob, initializer_range=initializer_range, - name='%sright_to_left_' % name + name='%sright_to_left_' % name, ) prev_left_output = left_output prev_right_output = right_output @@ -615,19 +581,14 @@ def cross_attention_tower( def layer_norm(input_tensor, name=None): """Run layer normalization on the last dimension of the tensor.""" - return tf_layer_norm( - inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name - ) + return tf_layer_norm(inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name) def reshape_to_matrix(input_tensor): """Reshapes a >= rank 2 tensor to a rank 2 tensor (i.e., a matrix).""" ndims = input_tensor.shape.ndims if ndims < 2: - raise ValueError( - 'Input tensor must have at least rank 2. Shape = %s' % - (input_tensor.shape) - ) + raise ValueError('Input tensor must have at least rank 2. Shape = %s' % (input_tensor.shape)) if ndims == 2: return input_tensor @@ -666,18 +627,14 @@ def create_attention_mask_from_input_mask(from_tensor, to_mask): to_shape = get_shape_list(to_mask, expected_rank=2) to_seq_length = to_shape[1] - to_mask = tf.cast( - tf.reshape(to_mask, [batch_size, 1, to_seq_length]), tf.float32 - ) + to_mask = tf.cast(tf.reshape(to_mask, [batch_size, 1, to_seq_length]), tf.float32) # We don't assume that `from_tensor` is a mask (although it could be). We # don't actually care if we attend *from* padding tokens (only *to* padding) # tokens so we create a tensor of all ones. # # `broadcast_ones` = [batch_size, from_seq_length, 1] - broadcast_ones = tf.ones( - shape=tf.stack([batch_size, from_seq_length, 1]), dtype=tf.float32 - ) + broadcast_ones = tf.ones(shape=tf.stack([batch_size, from_seq_length, 1]), dtype=tf.float32) # Here we broadcast along two dimensions to create the mask. mask = broadcast_ones * to_mask @@ -697,7 +654,7 @@ def embedding_postprocessor( reuse_position_embedding=None, initializer_range=0.02, max_position_embeddings=512, - dropout_prob=0.1 + dropout_prob=0.1, ): """Performs various post-processing on a word embedding tensor. @@ -737,36 +694,29 @@ def embedding_postprocessor( if use_token_type: if token_type_ids is None: - raise ValueError( - '`token_type_ids` must be specified if' - '`use_token_type` is True.' - ) + raise ValueError('`token_type_ids` must be specified if' '`use_token_type` is True.') with tf.variable_scope('token_type', reuse=reuse_token_type): token_type_table = tf.get_variable( name=token_type_embedding_name, shape=[token_type_vocab_size, width], - initializer=create_initializer(initializer_range) + initializer=create_initializer(initializer_range), ) # This vocab will be small so we always do one-hot here, since it is always # faster for a small vocabulary. flat_token_type_ids = tf.reshape(token_type_ids, [-1]) one_hot_ids = tf.one_hot(flat_token_type_ids, depth=token_type_vocab_size) token_type_embeddings = tf.matmul(one_hot_ids, token_type_table) - token_type_embeddings = tf.reshape( - token_type_embeddings, [batch_size, seq_length, width] - ) + token_type_embeddings = tf.reshape(token_type_embeddings, [batch_size, seq_length, width]) output += token_type_embeddings if use_position_embeddings: assert_op = tf.assert_less_equal(seq_length, max_position_embeddings) with tf.control_dependencies([assert_op]): - with tf.variable_scope( - 'position_embedding', reuse=reuse_position_embedding - ): + with tf.variable_scope('position_embedding', reuse=reuse_position_embedding): full_position_embeddings = tf.get_variable( name=position_embedding_name, shape=[max_position_embeddings, width], - initializer=create_initializer(initializer_range) + initializer=create_initializer(initializer_range), ) # Since the position embedding table is a learned variable, we create it # using a (long) sequence length `max_position_embeddings`. The actual @@ -777,9 +727,7 @@ def embedding_postprocessor( # for position [0, 1, 2, ..., max_position_embeddings-1], and the current # sequence has positions [0, 1, 2, ... seq_length-1], so we can just # perform a slice. - position_embeddings = tf.slice( - full_position_embeddings, [0, 0], [seq_length, -1] - ) + position_embeddings = tf.slice(full_position_embeddings, [0, 0], [seq_length, -1]) num_dims = len(output.shape.as_list()) # Only the last two dimensions are relevant (`seq_length` and `width`), so @@ -789,9 +737,7 @@ def embedding_postprocessor( for _ in range(num_dims - 2): position_broadcast_shape.append(1) position_broadcast_shape.extend([seq_length, width]) - position_embeddings = tf.reshape( - position_embeddings, position_broadcast_shape - ) + position_embeddings = tf.reshape(position_embeddings, position_broadcast_shape) output += position_embeddings output = layer_norm_and_dropout(output, dropout_prob) diff --git a/easy_rec/python/layers/senet.py b/easy_rec/python/layers/senet.py index 8add70e0b..91b27d4ec 100644 --- a/easy_rec/python/layers/senet.py +++ b/easy_rec/python/layers/senet.py @@ -21,14 +21,7 @@ class SENet: name: str, name of the layer. """ - def __init__( - self, - num_fields, - num_squeeze_group, - reduction_ratio, - l2_reg, - name='SENet' - ): + def __init__(self, num_fields, num_squeeze_group, reduction_ratio, l2_reg, name='SENet'): self.num_fields = num_fields self.num_squeeze_group = num_squeeze_group self.reduction_ratio = reduction_ratio @@ -45,9 +38,7 @@ def __call__(self, inputs): for input in inputs: emb_size += int(input.shape[-1]) - group_embs = [ - tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs - ] + group_embs = [tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs] squeezed = [] for emb in group_embs: @@ -60,14 +51,14 @@ def __call__(self, inputs): units=reduction_size, kernel_regularizer=self._l2_reg, activation='relu', - name='%s/reduce' % self._name + name='%s/reduce' % self._name, ) excited_weights = tf.layers.dense( inputs=reduced, units=emb_size, kernel_initializer='glorot_normal', - name='%s/excite' % self._name + name='%s/excite' % self._name, ) # Re-weight diff --git a/easy_rec/python/layers/seq_input_layer.py b/easy_rec/python/layers/seq_input_layer.py index fd5d92519..35adc27fe 100644 --- a/easy_rec/python/layers/seq_input_layer.py +++ b/easy_rec/python/layers/seq_input_layer.py @@ -17,22 +17,16 @@ class SeqInputLayer(object): - def __init__( self, feature_configs, feature_groups_config, embedding_regularizer=None, - ev_params=None + ev_params=None, ): - self._feature_groups_config = { - x.group_name: x - for x in feature_groups_config - } + self._feature_groups_config = {x.group_name: x for x in feature_groups_config} wide_and_deep_dict = self.get_wide_deep_dict() - self._fc_parser = FeatureColumnParser( - feature_configs, wide_and_deep_dict, ev_params=ev_params - ) + self._fc_parser = FeatureColumnParser(feature_configs, wide_and_deep_dict, ev_params=ev_params) self._embedding_regularizer = embedding_regularizer def __call__( @@ -41,7 +35,7 @@ def __call__( group_name, feature_name_to_output_tensors={}, allow_key_search=True, - scope_name=None + scope_name=None, ): feature_column_dict = self._fc_parser.deep_columns feature_column_dict.update(self._fc_parser.sequence_columns) @@ -61,9 +55,7 @@ def _seq_embed_summary_name(input_name): if scope_name is None: scope_name = group_name # name_scope is specified to avoid adding _1 _2 after scope_name - with variable_scope.variable_scope( - scope_name, reuse=variable_scope.AUTO_REUSE - ), ops.name_scope(scope_name + '/'): + with variable_scope.variable_scope(scope_name, reuse=variable_scope.AUTO_REUSE), ops.name_scope(scope_name + '/'): key_tensors = [] hist_tensors = [] check_op_list = [] @@ -74,59 +66,47 @@ def _seq_embed_summary_name(input_name): ): qfc = feature_column_dict[key] with variable_scope.variable_scope(qfc._var_scope_name): - tmp_key_tensor = feature_column_dict[key]._get_dense_tensor( - builder - ) + tmp_key_tensor = feature_column_dict[key]._get_dense_tensor(builder) regularizers.apply_regularization( - self._embedding_regularizer, weights_list=[tmp_key_tensor] + self._embedding_regularizer, + weights_list=[tmp_key_tensor], ) key_tensors.append(tmp_key_tensor) elif feature_name_to_output_tensors[key] is None: - assert feature_name_to_output_tensors[ - key - ] is not None, 'When allow_key_search is False, key: %s should defined in same feature group.' % key + assert feature_name_to_output_tensors[key] is not None, ( + 'When allow_key_search is False, key: %s should defined in same feature group.' % key + ) else: key_tensors.append(feature_name_to_output_tensors[key]) if tf_summary: for key_tensor in key_tensors: - tf.summary.histogram( - _seq_embed_summary_name(key_tensor.name), key_tensor - ) + tf.summary.histogram(_seq_embed_summary_name(key_tensor.name), key_tensor) cur_hist_seqs = [] for hist_seq in x.hist_seq: seq_fc = feature_column_dict[hist_seq] with variable_scope.variable_scope(seq_fc._var_scope_name): - cur_hist_seqs.append( - feature_column_dict[hist_seq]. - _get_sequence_dense_tensor(builder) - ) + cur_hist_seqs.append(feature_column_dict[hist_seq]._get_sequence_dense_tensor(builder)) hist_tensors.extend(cur_hist_seqs) aux_hist_emb_list = [] for aux_hist_seq in x.aux_hist_seq: seq_fc = feature_column_dict[aux_hist_seq] with variable_scope.variable_scope(seq_fc._var_scope_name): - aux_hist_embedding, _ = feature_column_dict[ - aux_hist_seq]._get_sequence_dense_tensor(builder) + aux_hist_embedding, _ = feature_column_dict[aux_hist_seq]._get_sequence_dense_tensor(builder) aux_hist_emb_list.append(aux_hist_embedding) if tf_summary: for hist_embed, hist_seq_len in hist_tensors: - tf.summary.histogram( - _seq_embed_summary_name(hist_embed.name), hist_embed - ) - tf.summary.histogram( - _seq_embed_summary_name(hist_seq_len.name), hist_seq_len - ) + tf.summary.histogram(_seq_embed_summary_name(hist_embed.name), hist_embed) + tf.summary.histogram(_seq_embed_summary_name(hist_seq_len.name), hist_seq_len) for idx in range(1, len(cur_hist_seqs)): check_op = tf.assert_equal( cur_hist_seqs[0][1], cur_hist_seqs[idx][1], - message= - 'SequenceFeature Error: The size of %s not equal to the size of %s.' - % (x.hist_seq[idx], x.hist_seq[0]) + message='SequenceFeature Error: The size of %s not equal to the size of %s.' + % (x.hist_seq[idx], x.hist_seq[0]), ) check_op_list.append(check_op) @@ -135,7 +115,7 @@ def _seq_embed_summary_name(input_name): 'key': tf.concat(key_tensors, axis=-1), 'hist_seq_emb': tf.concat([x[0] for x in hist_tensors], axis=-1), 'hist_seq_len': hist_tensors[0][1], - 'aux_hist_seq_emb_list': aux_hist_emb_list + 'aux_hist_seq_emb_list': aux_hist_emb_list, } return features diff --git a/easy_rec/python/layers/sequence_feature_layer.py b/easy_rec/python/layers/sequence_feature_layer.py index b40d7bc08..58495c28b 100644 --- a/easy_rec/python/layers/sequence_feature_layer.py +++ b/easy_rec/python/layers/sequence_feature_layer.py @@ -13,7 +13,6 @@ class SequenceFeatureLayer(object): - def __init__( self, feature_configs, @@ -22,7 +21,7 @@ def __init__( embedding_regularizer=None, kernel_regularizer=None, is_training=False, - is_predicting=False + is_predicting=False, ): self._seq_feature_groups_config = [] for x in feature_groups_config: @@ -34,7 +33,7 @@ def __init__( feature_configs, self._seq_feature_groups_config, embedding_regularizer=embedding_regularizer, - ev_params=ev_params + ev_params=ev_params, ) self._embedding_regularizer = embedding_regularizer self._kernel_regularizer = kernel_regularizer @@ -48,11 +47,14 @@ def negative_sampler_target_attention( concat_features, name, need_key_feature=True, - allow_key_transform=False + allow_key_transform=False, ): - cur_id, hist_id_col, seq_len, aux_hist_emb_list = deep_fea[ - 'key'], deep_fea['hist_seq_emb'], deep_fea['hist_seq_len'], deep_fea[ - 'aux_hist_seq_emb_list'] + cur_id, hist_id_col, seq_len, aux_hist_emb_list = ( + deep_fea['key'], + deep_fea['hist_seq_emb'], + deep_fea['hist_seq_len'], + deep_fea['aux_hist_seq_emb_list'], + ) seq_max_len = tf.shape(hist_id_col)[1] seq_emb_dim = hist_id_col.shape[2] @@ -64,36 +66,26 @@ def negative_sampler_target_attention( cur_id = tf.concat( [ pos_feature[:, tf.newaxis, :], - tf.tile(neg_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1]) + tf.tile(neg_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1]), ], - axis=1 + axis=1, ) # noqa: E126 neg_num_add_1 = tf.shape(cur_id)[1] - hist_id_col_tmp = tf.tile( - hist_id_col[:, :, :], multiples=[1, neg_num_add_1, 1] - ) - hist_id_col = tf.reshape( - hist_id_col_tmp, [batch_size * neg_num_add_1, seq_max_len, seq_emb_dim] - ) + hist_id_col_tmp = tf.tile(hist_id_col[:, :, :], multiples=[1, neg_num_add_1, 1]) + hist_id_col = tf.reshape(hist_id_col_tmp, [batch_size * neg_num_add_1, seq_max_len, seq_emb_dim]) - concat_features = tf.tile( - concat_features[:, tf.newaxis, :], multiples=[1, neg_num_add_1, 1] - ) + concat_features = tf.tile(concat_features[:, tf.newaxis, :], multiples=[1, neg_num_add_1, 1]) seq_len = tf.tile(seq_len, multiples=[neg_num_add_1]) if allow_key_transform and (cur_id_dim != seq_emb_dim): - cur_id = tf.layers.dense( - cur_id, seq_emb_dim, name='sequence_key_transform_layer' - ) + cur_id = tf.layers.dense(cur_id, seq_emb_dim, name='sequence_key_transform_layer') cur_ids = tf.tile(cur_id, [1, 1, seq_max_len]) - cur_ids = tf.reshape( - cur_ids, tf.shape(hist_id_col) - ) # (B * neg_num_add_1, seq_max_len, seq_emb_dim) + cur_ids = tf.reshape(cur_ids, tf.shape(hist_id_col)) # (B * neg_num_add_1, seq_max_len, seq_emb_dim) din_net = tf.concat( [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1 + axis=-1, ) # (B * neg_num_add_1, seq_max_len, seq_emb_dim*4) din_layer = dnn.DNN( @@ -102,23 +94,19 @@ def negative_sampler_target_attention( name, self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True + last_layer_no_batch_norm=True, ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) seq_len = tf.expand_dims(seq_len, 1) mask = tf.sequence_mask(seq_len) - padding = tf.ones_like(scores) * (-2**32 + 1) - scores = tf.where( - mask, scores, padding - ) # [B*neg_num_add_1, 1, seq_max_len] + padding = tf.ones_like(scores) * (-(2**32) + 1) + scores = tf.where(mask, scores, padding) # [B*neg_num_add_1, 1, seq_max_len] # Scale scores = tf.nn.softmax(scores) # (B * neg_num_add_1, 1, seq_max_len) - hist_din_emb = tf.matmul( - scores, hist_id_col - ) # [B * neg_num_add_1, 1, seq_emb_dim] + hist_din_emb = tf.matmul(scores, hist_id_col) # [B * neg_num_add_1, 1, seq_emb_dim] hist_din_emb = tf.reshape( hist_din_emb, [batch_size, neg_num_add_1, seq_emb_dim] ) # [B * neg_num_add_1, seq_emb_dim] @@ -141,11 +129,14 @@ def target_attention( name, need_key_feature=True, allow_key_transform=False, - transform_dnn=False + transform_dnn=False, ): - cur_id, hist_id_col, seq_len, aux_hist_emb_list = deep_fea[ - 'key'], deep_fea['hist_seq_emb'], deep_fea['hist_seq_len'], deep_fea[ - 'aux_hist_seq_emb_list'] + cur_id, hist_id_col, seq_len, aux_hist_emb_list = ( + deep_fea['key'], + deep_fea['hist_seq_emb'], + deep_fea['hist_seq_len'], + deep_fea['aux_hist_seq_emb_list'], + ) seq_max_len = tf.shape(hist_id_col)[1] seq_emb_dim = hist_id_col.shape[2] @@ -158,20 +149,16 @@ def target_attention( cur_key_layer_name = 'sequence_key_transform_layer_' + name cur_id = tf.layers.dense(cur_id, seq_emb_dim, name=cur_key_layer_name) cur_fea_layer_name = 'sequence_fea_transform_layer_' + name - hist_id_col = tf.layers.dense( - hist_id_col, seq_emb_dim, name=cur_fea_layer_name - ) + hist_id_col = tf.layers.dense(hist_id_col, seq_emb_dim, name=cur_fea_layer_name) else: - cur_id = cur_id[:tf.shape(hist_id_col)[0], ...] # for negative sampler + cur_id = cur_id[: tf.shape(hist_id_col)[0], ...] # for negative sampler cur_ids = tf.tile(cur_id, [1, seq_max_len]) - cur_ids = tf.reshape( - cur_ids, tf.shape(hist_id_col) - ) # (B, seq_max_len, seq_emb_dim) + cur_ids = tf.reshape(cur_ids, tf.shape(hist_id_col)) # (B, seq_max_len, seq_emb_dim) din_net = tf.concat( [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1 + axis=-1, ) # (B, seq_max_len, seq_emb_dim*4) din_layer = dnn.DNN( @@ -180,22 +167,20 @@ def target_attention( name, self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True + last_layer_no_batch_norm=True, ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) seq_len = tf.expand_dims(seq_len, 1) mask = tf.sequence_mask(seq_len) - padding = tf.ones_like(scores) * (-2**32 + 1) + padding = tf.ones_like(scores) * (-(2**32) + 1) scores = tf.where(mask, scores, padding) # [B, 1, seq_max_len] # Scale scores = tf.nn.softmax(scores) # (B, 1, seq_max_len) hist_din_emb = tf.matmul(scores, hist_id_col) # [B, 1, seq_emb_dim] - hist_din_emb = tf.reshape( - hist_din_emb, [-1, seq_emb_dim] - ) # [B, seq_emb_dim] + hist_din_emb = tf.reshape(hist_din_emb, [-1, seq_emb_dim]) # [B, seq_emb_dim] if len(aux_hist_emb_list) > 0: all_hist_dim_emb = [hist_din_emb] for hist_col in aux_hist_emb_list: @@ -216,7 +201,7 @@ def __call__( all_seq_att_map_config, feature_name_to_output_tensors=None, negative_sampler=False, - scope_name=None + scope_name=None, ): logging.info('use sequence feature layer.') all_seq_fea = [] @@ -230,29 +215,26 @@ def __call__( place_on_cpu = os.getenv('place_embedding_on_cpu') place_on_cpu = eval(place_on_cpu) if place_on_cpu else False - with conditional( - self._is_predicting and place_on_cpu, ops.device('/CPU:0') - ): + with conditional(self._is_predicting and place_on_cpu, ops.device('/CPU:0')): seq_features = self._seq_input_layer( - features, group_name, feature_name_to_output_tensors, - allow_key_search, scope_name + features, + group_name, + feature_name_to_output_tensors, + allow_key_search, + scope_name, ) # apply regularization for sequence feature key in seq_input_layer. - regularizers.apply_regularization( - self._embedding_regularizer, - weights_list=[seq_features['hist_seq_emb']] - ) + regularizers.apply_regularization(self._embedding_regularizer, weights_list=[seq_features['hist_seq_emb']]) seq_dnn_config = None if seq_att_map_config.HasField('seq_dnn'): seq_dnn_config = seq_att_map_config.seq_dnn else: - logging.info( - 'seq_dnn not set in seq_att_groups, will use default settings' - ) + logging.info('seq_dnn not set in seq_att_groups, will use default settings') # If not set seq_dnn, will use default settings from easy_rec.python.protos.dnn_pb2 import DNN + seq_dnn_config = DNN() seq_dnn_config.hidden_units.extend([128, 64, 32, 1]) cur_target_attention_name = 'seq_dnn' + group_name @@ -263,7 +245,7 @@ def __call__( concat_features, name=cur_target_attention_name, need_key_feature=need_key_feature, - allow_key_transform=allow_key_transform + allow_key_transform=allow_key_transform, ) else: seq_fea = self.target_attention( @@ -272,7 +254,7 @@ def __call__( name=cur_target_attention_name, need_key_feature=need_key_feature, allow_key_transform=allow_key_transform, - transform_dnn=transform_dnn + transform_dnn=transform_dnn, ) all_seq_fea.append(seq_fea) return concat_features, all_seq_fea diff --git a/easy_rec/python/layers/uniter.py b/easy_rec/python/layers/uniter.py index e8f119dea..4d98bc62b 100644 --- a/easy_rec/python/layers/uniter.py +++ b/easy_rec/python/layers/uniter.py @@ -17,9 +17,7 @@ class Uniter(object): https://arxiv.org/abs/1909.11740 """ - def __init__( - self, model_config, feature_configs, features, uniter_config, input_layer - ): + def __init__(self, model_config, feature_configs, features, uniter_config, input_layer): self._model_config = uniter_config tower_num = 0 self._img_features = None @@ -32,9 +30,7 @@ def __init__( tower_num += 1 self._txt_seq_features = None if input_layer.has_group('text'): - self._txt_seq_features, _, _ = input_layer( - features, 'text', is_combine=False - ) + self._txt_seq_features, _, _ = input_layer(features, 'text', is_combine=False) tower_num += 1 self._use_token_type = True if tower_num > 1 else False self._other_features = None @@ -51,21 +47,17 @@ def __init__( if fea_group.group_name == 'general': self._general_feature_num = len(fea_group.feature_names) general_feature_names = set(fea_group.feature_names) - assert self._general_feature_num == len(general_feature_names), ( - 'there are duplicate features in `general` feature group' - ) + assert self._general_feature_num == len( + general_feature_names + ), 'there are duplicate features in `general` feature group' elif fea_group.group_name == 'image': self._img_feature_num = len(fea_group.feature_names) img_feature_names = set(fea_group.feature_names) - assert self._img_feature_num == len(img_feature_names), ( - 'there are duplicate features in `image` feature group' - ) + assert self._img_feature_num == len(img_feature_names), 'there are duplicate features in `image` feature group' elif fea_group.group_name == 'text': self._txt_feature_num = len(fea_group.feature_names) txt_feature_names = set(fea_group.feature_names) - assert self._txt_feature_num == len(txt_feature_names), ( - 'there are duplicate features in `text` feature group' - ) + assert self._txt_feature_num == len(txt_feature_names), 'there are duplicate features in `text` feature group' if self._txt_feature_num > 1 or self._img_feature_num > 1: self._use_token_type = True @@ -91,44 +83,37 @@ def __init__( txt_fea_emb_dim_list.append(feature_config.embedding_dim) if feature_config.HasField('max_seq_len'): assert feature_config.max_seq_len > 0, ( - 'feature config `max_seq_len` must be greater than 0 for feature: ' - + fea_name + 'feature config `max_seq_len` must be greater than 0 for feature: ' + fea_name ) if feature_config.max_seq_len > max_seq_len: max_seq_len = feature_config.max_seq_len unique_dim_num = len(set(txt_fea_emb_dim_list)) - assert unique_dim_num <= 1 and len( - txt_fea_emb_dim_list - ) == self._txt_feature_num, ( - 'Uniter requires that all `text` feature dimensions must be consistent.' - ) + assert ( + unique_dim_num <= 1 and len(txt_fea_emb_dim_list) == self._txt_feature_num + ), 'Uniter requires that all `text` feature dimensions must be consistent.' unique_dim_num = len(set(img_fea_emb_dim_list)) - assert unique_dim_num <= 1 and len( - img_fea_emb_dim_list - ) == self._img_feature_num, ( - 'Uniter requires that all `image` feature dimensions must be consistent.' - ) + assert ( + unique_dim_num <= 1 and len(img_fea_emb_dim_list) == self._img_feature_num + ), 'Uniter requires that all `image` feature dimensions must be consistent.' unique_dim_num = len(set(general_emb_dim_list)) - assert unique_dim_num <= 1 and len( - general_emb_dim_list - ) == self._general_feature_num, ( - 'Uniter requires that all `general` feature dimensions must be consistent.' - ) + assert ( + unique_dim_num <= 1 and len(general_emb_dim_list) == self._general_feature_num + ), 'Uniter requires that all `general` feature dimensions must be consistent.' if self._txt_feature_num > 0 and uniter_config.use_position_embeddings: - assert uniter_config.max_position_embeddings > 0, ( - 'model config `max_position_embeddings` must be greater than 0. ' - ) + assert ( + uniter_config.max_position_embeddings > 0 + ), 'model config `max_position_embeddings` must be greater than 0. ' assert uniter_config.max_position_embeddings >= max_seq_len, ( - 'model config `max_position_embeddings` must be greater than or equal to the maximum of all feature config ' - '`max_seq_len`, which is %d' % max_seq_len - ) + 'model config `max_position_embeddings` must be greater than' + ' or equal to the maximum of all feature config ' + '`max_seq_len`, which is %d' + ) % max_seq_len self._img_emb_size = img_fea_emb_dim_list[0] if img_fea_emb_dim_list else 0 self._txt_emb_size = txt_fea_emb_dim_list[0] if txt_fea_emb_dim_list else 0 - self._general_emb_size = general_emb_dim_list[ - 0] if general_emb_dim_list else 0 + self._general_emb_size = general_emb_dim_list[0] if general_emb_dim_list else 0 if self._img_features is not None: assert self._img_emb_size > 0, '`image` feature dimensions must be greater than 0, set by `raw_input_dim`' @@ -140,15 +125,9 @@ def text_embeddings(self, token_type_id): general_features = self._general_features if self._general_emb_size != hidden_size: # Run a linear projection of `hidden_size` - general_features = tf.reshape( - general_features, shape=[-1, self._general_emb_size] - ) - general_features = tf.layers.dense( - general_features, hidden_size, name='txt_projection' - ) - general_features = tf.reshape( - general_features, shape=[-1, self._general_feature_num, hidden_size] - ) + general_features = tf.reshape(general_features, shape=[-1, self._general_emb_size]) + general_features = tf.layers.dense(general_features, hidden_size, name='txt_projection') + general_features = tf.reshape(general_features, shape=[-1, self._general_feature_num, hidden_size]) batch_size = tf.shape(general_features)[0] general_features = multihead_cross_attention.embedding_postprocessor( @@ -156,19 +135,17 @@ def text_embeddings(self, token_type_id): use_token_type=self._use_token_type, token_type_ids=tf.ones( shape=tf.stack([batch_size, self._general_feature_num]), - dtype=tf.int32 - ) * token_type_id, + dtype=tf.int32, + ) + * token_type_id, token_type_vocab_size=self._token_type_vocab_size, reuse_token_type=tf.AUTO_REUSE, use_position_embeddings=False, - dropout_prob=self._model_config.hidden_dropout_prob + dropout_prob=self._model_config.hidden_dropout_prob, ) all_txt_features.append(general_features) - mask = tf.ones( - shape=tf.stack([batch_size, self._general_feature_num]), - dtype=tf.int32 - ) + mask = tf.ones(shape=tf.stack([batch_size, self._general_feature_num]), dtype=tf.int32) input_masks.append(mask) if self._txt_seq_features is not None: @@ -183,29 +160,25 @@ def dynamic_mask(x, max_len): batch_size, max_seq_len, emb_size = get_shape_list(seq_fea, 3) if emb_size != hidden_size: seq_fea = tf.reshape(seq_fea, shape=[-1, emb_size]) - seq_fea = tf.layers.dense( - seq_fea, hidden_size, name='txt_seq_projection_%d' % i - ) + seq_fea = tf.layers.dense(seq_fea, hidden_size, name='txt_seq_projection_%d' % i) seq_fea = tf.reshape(seq_fea, shape=[-1, max_seq_len, hidden_size]) seq_fea = multihead_cross_attention.embedding_postprocessor( seq_fea, use_token_type=self._use_token_type, - token_type_ids=tf. - ones(shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * - (i + token_type_id), + token_type_ids=tf.ones(shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * (i + token_type_id), token_type_vocab_size=self._token_type_vocab_size, reuse_token_type=tf.AUTO_REUSE, use_position_embeddings=self._model_config.use_position_embeddings, max_position_embeddings=self._model_config.max_position_embeddings, position_embedding_name='txt_position_embeddings_%d' % i, - dropout_prob=self._model_config.hidden_dropout_prob + dropout_prob=self._model_config.hidden_dropout_prob, ) all_txt_features.append(seq_fea) input_mask = tf.map_fn( fn=lambda t: dynamic_mask(t, max_seq_len), - elems=tf.to_int32(seq_len) + elems=tf.to_int32(seq_len), ) input_masks.append(input_mask) @@ -218,29 +191,21 @@ def image_embeddings(self): image_features = self._img_features if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` - image_features = tf.reshape( - image_features, shape=[-1, self._img_emb_size] - ) - image_features = tf.layers.dense( - image_features, hidden_size, name='img_projection' - ) - image_features = tf.reshape( - image_features, shape=[-1, self._img_feature_num, hidden_size] - ) + image_features = tf.reshape(image_features, shape=[-1, self._img_emb_size]) + image_features = tf.layers.dense(image_features, hidden_size, name='img_projection') + image_features = tf.reshape(image_features, shape=[-1, self._img_feature_num, hidden_size]) batch_size = tf.shape(image_features)[0] img_fea = multihead_cross_attention.embedding_postprocessor( image_features, use_token_type=self._use_token_type, - token_type_ids=tf.zeros( - shape=tf.stack([batch_size, self._img_feature_num]), dtype=tf.int32 - ), + token_type_ids=tf.zeros(shape=tf.stack([batch_size, self._img_feature_num]), dtype=tf.int32), token_type_vocab_size=self._token_type_vocab_size, reuse_token_type=tf.AUTO_REUSE, use_position_embeddings=self._model_config.use_position_embeddings, max_position_embeddings=self._model_config.max_position_embeddings, position_embedding_name='img_position_embeddings', - dropout_prob=self._model_config.hidden_dropout_prob + dropout_prob=self._model_config.hidden_dropout_prob, ) return img_fea @@ -275,9 +240,7 @@ def __call__(self, is_training, *args, **kwargs): if img_fea is not None: all_features.append(img_fea) - mask = tf.ones( - shape=tf.stack([batch_size, self._img_feature_num]), dtype=tf.int32 - ) + mask = tf.ones(shape=tf.stack([batch_size, self._img_feature_num]), dtype=tf.int32) masks.append(mask) if txt_features: @@ -299,10 +262,9 @@ def __call__(self, is_training, *args, **kwargs): intermediate_size=self._model_config.intermediate_size, intermediate_act_fn=hidden_act, hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config. - attention_probs_dropout_prob, + attention_probs_dropout_prob=self._model_config.attention_probs_dropout_prob, initializer_range=self._model_config.initializer_range, - name='uniter' + name='uniter', ) # shape: [batch_size, seq_length, hidden_size] print('attention_fea:', attention_fea.shape) mm_fea = attention_fea[:, 0, :] # [CLS] feature @@ -312,8 +274,10 @@ def __call__(self, is_training, *args, **kwargs): if self._model_config.HasField('other_feature_dnn'): l2_reg = kwargs['l2_reg'] if 'l2_reg' in kwargs else 0 other_dnn_layer = dnn.DNN( - self._model_config.other_feature_dnn, l2_reg, 'other_dnn', - is_training + self._model_config.other_feature_dnn, + l2_reg, + 'other_dnn', + is_training, ) other_fea = other_dnn_layer(self._other_features) else: diff --git a/easy_rec/python/layers/utils.py b/easy_rec/python/layers/utils.py index c2f05fa61..4eedb70bf 100644 --- a/easy_rec/python/layers/utils.py +++ b/easy_rec/python/layers/utils.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== """Common util functions used by layers.""" + from __future__ import absolute_import, division, print_function import json @@ -34,7 +35,7 @@ def _tensor_to_map(tensor): return { 'node_path': tensor.name, 'shape': tensor.shape.as_list() if tensor.shape else None, - 'dtype': tensor.dtype.name + 'dtype': tensor.dtype.name, } @@ -76,15 +77,11 @@ def _process_item(collection_name, name, func): if key in ColumnNameInCollection: idx_found = ColumnNameInCollection[key] if idx_found >= len(col): - raise Exception( - 'Find column name in collection failed: index out of range' - ) + raise Exception('Find column name in collection failed: index out of range') item_found = json.loads(col[idx_found]) if item_found['name'] != name: - raise Exception( - 'Find column name in collection failed: item name not match' - ) + raise Exception('Find column name in collection failed: item name not match') func(item_found) col[idx_found] = json.dumps(item_found) else: @@ -94,7 +91,6 @@ def _process_item(collection_name, name, func): def append_attr_to_collection(collection_name, name, key, value): - def append(item_found): if key not in item_found: item_found[key] = [] @@ -104,7 +100,6 @@ def append(item_found): def update_attr_to_collection(collection_name, attrs): - def update(item_found): item_found.update(attrs) @@ -124,13 +119,7 @@ def unique_name_in_collection(collection_name, name): return unique_name -def gen_embedding_attrs( - column=None, - variable=None, - bucket_size=None, - combiner=None, - is_embedding_var=None -): +def gen_embedding_attrs(column=None, variable=None, bucket_size=None, combiner=None, is_embedding_var=None): attrs = dict() attrs['name'] = column.name attrs['bucket_size'] = bucket_size @@ -142,14 +131,11 @@ def gen_embedding_attrs( attrs['is_embedding_var'] = True attrs['embedding_var_keys'] = variable._shared_name + '-keys' attrs['embedding_var_values'] = variable._shared_name + '-values' - elif (isinstance(variable, variables.PartitionedVariable)) and \ - (isinstance(variable._get_variable_list()[0], kv_variable_ops.EmbeddingVariable)): - attrs['embedding_var_keys'] = [ - v._shared_name + '-keys' for v in variable - ] - attrs['embedding_var_values'] = [ - v._shared_name + '-values' for v in variable - ] + elif (isinstance(variable, variables.PartitionedVariable)) and ( + isinstance(variable._get_variable_list()[0], kv_variable_ops.EmbeddingVariable) + ): + attrs['embedding_var_keys'] = [v._shared_name + '-keys' for v in variable] + attrs['embedding_var_values'] = [v._shared_name + '-values' for v in variable] else: attrs['is_embedding_var'] = False else: @@ -160,10 +146,7 @@ def gen_embedding_attrs( def mark_input_src(name, src_desc): ops.add_to_collection( ops.GraphKeys.RANK_SERVICE_INPUT_SRC, - json.dumps({ - 'name': name, - 'src': src_desc - }) + json.dumps({'name': name, 'src': src_desc}), ) @@ -177,7 +160,6 @@ def is_proto_message(pb_obj, field): class Parameter(object): - def __init__(self, params, is_struct, l2_reg=None): self.params = params self.is_struct = is_struct diff --git a/easy_rec/python/layers/variational_dropout_layer.py b/easy_rec/python/layers/variational_dropout_layer.py index bd11828f6..a965aa5f1 100644 --- a/easy_rec/python/layers/variational_dropout_layer.py +++ b/easy_rec/python/layers/variational_dropout_layer.py @@ -5,8 +5,12 @@ import numpy as np import tensorflow as tf -from easy_rec.python.compat.feature_column.feature_column import _SharedEmbeddingColumn # NOQA -from easy_rec.python.compat.feature_column.feature_column_v2 import EmbeddingColumn # NOQA +from easy_rec.python.compat.feature_column.feature_column import ( # NOQA + _SharedEmbeddingColumn, +) +from easy_rec.python.compat.feature_column.feature_column_v2 import ( # NOQA + EmbeddingColumn, +) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -20,13 +24,7 @@ class VariationalDropoutLayer(object): arXiv: 1712.08645 """ - def __init__( - self, - variational_dropout_config, - features_dimension, - is_training=False, - name='' - ): + def __init__(self, variational_dropout_config, features_dimension, is_training=False, name=''): self._config = variational_dropout_config self.features_dimension = features_dimension self.features_total_dimension = sum(self.features_dimension.values()) @@ -44,11 +42,11 @@ def __init__( name=logit_p_name, shape=self.drop_param_shape, dtype=tf.float32, - initializer=None + initializer=None, ) tf.add_to_collection( 'variational_dropout', - json.dumps([name, list(self.features_dimension.items())]) + json.dumps([name, list(self.features_dimension.items())]), ) def get_lambda(self): @@ -66,9 +64,7 @@ def build_expand_index(self, batch_size): expanded_index = tf.tile(expanded_index, [batch_size, 1]) batch_size_range = tf.range(batch_size) expand_range_axis = tf.expand_dims(batch_size_range, 1) - batch_size_range_expand_dim_len = tf.tile( - expand_range_axis, [1, self.features_total_dimension] - ) + batch_size_range_expand_dim_len = tf.tile(expand_range_axis, [1, self.features_total_dimension]) index_i = tf.reshape(batch_size_range_expand_dim_len, [-1, 1]) expanded_index = tf.concat([index_i, expanded_index], 1) return expanded_index @@ -85,9 +81,7 @@ def sample_noisy_input(self, input): # expand dropout layer expanded_index = self.build_expand_index(batch_size) expanded_p = tf.gather_nd(p, expanded_index) - expanded_p = tf.reshape( - expanded_p, [-1, self.features_total_dimension] - ) + expanded_p = tf.reshape(expanded_p, [-1, self.features_total_dimension]) scaled_input = input * (1 - expanded_p) return scaled_input @@ -113,13 +107,13 @@ def sampled_from_logit_p(self, num_samples): def concrete_dropout_neuron(self, dropout_p, temp=1.0 / 10.0): EPSILON = np.finfo(float).eps - unif_noise = tf.random_uniform( - tf.shape(dropout_p), dtype=tf.float32, seed=None, name='unif_noise' - ) + unif_noise = tf.random_uniform(tf.shape(dropout_p), dtype=tf.float32, seed=None, name='unif_noise') approx = ( - tf.log(dropout_p + EPSILON) - tf.log(1. - dropout_p + EPSILON) + - tf.log(unif_noise + EPSILON) - tf.log(1. - unif_noise + EPSILON) + tf.log(dropout_p + EPSILON) + - tf.log(1.0 - dropout_p + EPSILON) + + tf.log(unif_noise + EPSILON) + - tf.log(1.0 - unif_noise + EPSILON) ) approx_output = tf.sigmoid(approx / temp) @@ -129,13 +123,10 @@ def __call__(self, output_features): batch_size = tf.shape(output_features)[0] noisy_input = self.sample_noisy_input(output_features) dropout_p = tf.sigmoid(self.logit_p) - variational_dropout_penalty = 1. - dropout_p - variational_dropout_penalty_lambda = self.get_lambda( - ) / tf.cast(batch_size, dtype=tf.float32) + variational_dropout_penalty = 1.0 - dropout_p + variational_dropout_penalty_lambda = self.get_lambda() / tf.cast(batch_size, dtype=tf.float32) variational_dropout_loss_sum = variational_dropout_penalty_lambda * tf.reduce_sum( variational_dropout_penalty, axis=0 ) - tf.add_to_collection( - 'variational_dropout_loss', variational_dropout_loss_sum - ) + tf.add_to_collection('variational_dropout_loss', variational_dropout_loss_sum) return noisy_input diff --git a/easy_rec/python/loss/circle_loss.py b/easy_rec/python/loss/circle_loss.py index a03dd4a6a..6e6dc9398 100644 --- a/easy_rec/python/loss/circle_loss.py +++ b/easy_rec/python/loss/circle_loss.py @@ -6,14 +6,7 @@ tf = tf.compat.v1 -def circle_loss( - embeddings, - labels, - sessions=None, - margin=0.25, - gamma=32, - embed_normed=False -): +def circle_loss(embeddings, labels, sessions=None, margin=0.25, gamma=32, embed_normed=False): """Paper: Circle Loss: A Unified Perspective of Pair Similarity Optimization. Link: http://arxiv.org/pdf/2002.10857.pdf @@ -26,12 +19,8 @@ def circle_loss( gamma: parameter of circle loss embed_normed: bool, whether input embeddings l2 normalized """ - norm_embeddings = embeddings if embed_normed else tf.nn.l2_normalize( - embeddings, axis=-1 - ) - pair_wise_cosine_matrix = tf.matmul( - norm_embeddings, norm_embeddings, transpose_b=True - ) + norm_embeddings = embeddings if embed_normed else tf.nn.l2_normalize(embeddings, axis=-1) + pair_wise_cosine_matrix = tf.matmul(norm_embeddings, norm_embeddings, transpose_b=True) positive_mask = get_anchor_positive_triplet_mask(labels, sessions) negative_mask = 1 - positive_mask - tf.eye(tf.shape(labels)[0]) @@ -39,17 +28,11 @@ def circle_loss( delta_p = 1 - margin delta_n = margin - ap = tf.nn.relu( - -tf.stop_gradient(pair_wise_cosine_matrix * positive_mask) + 1 + margin - ) - an = tf.nn.relu( - tf.stop_gradient(pair_wise_cosine_matrix * negative_mask) + margin - ) + ap = tf.nn.relu(-tf.stop_gradient(pair_wise_cosine_matrix * positive_mask) + 1 + margin) + an = tf.nn.relu(tf.stop_gradient(pair_wise_cosine_matrix * negative_mask) + margin) - logit_p = -ap * (pair_wise_cosine_matrix - delta_p - ) * gamma * positive_mask - (1 - positive_mask) * 1e12 - logit_n = an * (pair_wise_cosine_matrix - delta_n - ) * gamma * negative_mask - (1 - negative_mask) * 1e12 + logit_p = -ap * (pair_wise_cosine_matrix - delta_p) * gamma * positive_mask - (1 - positive_mask) * 1e12 + logit_n = an * (pair_wise_cosine_matrix - delta_n) * gamma * negative_mask - (1 - negative_mask) * 1e12 joint_neg_loss = tf.reduce_logsumexp(logit_n, axis=-1) joint_pos_loss = tf.reduce_logsumexp(logit_p, axis=-1) @@ -79,9 +62,7 @@ def get_anchor_positive_triplet_mask(labels, sessions=None): if sessions is None or sessions is labels: class_equal = labels_equal else: - sessions_equal = tf.equal( - tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1) - ) + sessions_equal = tf.equal(tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1)) class_equal = tf.logical_and(sessions_equal, labels_equal) # Combine the three masks diff --git a/easy_rec/python/loss/contrastive_loss.py b/easy_rec/python/loss/contrastive_loss.py index cadc36e3f..2e0c7fd2d 100644 --- a/easy_rec/python/loss/contrastive_loss.py +++ b/easy_rec/python/loss/contrastive_loss.py @@ -30,9 +30,7 @@ def info_nce_loss(query, positive, temperature=0.1): raise ValueError(' must have 2 dimensions.') # Embedding vectors should have same number of components. if query.shape[-1] != positive.shape[-1]: - raise ValueError( - 'Vectors of and should have the same number of components.' - ) + raise ValueError('Vectors of and should have the same number of components.') # Negative keys are implicitly off-diagonal positive keys. @@ -64,12 +62,8 @@ def nce_loss(z_i, z_j, temperature=1.0): N = 2 * batch_size z = tf.concat((z_i, z_j), axis=0) sim = tf.matmul(z, tf.transpose(z)) / temperature - sim_i_j = tf.matrix_diag_part( - tf.slice(sim, [batch_size, 0], [batch_size, batch_size]) - ) - sim_j_i = tf.matrix_diag_part( - tf.slice(sim, [0, batch_size], [batch_size, batch_size]) - ) + sim_i_j = tf.matrix_diag_part(tf.slice(sim, [batch_size, 0], [batch_size, batch_size])) + sim_j_i = tf.matrix_diag_part(tf.slice(sim, [0, batch_size], [batch_size, batch_size])) positive_samples = tf.reshape(tf.concat((sim_i_j, sim_j_i), axis=0), (N, 1)) mask = get_mask_matrix(batch_size) negative_samples = tf.reshape(tf.boolean_mask(sim, mask), (N, -1)) diff --git a/easy_rec/python/loss/f1_reweight_loss.py b/easy_rec/python/loss/f1_reweight_loss.py index ffe83a591..c1884a092 100644 --- a/easy_rec/python/loss/f1_reweight_loss.py +++ b/easy_rec/python/loss/f1_reweight_loss.py @@ -7,9 +7,7 @@ tf = tf.compat.v1 -def f1_reweight_sigmoid_cross_entropy( - labels, logits, beta_square, label_smoothing=0, weights=None -): +def f1_reweight_sigmoid_cross_entropy(labels, logits, beta_square, label_smoothing=0, weights=None): """Refer paper: Adaptive Scaling for Sparse Detection in Information Extraction.""" probs = tf.nn.sigmoid(logits) if len(logits.shape.as_list()) == 1: @@ -25,14 +23,10 @@ def f1_reweight_sigmoid_cross_entropy( tn = batch_size_float - tp neg_weight = tp / (beta_square * num_pos + num_neg - tn + 1e-8) neg_weight_tile = tf.tile(tf.expand_dims(neg_weight, 0), [batch_size, 1]) - final_weights = tf.where( - tf.equal(labels, 1.0), tf.ones_like(labels), neg_weight_tile - ) + final_weights = tf.where(tf.equal(labels, 1.0), tf.ones_like(labels), neg_weight_tile) if weights is not None: weights = tf.cast(weights, tf.float32) if len(weights.shape.as_list()) == 1: weights = tf.expand_dims(weights, -1) final_weights *= weights - return tf.losses.sigmoid_cross_entropy( - labels, logits, final_weights, label_smoothing=label_smoothing - ) + return tf.losses.sigmoid_cross_entropy(labels, logits, final_weights, label_smoothing=label_smoothing) diff --git a/easy_rec/python/loss/focal_loss.py b/easy_rec/python/loss/focal_loss.py index 5037380f5..52a5fefad 100644 --- a/easy_rec/python/loss/focal_loss.py +++ b/easy_rec/python/loss/focal_loss.py @@ -16,7 +16,7 @@ def sigmoid_focal_loss_with_logits( ohem_ratio=1.0, sample_weights=None, label_smoothing=0, - name='' + name='', ): """Implements the focal loss function. @@ -81,16 +81,14 @@ def sigmoid_focal_loss_with_logits( weights *= sample_weights if ohem_ratio == 1.0: - return tf.losses.sigmoid_cross_entropy( - y_true, logits, weights=weights, label_smoothing=label_smoothing - ) + return tf.losses.sigmoid_cross_entropy(y_true, logits, weights=weights, label_smoothing=label_smoothing) losses = tf.losses.sigmoid_cross_entropy( y_true, logits, weights=weights, label_smoothing=label_smoothing, - reduction=tf.losses.Reduction.NONE + reduction=tf.losses.Reduction.NONE, ) k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio) k = tf.to_int32(tf.math.rint(k)) diff --git a/easy_rec/python/loss/jrc_loss.py b/easy_rec/python/loss/jrc_loss.py index 368cc319a..625db1b5c 100644 --- a/easy_rec/python/loss/jrc_loss.py +++ b/easy_rec/python/loss/jrc_loss.py @@ -17,7 +17,7 @@ def jrc_loss( loss_weight_strategy='fixed', sample_weights=1.0, same_label_loss=True, - name='' + name='', ): """Joint Optimization of Ranking and Calibration with Contextualized Hybrid Model. @@ -35,15 +35,9 @@ def jrc_loss( name: the name of loss """ loss_name = name if name else 'jrc_loss' - logging.info( - '[{}] alpha: {}, loss_weight_strategy: {}'.format( - loss_name, alpha, loss_weight_strategy - ) - ) + logging.info('[{}] alpha: {}, loss_weight_strategy: {}'.format(loss_name, alpha, loss_weight_strategy)) - ce_loss = tf.losses.sparse_softmax_cross_entropy( - labels, logits, weights=sample_weights - ) + ce_loss = tf.losses.sparse_softmax_cross_entropy(labels, logits, weights=sample_weights) labels = tf.expand_dims(labels, 1) # [B, 1] labels = tf.concat([1 - labels, labels], axis=1) # [B, 2] @@ -52,9 +46,7 @@ def jrc_loss( # Mask: shape [B, B], mask[i,j]=1 indicates the i-th sample # and j-th sample are in the same context - mask = tf.equal( - tf.expand_dims(session_ids, 1), tf.expand_dims(session_ids, 0) - ) + mask = tf.equal(tf.expand_dims(session_ids, 1), tf.expand_dims(session_ids, 0)) mask = tf.to_float(mask) # Tile logits and label: [B, 2]->[B, B, 2] @@ -80,9 +72,7 @@ def jrc_loss( logging.info('[%s] enable same_label_loss' % loss_name) loss_pos = -tf.reduce_sum(y_pos * tf.nn.log_softmax(l_pos, axis=0), axis=0) loss_neg = -tf.reduce_sum(y_neg * tf.nn.log_softmax(l_neg, axis=0), axis=0) - ge_loss = tf.reduce_mean( - (loss_pos + loss_neg) / tf.reduce_sum(mask, axis=0) - ) + ge_loss = tf.reduce_mean((loss_pos + loss_neg) / tf.reduce_sum(mask, axis=0)) else: logging.info('[%s] disable same_label_loss' % loss_name) diag = tf.one_hot(tf.range(batch_size), batch_size) @@ -113,30 +103,22 @@ def jrc_loss( bern = tf.distributions.Bernoulli(probs=0.5, dtype=tf.float32) weights = bern.sample(2) loss_weight = tf.cond( - tf.equal(tf.reduce_sum(weights), 1), lambda: weights, - lambda: tf.convert_to_tensor([0.5, 0.5]) + tf.equal(tf.reduce_sum(weights), 1), + lambda: weights, + lambda: tf.convert_to_tensor([0.5, 0.5]), ) loss = loss_weight[0] * ce_loss + loss_weight[1] * ge_loss tf.summary.scalar('loss/%s_ce_weight' % loss_name, loss_weight[0]) tf.summary.scalar('loss/%s_ge_weight' % loss_name, loss_weight[1]) elif loss_weight_strategy == 'uncertainty': - uncertainty1 = tf.Variable( - 0, name='%s_ranking_loss_weight' % loss_name, dtype=tf.float32 - ) + uncertainty1 = tf.Variable(0, name='%s_ranking_loss_weight' % loss_name, dtype=tf.float32) tf.summary.scalar('loss/%s_ranking_uncertainty' % loss_name, uncertainty1) - uncertainty2 = tf.Variable( - 0, name='%s_calibration_loss_weight' % loss_name, dtype=tf.float32 - ) - tf.summary.scalar( - 'loss/%s_calibration_uncertainty' % loss_name, uncertainty2 - ) + uncertainty2 = tf.Variable(0, name='%s_calibration_loss_weight' % loss_name, dtype=tf.float32) + tf.summary.scalar('loss/%s_calibration_uncertainty' % loss_name, uncertainty2) loss = tf.exp(-uncertainty1) * ce_loss + 0.5 * uncertainty1 loss += tf.exp(-uncertainty2) * ge_loss + 0.5 * uncertainty2 else: - raise ValueError( - 'Unsupported loss weight strategy `%s` for jrc loss' % - loss_weight_strategy - ) + raise ValueError('Unsupported loss weight strategy `%s` for jrc loss' % loss_weight_strategy) if np.isscalar(sample_weights) and sample_weights != 1.0: return loss * sample_weights return loss diff --git a/easy_rec/python/loss/listwise_loss.py b/easy_rec/python/loss/listwise_loss.py index 3120e9d2e..83bf47ace 100644 --- a/easy_rec/python/loss/listwise_loss.py +++ b/easy_rec/python/loss/listwise_loss.py @@ -34,7 +34,7 @@ def listwise_rank_loss( label_is_logits=False, scale_logits=False, weights=1.0, - name='listwise_loss' + name='listwise_loss', ): r"""Computes listwise softmax cross entropy loss between `labels` and `logits`. @@ -58,25 +58,21 @@ def listwise_rank_loss( name: the name of loss """ loss_name = name if name else 'listwise_rank_loss' - logging.info( - '[{}] temperature: {}, scale logits: {}'.format( - loss_name, temperature, scale_logits - ) - ) + logging.info('[{}] temperature: {}, scale logits: {}'.format(loss_name, temperature, scale_logits)) labels = tf.to_float(labels) if scale_logits: with tf.variable_scope(loss_name): w = tf.get_variable( 'scale_w', dtype=tf.float32, - shape=(1, ), - initializer=tf.ones_initializer() + shape=(1,), + initializer=tf.ones_initializer(), ) b = tf.get_variable( 'scale_b', dtype=tf.float32, - shape=(1, ), - initializer=tf.zeros_initializer() + shape=(1,), + initializer=tf.zeros_initializer(), ) logits = logits * tf.abs(w) + b if temperature != 1.0: @@ -92,7 +88,7 @@ def listwise_rank_loss( losses = tf.map_fn( lambda x: _list_wise_loss(x, labels, logits, session_ids, label_is_logits), sessions, - dtype=tf.float32 + dtype=tf.float32, ) if tf.is_numeric_tensor(weights): logging.error('[%s] use unsupported sample weight' % loss_name) @@ -110,7 +106,7 @@ def listwise_distill_loss( label_clip_max_value=512, scale_logits=False, weights=1.0, - name='listwise_distill_loss' + name='listwise_distill_loss', ): r"""Computes listwise softmax cross entropy loss between `labels` and `logits`. @@ -146,14 +142,14 @@ def listwise_distill_loss( w = tf.get_variable( 'scale_w', dtype=tf.float32, - shape=(1, ), - initializer=tf.ones_initializer() + shape=(1,), + initializer=tf.ones_initializer(), ) b = tf.get_variable( 'scale_b', dtype=tf.float32, - shape=(1, ), - initializer=tf.zeros_initializer() + shape=(1,), + initializer=tf.zeros_initializer(), ) logits = logits * tf.abs(w) + b if temperature != 1.0: @@ -164,7 +160,7 @@ def listwise_distill_loss( losses = tf.map_fn( lambda x: _list_prob_loss(x, labels, logits, session_ids), sessions, - dtype=tf.float32 + dtype=tf.float32, ) if tf.is_numeric_tensor(weights): logging.error('[%s] use unsupported sample weight' % loss_name) diff --git a/easy_rec/python/loss/multi_similarity.py b/easy_rec/python/loss/multi_similarity.py index a625a662a..2b7f55e1e 100644 --- a/easy_rec/python/loss/multi_similarity.py +++ b/easy_rec/python/loss/multi_similarity.py @@ -17,7 +17,7 @@ def ms_loss( lamb=1.0, eps=0.1, ms_mining=False, - embed_normed=False + embed_normed=False, ): """Refer paper: Multi-Similarity Loss with General Pair Weighting for Deep Metric Learning. @@ -35,9 +35,7 @@ def ms_loss( mask_pos = get_anchor_positive_triplet_mask(labels, session_ids) mask_neg = 1 - mask_pos - tf.eye(batch_size) - sim_mat = tf.matmul( - embeddings, embeddings, transpose_a=False, transpose_b=True - ) + sim_mat = tf.matmul(embeddings, embeddings, transpose_a=False, transpose_b=True) sim_mat = tf.maximum(sim_mat, 0.0) pos_mat = tf.multiply(sim_mat, mask_pos) @@ -46,19 +44,13 @@ def ms_loss( if ms_mining: max_val = tf.reduce_max(neg_mat, axis=1, keepdims=True) tmp_max_val = tf.reduce_max(pos_mat, axis=1, keepdims=True) - min_val = tf.reduce_min( - tf.multiply(sim_mat - tmp_max_val, mask_pos), axis=1, keepdims=True - ) + tmp_max_val + min_val = tf.reduce_min(tf.multiply(sim_mat - tmp_max_val, mask_pos), axis=1, keepdims=True) + tmp_max_val max_val = tf.tile(max_val, [1, batch_size]) min_val = tf.tile(min_val, [1, batch_size]) - mask_pos = tf.where( - pos_mat < max_val + eps, mask_pos, tf.zeros_like(mask_pos) - ) - mask_neg = tf.where( - neg_mat > min_val - eps, mask_neg, tf.zeros_like(mask_neg) - ) + mask_pos = tf.where(pos_mat < max_val + eps, mask_pos, tf.zeros_like(mask_pos)) + mask_neg = tf.where(neg_mat > min_val - eps, mask_neg, tf.zeros_like(mask_neg)) pos_exp = tf.exp(-alpha * (pos_mat - lamb)) pos_exp = tf.where(mask_pos > 0.0, pos_exp, tf.zeros_like(pos_exp)) diff --git a/easy_rec/python/loss/pairwise_loss.py b/easy_rec/python/loss/pairwise_loss.py index ee3f4a0e3..3be51eb13 100644 --- a/easy_rec/python/loss/pairwise_loss.py +++ b/easy_rec/python/loss/pairwise_loss.py @@ -12,15 +12,7 @@ tf = tf.compat.v1 -def pairwise_loss( - labels, - logits, - session_ids=None, - margin=0, - temperature=1.0, - weights=1.0, - name='' -): +def pairwise_loss(labels, logits, session_ids=None, margin=0, temperature=1.0, weights=1.0, name=''): """Deprecated Pairwise loss. Also see `pairwise_logistic_loss` below. Args: @@ -37,23 +29,15 @@ def pairwise_loss( 'Please use the new `pairwise_logistic_loss` or `pairwise_focal_loss`' ) loss_name = name if name else 'pairwise_loss' - logging.info( - '[{}] margin: {}, temperature: {}'.format(loss_name, margin, temperature) - ) + logging.info('[{}] margin: {}, temperature: {}'.format(loss_name, margin, temperature)) if temperature != 1.0: logits /= temperature - pairwise_logits = tf.math.subtract( - tf.expand_dims(logits, -1), tf.expand_dims(logits, 0) - ) - margin - pairwise_mask = tf.greater( - tf.expand_dims(labels, -1), tf.expand_dims(labels, 0) - ) + pairwise_logits = tf.math.subtract(tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) - margin + pairwise_mask = tf.greater(tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) - group_equal = tf.equal( - tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0) - ) + group_equal = tf.equal(tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -70,9 +54,7 @@ def pairwise_loss( pairwise_weights = weights pairwise_pseudo_labels = tf.ones_like(pairwise_logits) - loss = tf.losses.sigmoid_cross_entropy( - pairwise_pseudo_labels, pairwise_logits, weights=pairwise_weights - ) + loss = tf.losses.sigmoid_cross_entropy(pairwise_pseudo_labels, pairwise_logits, weights=pairwise_weights) # set rank loss to zero if a batch has no positive sample. # loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss) return loss @@ -88,30 +70,27 @@ def pairwise_focal_loss( ohem_ratio=1.0, temperature=1.0, weights=1.0, - name='' + name='', ): loss_name = name if name else 'pairwise_focal_loss' assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]' logging.info( - '[{}] hinge margin: {}, gamma: {}, alpha: {}, ohem_ratio: {}, temperature: {}' - .format(loss_name, hinge_margin, gamma, alpha, ohem_ratio, temperature) + '[{}] hinge margin: {}, gamma: {}, alpha: {}, ohem_ratio: {}, temperature: {}'.format( + loss_name, hinge_margin, gamma, alpha, ohem_ratio, temperature + ) ) if temperature != 1.0: logits /= temperature pairwise_logits = tf.expand_dims(logits, -1) - tf.expand_dims(logits, 0) - pairwise_mask = tf.greater( - tf.expand_dims(labels, -1), tf.expand_dims(labels, 0) - ) + pairwise_mask = tf.greater(tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) if hinge_margin is not None: hinge_mask = tf.less(pairwise_logits, hinge_margin) pairwise_mask = tf.logical_and(pairwise_mask, hinge_mask) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) - group_equal = tf.equal( - tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0) - ) + group_equal = tf.equal(tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -134,7 +113,7 @@ def pairwise_focal_loss( gamma=gamma, alpha=alpha, ohem_ratio=ohem_ratio, - sample_weights=pairwise_weights + sample_weights=pairwise_weights, ) return loss @@ -148,7 +127,7 @@ def pairwise_logistic_loss( weights=1.0, ohem_ratio=1.0, use_label_margin=False, - name='' + name='', ): r"""Computes pairwise logistic loss between `labels` and `logits`, equivalent to RankNet loss. @@ -174,9 +153,7 @@ def pairwise_logistic_loss( loss_name = name if name else 'pairwise_logistic_loss' assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]' logging.info( - '[{}] hinge margin: {}, ohem_ratio: {}, temperature: {}'.format( - loss_name, hinge_margin, ohem_ratio, temperature - ) + '[{}] hinge margin: {}, ohem_ratio: {}, temperature: {}'.format(loss_name, hinge_margin, ohem_ratio, temperature) ) if temperature != 1.0: @@ -184,24 +161,16 @@ def pairwise_logistic_loss( if use_label_margin: labels /= temperature - pairwise_logits = tf.math.subtract( - tf.expand_dims(logits, -1), tf.expand_dims(logits, 0) - ) + pairwise_logits = tf.math.subtract(tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) if use_label_margin: - pairwise_logits -= tf.math.subtract( - tf.expand_dims(labels, -1), tf.expand_dims(labels, 0) - ) + pairwise_logits -= tf.math.subtract(tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) elif hinge_margin is not None: pairwise_logits -= hinge_margin - pairwise_mask = tf.greater( - tf.expand_dims(labels, -1), tf.expand_dims(labels, 0) - ) + pairwise_mask = tf.greater(tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) - group_equal = tf.equal( - tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0) - ) + group_equal = tf.equal(tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -209,9 +178,7 @@ def pairwise_logistic_loss( tf.summary.scalar('loss/%s_num_of_pairs' % loss_name, num_pair) # The following is the same as log(1 + exp(-pairwise_logits)). - losses = tf.nn.relu(-pairwise_logits) + tf.math.log1p( - tf.exp(-tf.abs(pairwise_logits)) - ) + losses = tf.nn.relu(-pairwise_logits) + tf.math.log1p(tf.exp(-tf.abs(pairwise_logits))) if tf.is_numeric_tensor(weights): logging.info('[%s] use sample weight' % loss_name) @@ -225,9 +192,7 @@ def pairwise_logistic_loss( if ohem_ratio == 1.0: return compute_weighted_loss(losses, pairwise_weights) - losses = compute_weighted_loss( - losses, pairwise_weights, reduction=tf.losses.Reduction.NONE - ) + losses = compute_weighted_loss(losses, pairwise_weights, reduction=tf.losses.Reduction.NONE) k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio) k = tf.to_int32(tf.math.rint(k)) topk = tf.nn.top_k(losses, k) @@ -246,7 +211,7 @@ def pairwise_hinge_loss( label_is_logits=True, use_label_margin=True, use_exponent=False, - name='' + name='', ): r"""Computes pairwise hinge loss between `labels` and `logits`. @@ -273,10 +238,16 @@ def pairwise_hinge_loss( loss_name = name if name else 'pairwise_hinge_loss' assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]' logging.info( - '[{}] margin: {}, ohem_ratio: {}, temperature: {}, use_exponent: {}, label_is_logits: {}, use_label_margin: {}' - .format( - loss_name, margin, ohem_ratio, temperature, use_exponent, - label_is_logits, use_label_margin + ( + '[{}] margin: {}, ohem_ratio: {}, temperature: {}, use_exponent: {},' ' label_is_logits: {}, use_label_margin: {}' + ).format( + loss_name, + margin, + ohem_ratio, + temperature, + use_exponent, + label_is_logits, + use_label_margin, ) ) @@ -288,19 +259,13 @@ def pairwise_hinge_loss( labels = tf.nn.sigmoid(labels) logits = tf.nn.sigmoid(labels) - pairwise_logits = tf.math.subtract( - tf.expand_dims(logits, -1), tf.expand_dims(logits, 0) - ) - pairwise_labels = tf.math.subtract( - tf.expand_dims(labels, -1), tf.expand_dims(labels, 0) - ) + pairwise_logits = tf.math.subtract(tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) + pairwise_labels = tf.math.subtract(tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) pairwise_mask = tf.greater(pairwise_labels, 0) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) - group_equal = tf.equal( - tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0) - ) + group_equal = tf.equal(tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -331,9 +296,7 @@ def pairwise_hinge_loss( if ohem_ratio == 1.0: return compute_weighted_loss(losses, pairwise_weights) - losses = compute_weighted_loss( - losses, pairwise_weights, reduction=tf.losses.Reduction.NONE - ) + losses = compute_weighted_loss(losses, pairwise_weights, reduction=tf.losses.Reduction.NONE) k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio) k = tf.to_int32(tf.math.rint(k)) topk = tf.nn.top_k(losses, k) diff --git a/easy_rec/python/loss/softmax_loss_with_negative_mining.py b/easy_rec/python/loss/softmax_loss_with_negative_mining.py index b9ac28c99..a64542db5 100644 --- a/easy_rec/python/loss/softmax_loss_with_negative_mining.py +++ b/easy_rec/python/loss/softmax_loss_with_negative_mining.py @@ -6,23 +6,17 @@ tf = tf.compat.v1 -def support_vector_guided_softmax_loss( - pos_score, neg_scores, margin=0, t=1, smooth=1.0, threshold=0, weights=1.0 -): +def support_vector_guided_softmax_loss(pos_score, neg_scores, margin=0, t=1, smooth=1.0, threshold=0, weights=1.0): """Refer paper: Support Vector Guided Softmax Loss for Face Recognition (https://128.84.21.199/abs/1812.11317).""" new_pos_score = pos_score - margin cond = tf.greater_equal(new_pos_score - neg_scores, threshold) - mask = tf.where( - cond, tf.zeros_like(cond, tf.float32), tf.ones_like(cond, tf.float32) - ) # I_k + mask = tf.where(cond, tf.zeros_like(cond, tf.float32), tf.ones_like(cond, tf.float32)) # I_k new_neg_scores = mask * (neg_scores * t + t - 1) + (1 - mask) * neg_scores logits = tf.concat([new_pos_score, new_neg_scores], axis=1) if 1.0 != smooth: logits *= smooth - loss = tf.losses.sparse_softmax_cross_entropy( - tf.zeros_like(pos_score, dtype=tf.int32), logits, weights=weights - ) + loss = tf.losses.sparse_softmax_cross_entropy(tf.zeros_like(pos_score, dtype=tf.int32), logits, weights=weights) # set rank loss to zero if a batch has no positive sample. loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss) return loss @@ -38,7 +32,7 @@ def softmax_loss_with_negative_mining( gamma=1.0, margin=0, t=1, - seed=None + seed=None, ): """Compute the softmax loss based on the cosine distance explained below. @@ -74,7 +68,7 @@ def softmax_loss_with_negative_mining( is_valid = tf.assert_less( num_negative_samples, batch_size, - message='`num_negative_samples` should be less than batch_size' + message='`num_negative_samples` should be less than batch_size', ) with tf.control_dependencies([is_valid]): if not embed_normed: @@ -96,13 +90,9 @@ def softmax_loss_with_negative_mining( weights = tf.boolean_mask(weights, mask) # sim_scores's shape: (num_of_pos_label_in_batch_size, num_negative_samples + 1) - sim_scores = tf.keras.backend.batch_dot( - mask_user_emb, mask_item_emb, axes=(1, 2) - ) + sim_scores = tf.keras.backend.batch_dot(mask_user_emb, mask_item_emb, axes=(1, 2)) pos_score = tf.slice(sim_scores, [0, 0], [-1, 1]) neg_scores = tf.slice(sim_scores, [0, 1], [-1, -1]) - loss = support_vector_guided_softmax_loss( - pos_score, neg_scores, margin=margin, t=t, smooth=gamma, weights=weights - ) + loss = support_vector_guided_softmax_loss(pos_score, neg_scores, margin=margin, t=t, smooth=gamma, weights=weights) return loss diff --git a/easy_rec/python/loss/zero_inflated_lognormal.py b/easy_rec/python/loss/zero_inflated_lognormal.py index c4897cf07..b99ea3d02 100644 --- a/easy_rec/python/loss/zero_inflated_lognormal.py +++ b/easy_rec/python/loss/zero_inflated_lognormal.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Zero-inflated lognormal loss for lifetime value prediction.""" + import logging import tensorflow as tf @@ -16,9 +17,7 @@ def log_sigmoid(x): return -tf.nn.softplus(-x) # 兼容 TF 1.12 -def zero_inflated_lognormal_pred( - logits, max_sigma=5.0, max_log_clip=20.0, return_log=False -): +def zero_inflated_lognormal_pred(logits, max_sigma=5.0, max_log_clip=20.0, return_log=False): """Calculates predicted mean of zero inflated lognormal logits. Arguments: @@ -37,9 +36,7 @@ def zero_inflated_lognormal_pred( log_positive_probs = log_sigmoid(logits[..., :1]) mu = logits[..., 1:2] sigma = tf.keras.backend.softplus(logits[..., 2:]) - sigma = tf.clip_by_value( - sigma, tf.math.sqrt(tf.keras.backend.epsilon()), max_sigma - ) + sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), max_sigma) log_mean_pos = mu + 0.5 * tf.keras.backend.square(sigma) log_preds = log_positive_probs + log_mean_pos if return_log: @@ -59,7 +56,7 @@ def zero_inflated_lognormal_loss( sigma_reg=0.01, class_weight=1.0, reg_weight=1.0, - name='' + name='', ): """Computes the zero inflated lognormal loss. @@ -94,25 +91,18 @@ def zero_inflated_lognormal_loss( positive = tf.cast(labels > 0, tf.float32) logits = tf.convert_to_tensor(logits, dtype=tf.float32) - logits.shape.assert_is_compatible_with( - tf.TensorShape(labels.shape[:-1].as_list() + [3]) - ) + logits.shape.assert_is_compatible_with(tf.TensorShape(labels.shape[:-1].as_list() + [3])) positive_logits = logits[..., :1] - classification_loss = tf.keras.backend.binary_crossentropy( - positive, positive_logits, from_logits=True - ) + classification_loss = tf.keras.backend.binary_crossentropy(positive, positive_logits, from_logits=True) classification_loss = tf.keras.backend.mean(classification_loss) tf.summary.scalar('loss/%s_classify' % loss_name, classification_loss) mu = logits[..., 1:2] sigma = tf.keras.backend.softplus(logits[..., 2:]) - sigma = tf.clip_by_value( - sigma, tf.math.sqrt(tf.keras.backend.epsilon()), max_sigma - ) + sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), max_sigma) - safe_labels = positive * labels + (1 - positive - ) * tf.keras.backend.ones_like(labels) + safe_labels = positive * labels + (1 - positive) * tf.keras.backend.ones_like(labels) logprob = tfd.LogNormal(loc=mu, scale=sigma).log_prob(safe_labels) num_pos = tf.reduce_sum(positive) + 1e-8 regression_loss = -(tf.reduce_sum(positive * logprob) / num_pos) diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index 8edebcc01..145bfbf0c 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -20,10 +20,23 @@ from easy_rec.python.model.easy_rec_estimator import EasyRecEstimator from easy_rec.python.model.easy_rec_model import EasyRecModel from easy_rec.python.protos.train_pb2 import DistributionStrategy - -from easy_rec.python.utils import config_util, constant, estimator_utils, fg_util, load_class # NOQA -from easy_rec.python.utils.config_util import get_eval_input_path, get_model_dir_path, get_train_input_path, set_eval_input_path # NOQA -from easy_rec.python.utils.export_big_model import export_big_model, export_big_model_to_oss # NOQA +from easy_rec.python.utils import ( # NOQA + config_util, + constant, + estimator_utils, + fg_util, + load_class, +) +from easy_rec.python.utils.config_util import ( # NOQA + get_eval_input_path, + get_model_dir_path, + get_train_input_path, + set_eval_input_path, +) +from easy_rec.python.utils.export_big_model import ( # NOQA + export_big_model, + export_big_model_to_oss, +) try: import horovod.tensorflow as hvd @@ -60,7 +73,7 @@ def _get_input_fn( data_path=None, export_config=None, check_mode=False, - **kwargs + **kwargs, ): """Build estimator input function. @@ -86,7 +99,7 @@ def _get_input_fn( task_index=task_id, task_num=task_num, check_mode=check_mode, - **kwargs + **kwargs, ) input_fn = input_obj.create_input(export_config) return input_fn @@ -98,16 +111,18 @@ def _create_estimator(pipeline_config, distribution=None, params={}): gpu_options = GPUOptions(allow_growth=True) # False) logging.info( - 'train_config.train_distribute=%s[value=%d]' % ( - DistributionStrategy.Name(pipeline_config.train_config.train_distribute - ), pipeline_config.train_config.train_distribute + 'train_config.train_distribute=%s[value=%d]' + % ( + DistributionStrategy.Name(pipeline_config.train_config.train_distribute), + pipeline_config.train_config.train_distribute, ) ) # set gpu options only under hvd scenes if hvd is not None and pipeline_config.train_config.train_distribute in [ DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy, DistributionStrategy.HorovodStrategy + DistributionStrategy.SokStrategy, + DistributionStrategy.HorovodStrategy, ]: local_rnk = hvd.local_rank() gpus = tf.config.experimental.list_physical_devices('GPU') @@ -121,13 +136,14 @@ def _create_estimator(pipeline_config, distribution=None, params={}): allow_soft_placement=True, log_device_placement=params.get('log_device_placement', False), inter_op_parallelism_threads=train_config.inter_op_parallelism_threads, - intra_op_parallelism_threads=train_config.intra_op_parallelism_threads + intra_op_parallelism_threads=train_config.intra_op_parallelism_threads, ) if constant.NO_ARITHMETRIC_OPTI in os.environ: logging.info('arithmetic_optimization is closed to improve performance') - session_config.graph_options.rewrite_options.arithmetic_optimization = \ - session_config.graph_options.rewrite_options.OFF + session_config.graph_options.rewrite_options.arithmetic_optimization = ( + session_config.graph_options.rewrite_options.OFF + ) session_config.device_filters.append('/job:ps') model_cls = EasyRecModel.create_class(model_config.model_class) @@ -152,12 +168,10 @@ def _create_estimator(pipeline_config, distribution=None, params={}): keep_checkpoint_max=train_config.keep_checkpoint_max, train_distribute=distribution, eval_distribute=distribution, - session_config=session_config + session_config=session_config, ) - estimator = EasyRecEstimator( - pipeline_config, model_cls, run_config=run_config, params=params - ) + estimator = EasyRecEstimator(pipeline_config, model_cls, run_config=run_config, params=params) return estimator, run_config @@ -168,9 +182,7 @@ def _create_eval_export_spec(pipeline_config, eval_data, check_mode=False): eval_config = pipeline_config.eval_config export_config = pipeline_config.export_config if eval_config.num_examples > 0: - eval_steps = int( - math.ceil(float(eval_config.num_examples) / data_config.batch_size) - ) + eval_steps = int(math.ceil(float(eval_config.num_examples) / data_config.batch_size)) logging.info('eval_steps = %d' % eval_steps) else: eval_steps = None @@ -184,40 +196,33 @@ def _create_eval_export_spec(pipeline_config, eval_data, check_mode=False): None, export_config, check_mode=check_mode, - **input_fn_kwargs + **input_fn_kwargs, ) if export_config.exporter_type == 'final': - exporters = [ - FinalExporter(name='final', serving_input_receiver_fn=export_input_fn) - ] + exporters = [FinalExporter(name='final', serving_input_receiver_fn=export_input_fn)] elif export_config.exporter_type == 'latest': exporters = [ LatestExporter( name='latest', serving_input_receiver_fn=export_input_fn, - exports_to_keep=export_config.exports_to_keep + exports_to_keep=export_config.exports_to_keep, ) ] elif export_config.exporter_type == 'best': logging.info( - 'will use BestExporter, metric is %s, the bigger the better: %d' % - (export_config.best_exporter_metric, export_config.metric_bigger) + 'will use BestExporter, metric is %s, the bigger the better: %d' + % (export_config.best_exporter_metric, export_config.metric_bigger) ) def _metric_cmp_fn(best_eval_result, current_eval_result): - logging.info( - 'metric: best = %s current = %s' % - (str(best_eval_result), str(current_eval_result)) - ) + logging.info('metric: best = %s current = %s' % (str(best_eval_result), str(current_eval_result))) if export_config.metric_bigger: return ( - best_eval_result[export_config.best_exporter_metric] < - current_eval_result[export_config.best_exporter_metric] + best_eval_result[export_config.best_exporter_metric] < current_eval_result[export_config.best_exporter_metric] ) else: return ( - best_eval_result[export_config.best_exporter_metric] > - current_eval_result[export_config.best_exporter_metric] + best_eval_result[export_config.best_exporter_metric] > current_eval_result[export_config.best_exporter_metric] ) exporters = [ @@ -225,7 +230,7 @@ def _metric_cmp_fn(best_eval_result, current_eval_result): name='best', serving_input_receiver_fn=export_input_fn, compare_fn=_metric_cmp_fn, - exports_to_keep=export_config.exports_to_keep + exports_to_keep=export_config.exports_to_keep, ) ] elif export_config.exporter_type == 'none': @@ -235,15 +240,13 @@ def _metric_cmp_fn(best_eval_result, current_eval_result): # set throttle_secs to a small number, so that we can control evaluation # interval steps by checkpoint saving steps - eval_input_fn = _get_input_fn( - data_config, feature_configs, eval_data, **input_fn_kwargs - ) + eval_input_fn = _get_input_fn(data_config, feature_configs, eval_data, **input_fn_kwargs) eval_spec = tf.estimator.EvalSpec( name='val', input_fn=eval_input_fn, steps=eval_steps, throttle_secs=10, - exporters=exporters + exporters=exporters, ) return eval_spec @@ -253,11 +256,11 @@ def _check_model_dir(model_dir, continue_train): if not tf.gfile.IsDirectory(model_dir): tf.gfile.MakeDirs(model_dir) else: - assert len(tf.gfile.Glob(model_dir + '/model.ckpt-*.meta')) == 0, \ - 'model_dir[=%s] already exists and not empty(if you ' \ - 'want to continue train on current model_dir please ' \ - 'delete dir %s or specify --continue_train[internal use only])' % ( - model_dir, model_dir) + assert len(tf.gfile.Glob(model_dir + '/model.ckpt-*.meta')) == 0, ( + 'model_dir[=%s] already exists and not empty(if you ' + 'want to continue train on current model_dir please ' + 'delete dir %s or specify --continue_train[internal use only])' % (model_dir, model_dir) + ) else: if not tf.gfile.IsDirectory(model_dir): logging.info('%s does not exists, create it automatically' % model_dir) @@ -274,12 +277,10 @@ def _get_ckpt_path(pipeline_config, checkpoint_path): ckpt_path = estimator_utils.latest_checkpoint(pipeline_config.model_dir) logging.info( 'checkpoint_path is not specified, ' - 'will use latest checkpoint %s from %s' % - (ckpt_path, pipeline_config.model_dir) + 'will use latest checkpoint %s from %s' % (ckpt_path, pipeline_config.model_dir) ) else: - assert False, 'pipeline_config.model_dir(%s) does not exist' \ - % pipeline_config.model_dir + assert False, 'pipeline_config.model_dir(%s) does not exist' % pipeline_config.model_dir return ckpt_path @@ -296,12 +297,8 @@ def train_and_evaluate(pipeline_config_path, continue_train=False): Returns: None, the model will be saved into pipeline_config.model_dir """ - assert tf.gfile.Exists( - pipeline_config_path - ), 'pipeline_config_path not exists' - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path - ) + assert tf.gfile.Exists(pipeline_config_path), 'pipeline_config_path not exists' + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) _train_and_evaluate_impl(pipeline_config, continue_train) @@ -313,14 +310,13 @@ def _train_and_evaluate_impl( continue_train=False, check_mode=False, fit_on_eval=False, - fit_on_eval_steps=None + fit_on_eval_steps=None, ): train_config = pipeline_config.train_config data_config = pipeline_config.data_config feature_configs = config_util.get_compatible_feature_configs(pipeline_config) - if train_config.train_distribute != DistributionStrategy.NoStrategy\ - and train_config.sync_replicas: + if train_config.train_distribute != DistributionStrategy.NoStrategy and train_config.sync_replicas: logging.warning( 'will set sync_replicas to False, because train_distribute[%s] != NoStrategy' % pipeline_config.train_config.train_distribute @@ -334,25 +330,21 @@ def _train_and_evaluate_impl( params = {} if train_config.is_profiling: params['log_device_placement'] = True - estimator, run_config = _create_estimator( - pipeline_config, distribution=distribution, params=params - ) + estimator, run_config = _create_estimator(pipeline_config, distribution=distribution, params=params) version_file = os.path.join(pipeline_config.model_dir, 'version') if estimator_utils.is_chief(): _check_model_dir(pipeline_config.model_dir, continue_train) - config_util.save_pipeline_config( - pipeline_config, pipeline_config.model_dir - ) + config_util.save_pipeline_config(pipeline_config, pipeline_config.model_dir) with tf.gfile.GFile(version_file, 'w') as f: f.write(easy_rec.__version__ + '\n') train_steps = None if train_config.HasField('num_steps') and train_config.num_steps > 0: train_steps = train_config.num_steps - assert train_steps is not None or data_config.num_epochs > 0, ( - 'either num_steps and num_epochs must be set to an integer > 0.' - ) + assert ( + train_steps is not None or data_config.num_epochs > 0 + ), 'either num_steps and num_epochs must be set to an integer > 0.' if train_steps and data_config.num_epochs: logging.info('Both num_steps and num_epochs are set.') @@ -374,16 +366,14 @@ def _train_and_evaluate_impl( feature_configs, train_data, check_mode=check_mode, - **input_fn_kwargs + **input_fn_kwargs, ) # Currently only a single Eval Spec is allowed. - train_spec = tf.estimator.TrainSpec( - input_fn=train_input_fn, max_steps=train_steps - ) + train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn, max_steps=train_steps) embedding_parallel = train_config.train_distribute in ( DistributionStrategy.SokStrategy, - DistributionStrategy.EmbeddingParallelStrategy + DistributionStrategy.EmbeddingParallelStrategy, ) if embedding_parallel: @@ -391,31 +381,24 @@ def _train_and_evaluate_impl( input_fn=train_input_fn, max_steps=train_spec.max_steps, hooks=list(train_spec.hooks), - saving_listeners=train_spec.saving_listeners + saving_listeners=train_spec.saving_listeners, ) train_input_fn.input_creator.stop() else: # create eval spec - eval_spec = _create_eval_export_spec( - pipeline_config, eval_data, check_mode=check_mode - ) + eval_spec = _create_eval_export_spec(pipeline_config, eval_data, check_mode=check_mode) estimator_train.train_and_evaluate(estimator, train_spec, eval_spec) logging.info('Train and evaluate finish') if fit_on_eval and (not estimator_utils.is_evaluator()): tf.reset_default_graph() logging.info('Start continue training on eval data') - eval_input_fn = _get_input_fn( - data_config, feature_configs, eval_data, **input_fn_kwargs - ) + eval_input_fn = _get_input_fn(data_config, feature_configs, eval_data, **input_fn_kwargs) if fit_on_eval_steps is not None: # wait estimator train done to get the correct train_steps while not estimator_train.estimator_train_done(estimator): time.sleep(1) train_steps = estimator_utils.get_trained_steps(estimator.model_dir) - logging.info( - '\ttrain_steps=%d fit_on_eval_steps=%d' % - (train_steps, fit_on_eval_steps) - ) + logging.info('\ttrain_steps=%d fit_on_eval_steps=%d' % (train_steps, fit_on_eval_steps)) fit_on_eval_steps += train_steps # Do not use estimator_train.train_and_evaluate as it starts tf.Server, # which is redundant and reports port not available error. @@ -423,8 +406,7 @@ def _train_and_evaluate_impl( input_fn=eval_input_fn, max_steps=fit_on_eval_steps, hooks=list(train_spec.hooks), - saving_listeners=train_spec.saving_listeners - if hasattr(train_spec, 'saving_listeners') else None + saving_listeners=(train_spec.saving_listeners if hasattr(train_spec, 'saving_listeners') else None), ) logging.info('Finished training on eval data') # return estimator for custom training using estimator.train @@ -435,7 +417,7 @@ def evaluate( pipeline_config, eval_checkpoint_path='', eval_data_path=None, - eval_result_filename='eval_result.txt' + eval_result_filename='eval_result.txt', ): """Evaluate a EasyRec model defined in pipeline_config_path. @@ -473,11 +455,10 @@ def evaluate( if 'TF_CONFIG' in os.environ: tf_config = estimator_utils.chief_to_master() from tensorflow.python.training import server_lib + if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = server_lib.Server( - cluster, job_name='ps', task_index=tf_config['task']['index'] - ) + server = server_lib.Server(cluster, job_name='ps', task_index=tf_config['task']['index']) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: @@ -493,39 +474,29 @@ def evaluate( if server_target: # evaluate with parameter server - input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL - ).make_one_shot_iterator() + input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() input_feas, input_lbls = input_iter.get_next() from tensorflow.python.framework.ops import device from tensorflow.python.training.device_setter import replica_device_setter - from tensorflow.python.training.monitored_session import ChiefSessionCreator # NOQA - from tensorflow.python.training.monitored_session import MonitoredSession - with device( - replica_device_setter( - worker_device='/job:master/task:0', cluster=cluster - ) - ): - estimator_spec = estimator._eval_model_fn( - input_feas, input_lbls, run_config - ) - - session_config = ConfigProto( - allow_soft_placement=True, log_device_placement=True + from tensorflow.python.training.monitored_session import ( # NOQA + ChiefSessionCreator, + MonitoredSession, ) + + with device(replica_device_setter(worker_device='/job:master/task:0', cluster=cluster)): + estimator_spec = estimator._eval_model_fn(input_feas, input_lbls, run_config) + + session_config = ConfigProto(allow_soft_placement=True, log_device_placement=True) chief_sess_creator = ChiefSessionCreator( master=server_target, checkpoint_filename_with_path=ckpt_path, - config=session_config + config=session_config, ) eval_metric_ops = estimator_spec.eval_metric_ops update_ops = [eval_metric_ops[x][1] for x in eval_metric_ops.keys()] metric_ops = {x: eval_metric_ops[x][0] for x in eval_metric_ops.keys()} update_op = tf.group(update_ops) - with MonitoredSession( - session_creator=chief_sess_creator, - hooks=None, - stop_grace_period_secs=120 - ) as sess: + with MonitoredSession(session_creator=chief_sess_creator, hooks=None, stop_grace_period_secs=120) as sess: while True: try: sess.run(update_op) @@ -538,9 +509,7 @@ def evaluate( # with tf.device( # replica_device_setter( # worker_device='/job:master/task:0', cluster=cluster)): - eval_result = estimator.evaluate( - eval_spec.input_fn, eval_spec.steps, checkpoint_path=ckpt_path - ) + eval_result = estimator.evaluate(eval_spec.input_fn, eval_spec.steps, checkpoint_path=ckpt_path) eval_spec.input_fn.input_creator.stop() logging.info('Evaluate finish') @@ -566,7 +535,7 @@ def distribute_evaluate( pipeline_config, eval_checkpoint_path='', eval_data_path=None, - eval_result_filename='distribute_eval_result.txt' + eval_result_filename='distribute_eval_result.txt', ): """Evaluate a EasyRec model defined in pipeline_config_path. @@ -598,9 +567,7 @@ def distribute_evaluate( eval_data = get_eval_input_path(pipeline_config) data_config = pipeline_config.data_config if data_config.HasField('sampler'): - logging.warning( - 'It is not accuracy to use eval with negative sampler, recommand to use hitrate.py!' - ) + logging.warning('It is not accuracy to use eval with negative sampler, recommand to use hitrate.py!') eval_result = {} return eval_result model_dir = get_model_dir_path(pipeline_config) @@ -608,9 +575,7 @@ def distribute_evaluate( if not tf.gfile.IsDirectory(eval_tmp_results_dir): logging.info('create eval tmp results dir {}'.format(eval_tmp_results_dir)) tf.gfile.MakeDirs(eval_tmp_results_dir) - assert tf.gfile.IsDirectory( - eval_tmp_results_dir - ), 'tmp results dir not create success.' + assert tf.gfile.IsDirectory(eval_tmp_results_dir), 'tmp results dir not create success.' os.environ['eval_tmp_results_dir'] = eval_tmp_results_dir server_target = None @@ -619,20 +584,17 @@ def distribute_evaluate( tf_config = estimator_utils.chief_to_master() from tensorflow.python.training import server_lib + if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = server_lib.Server( - cluster, job_name='ps', task_index=tf_config['task']['index'] - ) + server = server_lib.Server(cluster, job_name='ps', task_index=tf_config['task']['index']) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: cur_job_name = tf_config['task']['type'] cur_task_index = tf_config['task']['index'] cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = server_lib.Server( - cluster, job_name=cur_job_name, task_index=cur_task_index - ) + server = server_lib.Server(cluster, job_name=cur_job_name, task_index=cur_task_index) server_target = server.target print('server_target = %s' % server_target) elif tf_config['task']['type'] == 'worker': @@ -640,68 +602,53 @@ def distribute_evaluate( cur_job_name = tf_config['task']['type'] cur_task_index = tf_config['task']['index'] cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = server_lib.Server( - cluster, job_name=cur_job_name, task_index=cur_task_index - ) + server = server_lib.Server(cluster, job_name=cur_job_name, task_index=cur_task_index) server_target = server.target print('server_target = %s' % server_target) if server_target: from tensorflow.python.framework.ops import device from tensorflow.python.training.device_setter import replica_device_setter - from tensorflow.python.training.monitored_session import ChiefSessionCreator # NOQA - from tensorflow.python.training.monitored_session import WorkerSessionCreator # NOQA - from tensorflow.python.training.monitored_session import MonitoredSession + from tensorflow.python.training.monitored_session import ( # NOQA + ChiefSessionCreator, + MonitoredSession, + WorkerSessionCreator, + ) from easy_rec.python.utils.estimator_utils import EvaluateExitBarrierHook + cur_work_device = '/job:' + cur_job_name + '/task:' + str(cur_task_index) cur_ps_num = len(tf_config['cluster']['ps']) - with device( - replica_device_setter( - ps_tasks=cur_ps_num, worker_device=cur_work_device, cluster=cluster - ) - ): + with device(replica_device_setter(ps_tasks=cur_ps_num, worker_device=cur_work_device, cluster=cluster)): distribution = strategy_builder.build(train_config) estimator, run_config = _create_estimator(pipeline_config, distribution) eval_spec = _create_eval_export_spec(pipeline_config, eval_data) ckpt_path = _get_ckpt_path(pipeline_config, eval_checkpoint_path) ckpt_dir = os.path.dirname(ckpt_path) - input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL - ).make_one_shot_iterator() + input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() input_feas, input_lbls = input_iter.get_next() - estimator_spec = estimator._distribute_eval_model_fn( - input_feas, input_lbls, run_config - ) + estimator_spec = estimator._distribute_eval_model_fn(input_feas, input_lbls, run_config) session_config = ConfigProto( allow_soft_placement=True, log_device_placement=True, - device_filters=['/job:ps', - '/job:worker/task:%d' % cur_task_index] + device_filters=['/job:ps', '/job:worker/task:%d' % cur_task_index], ) if cur_job_name == 'master': metric_variables = tf.get_collection(tf.GraphKeys.METRIC_VARIABLES) - model_ready_for_local_init_op = tf.variables_initializer( - metric_variables - ) + model_ready_for_local_init_op = tf.variables_initializer(metric_variables) global_variables = tf.global_variables() - remain_variables = list( - set(global_variables).difference(set(metric_variables)) - ) + remain_variables = list(set(global_variables).difference(set(metric_variables))) cur_saver = tf.train.Saver(var_list=remain_variables, sharded=True) - cur_scaffold = tf.train.Scaffold( - saver=cur_saver, ready_for_local_init_op=model_ready_for_local_init_op - ) + cur_scaffold = tf.train.Scaffold(saver=cur_saver, ready_for_local_init_op=model_ready_for_local_init_op) cur_sess_creator = ChiefSessionCreator( scaffold=cur_scaffold, master=server_target, checkpoint_filename_with_path=ckpt_path, - config=session_config + config=session_config, ) else: - cur_sess_creator = WorkerSessionCreator( - master=server_target, config=session_config - ) + cur_sess_creator = WorkerSessionCreator(master=server_target, config=session_config) eval_metric_ops = estimator_spec.eval_metric_ops update_ops = [eval_metric_ops[x][1] for x in eval_metric_ops.keys()] metric_ops = {x: eval_metric_ops[x][0] for x in eval_metric_ops.keys()} @@ -709,18 +656,14 @@ def distribute_evaluate( cur_worker_num = len(tf_config['cluster']['worker']) + 1 if cur_job_name == 'master': cur_stop_grace_period_sesc = 120 - cur_hooks = EvaluateExitBarrierHook( - cur_worker_num, True, ckpt_dir, metric_ops - ) + cur_hooks = EvaluateExitBarrierHook(cur_worker_num, True, ckpt_dir, metric_ops) else: cur_stop_grace_period_sesc = 10 - cur_hooks = EvaluateExitBarrierHook( - cur_worker_num, False, ckpt_dir, metric_ops - ) + cur_hooks = EvaluateExitBarrierHook(cur_worker_num, False, ckpt_dir, metric_ops) with MonitoredSession( session_creator=cur_sess_creator, hooks=[cur_hooks], - stop_grace_period_secs=cur_stop_grace_period_sesc + stop_grace_period_secs=cur_stop_grace_period_sesc, ) as sess: while True: try: @@ -785,9 +728,7 @@ def predict(pipeline_config, checkpoint_path='', data_path=None): ckpt_path = _get_ckpt_path(pipeline_config, checkpoint_path) - pred_result = estimator.predict( - eval_spec.input_fn, checkpoint_path=ckpt_path - ) + pred_result = estimator.predict(eval_spec.input_fn, checkpoint_path=ckpt_path) logging.info('Predict finish') return pred_result @@ -798,7 +739,7 @@ def export( checkpoint_path='', asset_files=None, verbose=False, - **extra_params + **extra_params, ): """Export model defined in pipeline_config_path. @@ -844,9 +785,7 @@ def export( logging.info('will add asset files: %s' % asset_files) for asset_file in asset_files.split(','): asset_file = asset_file.strip() - if ':' not in asset_file or asset_file.startswith( - 'oss:' - ) or asset_file.startswith('hdfs:'): + if ':' not in asset_file or asset_file.startswith('oss:') or asset_file.startswith('hdfs:'): _, asset_name = os.path.split(asset_file) else: asset_name, asset_file = asset_file.split(':', 1) @@ -859,9 +798,7 @@ def export( input_fn_kwargs = {'pipeline_config': pipeline_config} if data_config.input_type == data_config.InputType.OdpsRTPInputV2: input_fn_kwargs['fg_json_path'] = pipeline_config.fg_json_path - serving_input_fn = _get_input_fn( - data_config, feature_configs, None, export_config, **input_fn_kwargs - ) + serving_input_fn = _get_input_fn(data_config, feature_configs, None, export_config, **input_fn_kwargs) ckpt_path = _get_ckpt_path(pipeline_config, checkpoint_path) if 'oss_path' in extra_params: if pipeline_config.train_config.HasField('incr_save_config'): @@ -870,34 +807,40 @@ def export( incr_save_type = incr_save_config.WhichOneof('incr_update') logging.info('incr_save_type=%s' % incr_save_type) if incr_save_type: - extra_params['incr_update'][incr_save_type] = getattr( - incr_save_config, incr_save_type - ) + extra_params['incr_update'][incr_save_type] = getattr(incr_save_config, incr_save_type) return export_big_model_to_oss( - export_dir, pipeline_config, extra_params, serving_input_fn, estimator, - ckpt_path, verbose + export_dir, + pipeline_config, + extra_params, + serving_input_fn, + estimator, + ckpt_path, + verbose, ) if 'redis_url' in extra_params: return export_big_model( - export_dir, pipeline_config, extra_params, serving_input_fn, estimator, - ckpt_path, verbose + export_dir, + pipeline_config, + extra_params, + serving_input_fn, + estimator, + ckpt_path, + verbose, ) final_export_dir = estimator.export_savedmodel( export_dir_base=export_dir, serving_input_receiver_fn=serving_input_fn, checkpoint_path=ckpt_path, - strip_default_attrs=True + strip_default_attrs=True, ) # add export ts as version info saved_model = saved_model_pb2.SavedModel() - if type(final_export_dir) not in [type(''), type(u'')]: + if type(final_export_dir) not in [type(''), type('')]: final_export_dir = final_export_dir.decode('utf-8') - export_ts = [ - x for x in final_export_dir.split('/') if x != '' and x is not None - ] + export_ts = [x for x in final_export_dir.split('/') if x != '' and x is not None] export_ts = export_ts[-1] saved_pb_path = os.path.join(final_export_dir, 'saved_model.pb') with tf.gfile.GFile(saved_pb_path, 'rb') as fin: @@ -932,7 +875,7 @@ def export_checkpoint( checkpoint_path='', asset_files=None, verbose=False, - mode=tf.estimator.ModeKeys.PREDICT + mode=tf.estimator.ModeKeys.PREDICT, ): """Export the EasyRec model as checkpoint.""" pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config) @@ -954,15 +897,13 @@ def export_checkpoint( # construct serving input fn export_config = pipeline_config.export_config - serving_input_fn = _get_input_fn( - data_config, feature_configs, None, export_config, **input_fn_kwargs - ) + serving_input_fn = _get_input_fn(data_config, feature_configs, None, export_config, **input_fn_kwargs) ckpt_path = _get_ckpt_path(pipeline_config, checkpoint_path) estimator.export_checkpoint( export_path=export_path, serving_input_receiver_fn=serving_input_fn, checkpoint_path=ckpt_path, - mode=mode + mode=mode, ) logging.info('model checkpoint has been exported successfully') diff --git a/easy_rec/python/model/autoint.py b/easy_rec/python/model/autoint.py index 958eb760e..5156426fa 100644 --- a/easy_rec/python/model/autoint.py +++ b/easy_rec/python/model/autoint.py @@ -6,7 +6,6 @@ from easy_rec.python.layers import multihead_attention from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.autoint_pb2 import AutoInt as AutoIntConfig # NOQA if tf.__version__ >= '2.0': @@ -14,20 +13,11 @@ class AutoInt(RankModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(AutoInt, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(AutoInt, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'autoint', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model') ) - assert self._model_config.WhichOneof('model') == 'autoint', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._features, _ = self._input_layer(self._feature_dict, 'all') self._feature_num = len(self._model_config.feature_groups[0].feature_names) self._seq_key_num = 0 @@ -42,8 +32,9 @@ def __init__( fea_emb_dim_list = [] for feature_config in feature_configs: fea_emb_dim_list.append(feature_config.embedding_dim) - assert len(set(fea_emb_dim_list)) == 1 and len(fea_emb_dim_list) == self._feature_num, \ - 'AutoInt requires that all feature dimensions must be consistent.' + assert ( + len(set(fea_emb_dim_list)) == 1 and len(fea_emb_dim_list) == self._feature_num + ), 'AutoInt requires that all feature dimensions must be consistent.' self._d_model = fea_emb_dim_list[0] self._head_num = self._model_config.multi_head_num @@ -54,7 +45,7 @@ def build_predict_graph(self): attention_fea = tf.reshape( self._features, - shape=[-1, self._feature_num + self._seq_key_num, self._d_model] + shape=[-1, self._feature_num + self._seq_key_num, self._d_model], ) for i in range(self._model_config.interacting_layer_num): @@ -63,14 +54,11 @@ def build_predict_graph(self): head_size=self._head_size, l2_reg=self._l2_reg, use_res=True, - name='multi_head_self_attention_layer_%d' % i + name='multi_head_self_attention_layer_%d' % i, ) attention_fea = attention_layer(attention_fea) - attention_fea = tf.reshape( - attention_fea, - shape=[-1, attention_fea.shape[1] * attention_fea.shape[2]] - ) + attention_fea = tf.reshape(attention_fea, shape=[-1, attention_fea.shape[1] * attention_fea.shape[2]]) final = tf.layers.dense(attention_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/cmbf.py b/easy_rec/python/model/cmbf.py index 2dcd18ac1..e65178322 100644 --- a/easy_rec/python/model/cmbf.py +++ b/easy_rec/python/model/cmbf.py @@ -4,7 +4,6 @@ from easy_rec.python.layers import cmbf, dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.cmbf_pb2 import CMBF as CMBFConfig # NOQA if tf.__version__ >= '2.0': @@ -19,33 +18,24 @@ class CMBF(RankModel): https://www.mdpi.com/1424-8220/21/16/5275 """ - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(CMBF, self).__init__( - model_config, feature_configs, features, labels, is_training - ) - assert self._model_config.WhichOneof('model') == 'cmbf', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(CMBF, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'cmbf', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model' ) self._cmbf_layer = cmbf.CMBF( - model_config, feature_configs, features, self._model_config.cmbf.config, - self._input_layer + model_config, + feature_configs, + features, + self._model_config.cmbf.config, + self._input_layer, ) self._model_config = self._model_config.cmbf def build_predict_graph(self): hidden = self._cmbf_layer(self._is_training, l2_reg=self._l2_reg) - final_dnn_layer = dnn.DNN( - self._model_config.final_dnn, self._l2_reg, 'final_dnn', - self._is_training - ) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) all_fea = final_dnn_layer(hidden) final = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/collaborative_metric_learning.py b/easy_rec/python/model/collaborative_metric_learning.py index 2015f01d9..066b946f8 100644 --- a/easy_rec/python/model/collaborative_metric_learning.py +++ b/easy_rec/python/model/collaborative_metric_learning.py @@ -1,34 +1,35 @@ import tensorflow as tf +from easy_rec.python.core.metrics import ( # NOQA + metric_learning_average_precision_at_k, + metric_learning_recall_at_k, +) from easy_rec.python.layers import dnn from easy_rec.python.layers.common_layers import highway from easy_rec.python.loss.circle_loss import circle_loss from easy_rec.python.loss.multi_similarity import ms_loss from easy_rec.python.model.easy_rec_model import EasyRecModel +from easy_rec.python.protos.collaborative_metric_learning_pb2 import ( + CoMetricLearningI2I as MetricLearningI2IConfig, # NOQA +) from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.utils.activation import gelu from easy_rec.python.utils.proto_util import copy_obj -from easy_rec.python.core.metrics import metric_learning_average_precision_at_k, metric_learning_recall_at_k # NOQA -from easy_rec.python.protos.collaborative_metric_learning_pb2 import CoMetricLearningI2I as MetricLearningI2IConfig # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 class CoMetricLearningI2I(EasyRecModel): - def __init__( self, model_config, # pipeline.model_config feature_configs, # pipeline.feature_configs features, # same as model_fn input labels=None, - is_training=False + is_training=False, ): - super(CoMetricLearningI2I, self).__init__( - model_config, feature_configs, features, labels, is_training - ) + super(CoMetricLearningI2I, self).__init__(model_config, feature_configs, features, labels, is_training) model = self._model_config.WhichOneof('model') assert model == 'metric_learning', 'invalid model config: %s' % model @@ -46,25 +47,19 @@ def __init__( elif self._loss_type == LossType.MULTI_SIMILARITY_LOSS: self.loss = self._model_config.multi_similarity_loss else: - raise ValueError( - 'unsupported loss type: %s' % LossType.Name(self._loss_type) - ) + raise ValueError('unsupported loss type: %s' % LossType.Name(self._loss_type)) if not self.has_backbone: self._highway_features = {} self._highway_num = len(self._model_config.highway) for _id in range(self._highway_num): highway_cfg = self._model_config.highway[_id] - highway_feature, _ = self._input_layer( - self._feature_dict, highway_cfg.input - ) + highway_feature, _ = self._input_layer(self._feature_dict, highway_cfg.input) self._highway_features[highway_cfg.input] = highway_feature self.input_features = [] if self._model_config.HasField('input'): - input_feature, _ = self._input_layer( - self._feature_dict, self._model_config.input - ) + input_feature, _ = self._input_layer(self._feature_dict, self._model_config.input) self.input_features.append(input_feature) self.dnn = copy_obj(self._model_config.dnn) @@ -93,13 +88,13 @@ def build_predict_graph(self): self._highway_features[highway_cfg.input], training=self._is_training, trainable=True, - name='highway_%s_bn' % highway_cfg.input + name='highway_%s_bn' % highway_cfg.input, ) highway_fea = highway( highway_fea, highway_cfg.emb_size, activation=gelu, - scope='highway_%s' % _id + scope='highway_%s' % _id, ) print('highway_fea: ', highway_fea) self.input_features.append(highway_fea) @@ -114,24 +109,18 @@ def build_predict_graph(self): inputs=net_output, units=last_hidden, kernel_regularizer=self._l2_reg, - name='dnn/dnn_%d' % (num_dnn_layer - 1) + name='dnn/dnn_%d' % (num_dnn_layer - 1), ) if self._model_config.output_l2_normalized_emb: norm_emb = tf.nn.l2_normalize(tower_emb, axis=-1) self._prediction_dict['norm_emb'] = norm_emb - self._prediction_dict['norm_embedding'] = tf.reduce_join( - tf.as_string(norm_emb), axis=-1, separator=',' - ) + self._prediction_dict['norm_embedding'] = tf.reduce_join(tf.as_string(norm_emb), axis=-1, separator=',') self._prediction_dict['float_emb'] = tower_emb - self._prediction_dict['embedding'] = tf.reduce_join( - tf.as_string(tower_emb), axis=-1, separator=',' - ) + self._prediction_dict['embedding'] = tf.reduce_join(tf.as_string(tower_emb), axis=-1, separator=',') if self.sample_id is not None and self.sample_id in self._feature_dict: - self._prediction_dict['sample_id'] = tf.identity( - self._feature_dict[self.sample_id] - ) + self._prediction_dict['sample_id'] = tf.identity(self._feature_dict[self.sample_id]) return self._prediction_dict def build_loss_graph(self): @@ -145,7 +134,7 @@ def build_loss_graph(self): self.session_ids, self.loss.margin, self.loss.gamma, - embed_normed=emb_normed + embed_normed=emb_normed, ) elif self._loss_type == LossType.MULTI_SIMILARITY_LOSS: self._loss_dict['ms_loss'] = ms_loss( @@ -156,12 +145,10 @@ def build_loss_graph(self): self.loss.beta, self.loss.lamb, self.loss.eps, - embed_normed=emb_normed + embed_normed=emb_normed, ) else: - raise ValueError( - 'invalid loss type: %s' % LossType.Name(self._loss_type) - ) + raise ValueError('invalid loss type: %s' % LossType.Name(self._loss_type)) return self._loss_dict @@ -186,15 +173,7 @@ def build_metric_graph(self, eval_config): emb = self._prediction_dict['float_emb'] if len(recall_at_k) > 0: - metric_dict.update( - metric_learning_recall_at_k( - recall_at_k, emb, self.labels, self.session_ids - ) - ) + metric_dict.update(metric_learning_recall_at_k(recall_at_k, emb, self.labels, self.session_ids)) if len(precision_at_k) > 0: - metric_dict.update( - metric_learning_average_precision_at_k( - precision_at_k, emb, self.labels, self.session_ids - ) - ) + metric_dict.update(metric_learning_average_precision_at_k(precision_at_k, emb, self.labels, self.session_ids)) return metric_dict diff --git a/easy_rec/python/model/dat.py b/easy_rec/python/model/dat.py index 9b86387e1..32adc7e44 100644 --- a/easy_rec/python/model/dat.py +++ b/easy_rec/python/model/dat.py @@ -15,23 +15,13 @@ class DAT(MatchModel): """Dual Augmented Two-tower Model.""" - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(DAT, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(DAT, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'dat', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model' ) - assert self._model_config.WhichOneof('model') == 'dat', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') - feature_group_names = [ - fg.group_name for fg in self._model_config.feature_groups - ] + feature_group_names = [fg.group_name for fg in self._model_config.feature_groups] assert 'user' in feature_group_names, 'user feature group not found' assert 'item' in feature_group_names, 'item feature group not found' assert 'user_id_augment' in feature_group_names, 'user_id_augment feature group not found' @@ -42,15 +32,11 @@ def __init__( self.user_tower = copy_obj(self._model_config.user_tower) self.user_deep_feature, _ = self._input_layer(self._feature_dict, 'user') - self.user_augmented_vec, _ = self._input_layer( - self._feature_dict, 'user_id_augment' - ) + self.user_augmented_vec, _ = self._input_layer(self._feature_dict, 'user_id_augment') self.item_tower = copy_obj(self._model_config.item_tower) self.item_deep_feature, _ = self._input_layer(self._feature_dict, 'item') - self.item_augmented_vec, _ = self._input_layer( - self._feature_dict, 'item_id_augment' - ) + self.item_augmented_vec, _ = self._input_layer(self._feature_dict, 'item_id_augment') self._user_tower_emb = None self._item_tower_emb = None @@ -58,36 +44,28 @@ def __init__( def build_predict_graph(self): num_user_dnn_layer = len(self.user_tower.dnn.hidden_units) last_user_hidden = self.user_tower.dnn.hidden_units.pop() - user_dnn = dnn.DNN( - self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training - ) + user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training) - user_tower_feature = tf.concat( - [self.user_deep_feature, self.user_augmented_vec], axis=-1 - ) + user_tower_feature = tf.concat([self.user_deep_feature, self.user_augmented_vec], axis=-1) user_tower_emb = user_dnn(user_tower_feature) user_tower_emb = tf.layers.dense( inputs=user_tower_emb, units=last_user_hidden, kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1) + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), ) num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() - item_dnn = dnn.DNN( - self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training - ) + item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training) - item_tower_feature = tf.concat( - [self.item_deep_feature, self.item_augmented_vec], axis=-1 - ) + item_tower_feature = tf.concat([self.item_deep_feature, self.item_augmented_vec], axis=-1) item_tower_emb = item_dnn(item_tower_feature) item_tower_emb = tf.layers.dense( inputs=item_tower_emb, units=last_item_hidden, kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1) + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), ) user_tower_emb = self.norm(user_tower_emb) @@ -100,9 +78,7 @@ def build_predict_graph(self): raise ValueError('Currently DAT model only supports list wise mode.') if self._loss_type == LossType.CLASSIFICATION: - raise ValueError( - 'Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.' - ) + raise ValueError('Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.') elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: y_pred = self._mask_in_batch(y_pred) self._prediction_dict['logits'] = y_pred @@ -112,12 +88,8 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb - self._prediction_dict['user_emb'] = tf.reduce_join( - tf.as_string(user_tower_emb), axis=-1, separator=',' - ) - self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_tower_emb), axis=-1, separator=',' - ) + self._prediction_dict['user_emb'] = tf.reduce_join(tf.as_string(user_tower_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_tower_emb), axis=-1, separator=',') augmented_p_u = tf.stop_gradient(user_tower_emb) augmented_p_i = tf.stop_gradient(item_tower_emb) @@ -132,20 +104,21 @@ def build_predict_graph(self): def get_outputs(self): if self._loss_type == LossType.CLASSIFICATION: - raise ValueError( - 'Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.' - ) + raise ValueError('Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.') elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: - self._prediction_dict['logits'] = tf.squeeze( - self._prediction_dict['logits'], axis=-1 - ) - self._prediction_dict['probs'] = tf.nn.sigmoid( - self._prediction_dict['logits'] - ) + self._prediction_dict['logits'] = tf.squeeze(self._prediction_dict['logits'], axis=-1) + self._prediction_dict['probs'] = tf.nn.sigmoid(self._prediction_dict['logits']) return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb', 'augmented_p_u', 'augmented_p_i', 'augmented_a_u', - 'augmented_a_i' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_tower_emb', + 'item_tower_emb', + 'augmented_p_u', + 'augmented_p_i', + 'augmented_a_u', + 'augmented_a_i', ] else: raise ValueError('invalid loss type: %s' % str(self._loss_type)) diff --git a/easy_rec/python/model/dbmtl.py b/easy_rec/python/model/dbmtl.py index 261799e36..b421bfdd3 100644 --- a/easy_rec/python/model/dbmtl.py +++ b/easy_rec/python/model/dbmtl.py @@ -11,37 +11,32 @@ class DBMTL(MultiTaskModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(DBMTL, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(DBMTL, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'dbmtl', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model') ) - assert self._model_config.WhichOneof('model') == 'dbmtl', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.dbmtl assert isinstance(self._model_config, DBMTLConfig) if self._model_config.HasField('bottom_cmbf'): self._cmbf_layer = cmbf.CMBF( - model_config, feature_configs, features, - self._model_config.bottom_cmbf, self._input_layer + model_config, + feature_configs, + features, + self._model_config.bottom_cmbf, + self._input_layer, ) elif self._model_config.HasField('bottom_uniter'): self._uniter_layer = uniter.Uniter( - model_config, feature_configs, features, - self._model_config.bottom_uniter, self._input_layer + model_config, + feature_configs, + features, + self._model_config.bottom_uniter, + self._input_layer, ) elif not self.has_backbone: - self._features, self._feature_list = self._input_layer( - self._feature_dict, 'all' - ) + self._features, self._feature_list = self._input_layer(self._feature_dict, 'all') else: assert False, 'invalid code branch' self._init_towers(self._model_config.task_towers) @@ -58,7 +53,7 @@ def build_predict_graph(self): self._model_config.bottom_dnn, self._l2_reg, name='bottom_dnn', - is_training=self._is_training + is_training=self._is_training, ) bottom_fea = bottom_dnn(self._features) else: @@ -70,7 +65,7 @@ def build_predict_graph(self): self._model_config.expert_dnn, l2_reg=self._l2_reg, num_task=self._task_num, - num_expert=self._model_config.num_expert + num_expert=self._model_config.num_expert, ) task_input_list = mmoe_layer(bottom_fea) else: @@ -85,7 +80,7 @@ def build_predict_graph(self): task_tower_cfg.dnn, self._l2_reg, name=tower_name + '/dnn', - is_training=self._is_training + is_training=self._is_training, ) tower_fea = tower_dnn(task_input_list[i]) tower_features[tower_name] = tower_fea @@ -101,14 +96,12 @@ def build_predict_graph(self): task_tower_cfg.relation_dnn, self._l2_reg, name=tower_name + '/relation_dnn', - is_training=self._is_training + is_training=self._is_training, ) tower_inputs = [tower_features[tower_name]] for relation_tower_name in task_tower_cfg.relation_tower_names: tower_inputs.append(relation_features[relation_tower_name]) - relation_input = tf.concat( - tower_inputs, axis=-1, name=tower_name + '/relation_input' - ) + relation_input = tf.concat(tower_inputs, axis=-1, name=tower_name + '/relation_input') relation_fea = relation_dnn(relation_input) relation_features[tower_name] = relation_fea @@ -116,7 +109,7 @@ def build_predict_graph(self): relation_fea, task_tower_cfg.num_class, kernel_regularizer=self._l2_reg, - name=tower_name + '/output' + name=tower_name + '/output', ) tower_outputs[tower_name] = output_logits diff --git a/easy_rec/python/model/dcn.py b/easy_rec/python/model/dcn.py index 25b0c5b09..7e44545c9 100644 --- a/easy_rec/python/model/dcn.py +++ b/easy_rec/python/model/dcn.py @@ -5,7 +5,6 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.dcn_pb2 import DCN as DCNConfig # NOQA if tf.__version__ >= '2.0': @@ -13,20 +12,11 @@ class DCN(RankModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(DCN, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(DCN, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'dcn', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model' ) - assert self._model_config.WhichOneof('model') == 'dcn', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.dcn assert isinstance(self._model_config, DCNConfig) @@ -42,9 +32,7 @@ def _cross_net(self, tensor, num_cross_layers): dtype=tf.float32, shape=(input_dim), ) - b = tf.get_variable( - name=name + '_b', dtype=tf.float32, shape=(input_dim) - ) + b = tf.get_variable(name=name + '_b', dtype=tf.float32, shape=(input_dim)) xw = tf.reduce_sum(x * w, axis=1, keepdims=True) # (B, 1) x = tf.math.add(tf.math.add(x0 * xw, b), x) return x @@ -54,9 +42,7 @@ def build_predict_graph(self): # deep tower deep_tower_config = self._model_config.deep_tower - dnn_layer = dnn.DNN( - deep_tower_config.dnn, self._l2_reg, 'dnn', self._is_training - ) + dnn_layer = dnn.DNN(deep_tower_config.dnn, self._l2_reg, 'dnn', self._is_training) deep_tensor = dnn_layer(self._features) tower_fea_arr.append(deep_tensor) # cross tower @@ -66,10 +52,7 @@ def build_predict_graph(self): tower_fea_arr.append(cross_tensor) # final tower all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn_layer = dnn.DNN( - self._model_config.final_dnn, self._l2_reg, 'final_dnn', - self._is_training - ) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/deepfm.py b/easy_rec/python/model/deepfm.py index ffbef4a87..c3a41dae3 100644 --- a/easy_rec/python/model/deepfm.py +++ b/easy_rec/python/model/deepfm.py @@ -13,33 +13,20 @@ class DeepFM(RankModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(DeepFM, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(DeepFM, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'deepfm', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model') ) - assert self._model_config.WhichOneof('model') == 'deepfm', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.deepfm assert isinstance(self._model_config, DeepFMConfig) # backward compatibility if self._model_config.HasField('wide_regularization'): - tf.logging.warn( - 'wide_regularization is deprecated, please use l2_regularization' - ) + tf.logging.warn('wide_regularization is deprecated, please use l2_regularization') self._wide_features, _ = self._input_layer(self._feature_dict, 'wide') - self._deep_features, self._fm_features = self._input_layer( - self._feature_dict, 'deep' - ) + self._deep_features, self._fm_features = self._input_layer(self._feature_dict, 'deep') if 'fm' in self._input_layer._feature_groups: _, self._fm_features = self._input_layer(self._feature_dict, 'fm') @@ -50,56 +37,45 @@ def build_input_layer(self, model_config, feature_configs): assert model_config.deepfm.wide_output_dim == model_config.num_class self._wide_output_dim = model_config.deepfm.wide_output_dim if self._wide_output_dim != self._num_class: - logging.warning( - 'wide_output_dim not equal to 1, it is not a standard model' - ) + logging.warning('wide_output_dim not equal to 1, it is not a standard model') super(DeepFM, self).build_input_layer(model_config, feature_configs) def build_predict_graph(self): # Wide if self._num_class > 1 and self._wide_output_dim == self._num_class: wide_shape = tf.shape(self._wide_features) - new_shape = tf.stack( - [-1, wide_shape[1] // self._num_class, self._num_class] - ) + new_shape = tf.stack([-1, wide_shape[1] // self._num_class, self._num_class]) wide_fea = tf.reshape(self._wide_features, new_shape) wide_fea = tf.reduce_sum(wide_fea, axis=1, name='wide_feature') else: - wide_fea = tf.reduce_sum( - self._wide_features, axis=1, keepdims=True, name='wide_feature' - ) + wide_fea = tf.reduce_sum(self._wide_features, axis=1, keepdims=True, name='wide_feature') # FM fm_fea = fm.FM(name='fm_feature')(self._fm_features) self._fm_outputs = fm_fea # Deep - deep_layer = dnn.DNN( - self._model_config.dnn, self._l2_reg, 'deep_feature', self._is_training - ) + deep_layer = dnn.DNN(self._model_config.dnn, self._l2_reg, 'deep_feature', self._is_training) deep_fea = deep_layer(self._deep_features) # Final if len(self._model_config.final_dnn.hidden_units) > 0: all_fea = tf.concat([wide_fea, fm_fea, deep_fea], axis=1) final_dnn_layer = dnn.DNN( - self._model_config.final_dnn, self._l2_reg, 'final_dnn', - self._is_training + self._model_config.final_dnn, + self._l2_reg, + 'final_dnn', + self._is_training, ) all_fea = final_dnn_layer(all_fea) - output = tf.layers.dense( - all_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='output' - ) + output = tf.layers.dense(all_fea, self._num_class, kernel_regularizer=self._l2_reg, name='output') else: if self._num_class > 1: fm_fea = tf.layers.dense( fm_fea, self._num_class, kernel_regularizer=self._l2_reg, - name='fm_logits' + name='fm_logits', ) else: fm_fea = tf.reduce_sum(fm_fea, 1, keepdims=True) @@ -107,7 +83,7 @@ def build_predict_graph(self): deep_fea, self._num_class, kernel_regularizer=self._l2_reg, - name='deep_logits' + name='deep_logits', ) output = wide_fea + fm_fea + deep_fea @@ -119,16 +95,9 @@ def build_feature_output_dict(self): outputs = super(DeepFM, self).build_feature_output_dict() outputs.update( { - 'wide_features': - tf.reduce_join( - tf.as_string(self._wide_features), axis=-1, separator=',' - ), - 'deep_features': - tf.reduce_join( - tf.as_string(self._deep_features), axis=-1, separator=',' - ), - 'fm_outputs': - tf.reduce_join(tf.as_string(self._fm_outputs), axis=-1, separator=',') + 'wide_features': tf.reduce_join(tf.as_string(self._wide_features), axis=-1, separator=','), + 'deep_features': tf.reduce_join(tf.as_string(self._deep_features), axis=-1, separator=','), + 'fm_outputs': tf.reduce_join(tf.as_string(self._fm_outputs), axis=-1, separator=','), } ) return outputs diff --git a/easy_rec/python/model/dlrm.py b/easy_rec/python/model/dlrm.py index 00ad46287..7d2a8df95 100755 --- a/easy_rec/python/model/dlrm.py +++ b/easy_rec/python/model/dlrm.py @@ -6,7 +6,6 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.dlrm_pb2 import DLRM as DLRMConfig # NOQA if tf.__version__ >= '2.0': @@ -16,42 +15,30 @@ class DLRM(RankModel): """Implements Deep Learning Recommendation Model for Personalization and Recommendation Systems(FaceBook).""" - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(DLRM, self).__init__( - model_config, feature_configs, features, labels, is_training - ) - assert model_config.WhichOneof('model') == 'dlrm', \ - 'invalid model config: %s' % model_config.WhichOneof('model') + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(DLRM, self).__init__(model_config, feature_configs, features, labels, is_training) + assert model_config.WhichOneof('model') == 'dlrm', 'invalid model config: %s' % model_config.WhichOneof('model') self._model_config = model_config.dlrm assert isinstance(self._model_config, DLRMConfig) - assert self._input_layer.has_group( - 'sparse' - ), 'sparse group is not specified' + assert self._input_layer.has_group('sparse'), 'sparse group is not specified' _, self._sparse_features = self._input_layer(self._feature_dict, 'sparse') assert self._input_layer.has_group('dense'), 'dense group is not specified' self._dense_feature, _ = self._input_layer(self._feature_dict, 'dense') def build_predict_graph(self): - bot_dnn = dnn.DNN( - self._model_config.bot_dnn, self._l2_reg, 'bot_dnn', self._is_training - ) + bot_dnn = dnn.DNN(self._model_config.bot_dnn, self._l2_reg, 'bot_dnn', self._is_training) dense_fea = bot_dnn(self._dense_feature) - logging.info( - 'arch_interaction_op = %s' % self._model_config.arch_interaction_op - ) + logging.info('arch_interaction_op = %s' % self._model_config.arch_interaction_op) if self._model_config.arch_interaction_op == 'cat': all_fea = tf.concat([dense_fea] + self._sparse_features, axis=1) elif self._model_config.arch_interaction_op == 'dot': - assert dense_fea.get_shape()[1] == self._sparse_features[0].get_shape()[1], \ - 'bot_dnn last hidden[%d] != sparse feature embedding_dim[%d]' % ( - dense_fea.get_shape()[1], self._sparse_features[0].get_shape()[1]) + assert dense_fea.get_shape()[1] == self._sparse_features[0].get_shape()[1], ( + 'bot_dnn last hidden[%d] != sparse feature embedding_dim[%d]' + % ( + dense_fea.get_shape()[1], + self._sparse_features[0].get_shape()[1], + ) + ) all_feas = [dense_fea] + self._sparse_features all_feas = [x[:, None, :] for x in all_feas] @@ -61,20 +48,16 @@ def build_predict_graph(self): offset = 0 if self._model_config.arch_interaction_itself else 1 upper_tri = [] for i in range(num_fea): - upper_tri.append(interaction[:, i, (i + offset):num_fea]) + upper_tri.append(interaction[:, i, (i + offset) : num_fea]) upper_tri = tf.concat(upper_tri, axis=1) concat_feas = [upper_tri] + self._sparse_features if self._model_config.arch_with_dense_feature: concat_feas.append(dense_fea) all_fea = tf.concat(concat_feas, axis=1) - top_dnn = dnn.DNN( - self._model_config.top_dnn, self._l2_reg, 'top_dnn', self._is_training - ) + top_dnn = dnn.DNN(self._model_config.top_dnn, self._l2_reg, 'top_dnn', self._is_training) all_fea = top_dnn(all_fea) - logits = tf.layers.dense( - all_fea, 1, kernel_regularizer=self._l2_reg, name='output' - ) + logits = tf.layers.dense(all_fea, 1, kernel_regularizer=self._l2_reg, name='output') self._add_to_prediction_dict(logits) diff --git a/easy_rec/python/model/dropoutnet.py b/easy_rec/python/model/dropoutnet.py index 98b4437c2..d77d4d709 100644 --- a/easy_rec/python/model/dropoutnet.py +++ b/easy_rec/python/model/dropoutnet.py @@ -4,22 +4,21 @@ from easy_rec.python.layers import dnn from easy_rec.python.loss.pairwise_loss import pairwise_loss +from easy_rec.python.loss.softmax_loss_with_negative_mining import ( # NOQA + softmax_loss_with_negative_mining, +) from easy_rec.python.model.easy_rec_model import EasyRecModel +from easy_rec.python.protos.dropoutnet_pb2 import DropoutNet as DropoutNetConfig # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.utils.proto_util import copy_obj -from easy_rec.python.loss.softmax_loss_with_negative_mining import softmax_loss_with_negative_mining # NOQA -from easy_rec.python.protos.dropoutnet_pb2 import DropoutNet as DropoutNetConfig # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses def cosine_similarity(user_emb, item_emb): - user_item_sim = tf.reduce_sum( - tf.multiply(user_emb, item_emb), axis=1, name='cosine' - ) + user_item_sim = tf.reduce_sum(tf.multiply(user_emb, item_emb), axis=1, name='cosine') return user_item_sim @@ -33,23 +32,11 @@ def bernoulli_dropout(x, rate, training=False): class DropoutNet(EasyRecModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(DropoutNet, self).__init__( - model_config, feature_configs, features, labels, is_training - ) + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(DropoutNet, self).__init__(model_config, feature_configs, features, labels, is_training) self._losses = self._model_config.losses - assert self._model_config.WhichOneof( - 'model' - ) == 'dropoutnet', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model' + assert self._model_config.WhichOneof('model') == 'dropoutnet', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model') ) self._model_config = self._model_config.dropoutnet assert isinstance(self._model_config, DropoutNetConfig) @@ -60,13 +47,9 @@ def __init__( self.user_tower_layers = copy_obj(self._model_config.user_tower) self.user_content_feature, self.user_preference_feature = None, None if self._input_layer.has_group('user_content'): - self.user_content_feature, _ = self._input_layer( - self._feature_dict, 'user_content' - ) + self.user_content_feature, _ = self._input_layer(self._feature_dict, 'user_content') if self._input_layer.has_group('user_preference'): - self.user_preference_feature, _ = self._input_layer( - self._feature_dict, 'user_preference' - ) + self.user_preference_feature, _ = self._input_layer(self._feature_dict, 'user_preference') assert self.user_content_feature is not None or self.user_preference_feature is not None, 'no user feature' # copy_obj so that any modification will not affect original config @@ -75,13 +58,9 @@ def __init__( self.item_tower_layers = copy_obj(self._model_config.item_tower) self.item_content_feature, self.item_preference_feature = None, None if self._input_layer.has_group('item_content'): - self.item_content_feature, _ = self._input_layer( - self._feature_dict, 'item_content' - ) + self.item_content_feature, _ = self._input_layer(self._feature_dict, 'item_content') if self._input_layer.has_group('item_preference'): - self.item_preference_feature, _ = self._input_layer( - self._feature_dict, 'item_preference' - ) + self.item_preference_feature, _ = self._input_layer(self._feature_dict, 'item_preference') assert self.item_content_feature is not None or self.item_preference_feature is not None, 'no item feature' def build_predict_graph(self): @@ -96,34 +75,37 @@ def build_predict_graph(self): user_features = [] if self.user_content_feature is not None: user_content_dnn = dnn.DNN( - self.user_content_layers, self._l2_reg, 'user_content', - self._is_training + self.user_content_layers, + self._l2_reg, + 'user_content', + self._is_training, ) content_feature = user_content_dnn(self.user_content_feature) user_features.append(content_feature) if self.user_preference_feature is not None: user_prefer_feature = bernoulli_dropout( - self.user_preference_feature, self._model_config.user_dropout_rate, - self._is_training + self.user_preference_feature, + self._model_config.user_dropout_rate, + self._is_training, ) user_prefer_dnn = dnn.DNN( - self.user_preference_layers, self._l2_reg, 'user_preference', - self._is_training + self.user_preference_layers, + self._l2_reg, + 'user_preference', + self._is_training, ) prefer_feature = user_prefer_dnn(user_prefer_feature) user_features.append(prefer_feature) user_tower_feature = tf.concat(user_features, axis=-1) - user_dnn = dnn.DNN( - self.user_tower_layers, self._l2_reg, 'user_dnn', self._is_training - ) + user_dnn = dnn.DNN(self.user_tower_layers, self._l2_reg, 'user_dnn', self._is_training) user_hidden = user_dnn(user_tower_feature) user_tower_emb = tf.layers.dense( inputs=user_hidden, units=last_user_hidden, kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1) + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), ) # --------------------------build item tower----------------------------------- @@ -131,34 +113,37 @@ def build_predict_graph(self): item_features = [] if self.item_content_feature is not None: item_content_dnn = dnn.DNN( - self.item_content_layers, self._l2_reg, 'item_content', - self._is_training + self.item_content_layers, + self._l2_reg, + 'item_content', + self._is_training, ) content_feature = item_content_dnn(self.item_content_feature) item_features.append(content_feature) if self.item_preference_feature is not None: item_prefer_feature = bernoulli_dropout( - self.item_preference_feature, self._model_config.item_dropout_rate, - self._is_training + self.item_preference_feature, + self._model_config.item_dropout_rate, + self._is_training, ) item_prefer_dnn = dnn.DNN( - self.item_preference_layers, self._l2_reg, 'item_preference', - self._is_training + self.item_preference_layers, + self._l2_reg, + 'item_preference', + self._is_training, ) prefer_feature = item_prefer_dnn(item_prefer_feature) item_features.append(prefer_feature) item_tower_feature = tf.concat(item_features, axis=-1) - item_dnn = dnn.DNN( - self.item_tower_layers, self._l2_reg, 'item_dnn', self._is_training - ) + item_dnn = dnn.DNN(self.item_tower_layers, self._l2_reg, 'item_dnn', self._is_training) item_hidden = item_dnn(item_tower_feature) item_tower_emb = tf.layers.dense( inputs=item_hidden, units=last_item_hidden, kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1) + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), ) user_emb = tf.nn.l2_normalize(user_tower_emb, axis=-1) @@ -167,12 +152,8 @@ def build_predict_graph(self): self._prediction_dict['similarity'] = cosine self._prediction_dict['float_user_emb'] = user_emb self._prediction_dict['float_item_emb'] = item_emb - self._prediction_dict['user_emb'] = tf.reduce_join( - tf.as_string(user_emb), axis=-1, separator=',' - ) - self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_emb), axis=-1, separator=',' - ) + self._prediction_dict['user_emb'] = tf.reduce_join(tf.as_string(user_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_emb), axis=-1, separator=',') return self._prediction_dict def build_loss_graph(self): @@ -180,9 +161,7 @@ def build_loss_graph(self): logits = self._prediction_dict['similarity'] for loss in self._losses: if loss.loss_type == LossType.SOFTMAX_CROSS_ENTROPY_WITH_NEGATIVE_MINING: - assert self._model_config.HasField( - 'softmax_loss' - ), '`softmax_loss` must be configured' + assert self._model_config.HasField('softmax_loss'), '`softmax_loss` must be configured' user_emb = self._prediction_dict['float_user_emb'] item_emb = self._prediction_dict['float_item_emb'] loss_value = softmax_loss_with_negative_mining( @@ -194,21 +173,20 @@ def build_loss_graph(self): weights=self._sample_weight, margin=self._model_config.softmax_loss.margin, gamma=self._model_config.softmax_loss.gamma, - t=self._model_config.softmax_loss.coefficient_of_support_vector + t=self._model_config.softmax_loss.coefficient_of_support_vector, ) self._loss_dict['softmax_loss'] = loss_value * loss.weight elif loss.loss_type == LossType.PAIR_WISE_LOSS: loss_value = pairwise_loss(labels, logits) self._loss_dict['pairwise_loss'] = loss_value * loss.weight elif loss.loss_type == LossType.CLASSIFICATION: - loss_value = tf.losses.sigmoid_cross_entropy( - labels, logits, self._sample_weight - ) + loss_value = tf.losses.sigmoid_cross_entropy(labels, logits, self._sample_weight) self._loss_dict['sigmoid_loss'] = loss_value * loss.weight return self._loss_dict def build_metric_graph(self, eval_config): from easy_rec.python.core.easyrec_metrics import metrics_tf as metrics + metric_dict = {} labels = list(self._labels.values())[0] sim_score = self._prediction_dict['similarity'] @@ -216,21 +194,13 @@ def build_metric_graph(self, eval_config): predict = tf.greater(prob, 0.5) for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'auc': - metric_dict['auc'] = metrics.auc( - labels, prob, weights=self._sample_weight - ) + metric_dict['auc'] = metrics.auc(labels, prob, weights=self._sample_weight) elif metric.WhichOneof('metric') == 'accuracy': - metric_dict['accuracy'] = metrics.accuracy( - tf.cast(labels, tf.bool), predict, weights=self._sample_weight - ) + metric_dict['accuracy'] = metrics.accuracy(tf.cast(labels, tf.bool), predict, weights=self._sample_weight) elif metric.WhichOneof('metric') == 'precision': - metric_dict['precision'] = metrics.precision( - labels, predict, weights=self._sample_weight - ) + metric_dict['precision'] = metrics.precision(labels, predict, weights=self._sample_weight) elif metric.WhichOneof('metric') == 'recall': - metric_dict['recall'] = metrics.recall( - labels, predict, weights=self._sample_weight - ) + metric_dict['recall'] = metrics.recall(labels, predict, weights=self._sample_weight) else: ValueError('invalid metric type: %s' % str(metric)) return metric_dict diff --git a/easy_rec/python/model/dssm.py b/easy_rec/python/model/dssm.py index 2dbeb282e..dba07d170 100644 --- a/easy_rec/python/model/dssm.py +++ b/easy_rec/python/model/dssm.py @@ -15,20 +15,11 @@ class DSSM(MatchModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(DSSM, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(DSSM, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'dssm', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model' ) - assert self._model_config.WhichOneof('model') == 'dssm', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.dssm assert isinstance(self._model_config, DSSMConfig) @@ -44,28 +35,24 @@ def __init__( def build_predict_graph(self): num_user_dnn_layer = len(self.user_tower.dnn.hidden_units) last_user_hidden = self.user_tower.dnn.hidden_units.pop() - user_dnn = dnn.DNN( - self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training - ) + user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training) user_tower_emb = user_dnn(self.user_tower_feature) user_tower_emb = tf.layers.dense( inputs=user_tower_emb, units=last_user_hidden, kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1) + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), ) num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() - item_dnn = dnn.DNN( - self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training - ) + item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training) item_tower_emb = item_dnn(self.item_tower_feature) item_tower_emb = tf.layers.dense( inputs=item_tower_emb, units=last_item_hidden, kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1) + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), ) if self._model_config.simi_func == Similarity.COSINE: @@ -77,18 +64,8 @@ def build_predict_graph(self): user_item_sim = self.sim(user_tower_emb, item_tower_emb) / temperature if self._model_config.scale_simi: - sim_w = tf.get_variable( - 'sim_w', - dtype=tf.float32, - shape=(1), - initializer=tf.ones_initializer() - ) - sim_b = tf.get_variable( - 'sim_b', - dtype=tf.float32, - shape=(1), - initializer=tf.zeros_initializer() - ) + sim_w = tf.get_variable('sim_w', dtype=tf.float32, shape=(1), initializer=tf.ones_initializer()) + sim_b = tf.get_variable('sim_b', dtype=tf.float32, shape=(1), initializer=tf.zeros_initializer()) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -108,30 +85,30 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb - self._prediction_dict['user_emb'] = tf.reduce_join( - tf.as_string(user_tower_emb), axis=-1, separator=',' - ) - self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_tower_emb), axis=-1, separator=',' - ) + self._prediction_dict['user_emb'] = tf.reduce_join(tf.as_string(user_tower_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_tower_emb), axis=-1, separator=',') return self._prediction_dict def get_outputs(self): if self._loss_type == LossType.CLASSIFICATION: return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_tower_emb', + 'item_tower_emb', ] elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: - self._prediction_dict['logits'] = tf.squeeze( - self._prediction_dict['logits'], axis=-1 - ) - self._prediction_dict['probs'] = tf.nn.sigmoid( - self._prediction_dict['logits'] - ) + self._prediction_dict['logits'] = tf.squeeze(self._prediction_dict['logits'], axis=-1) + self._prediction_dict['probs'] = tf.nn.sigmoid(self._prediction_dict['logits']) return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_tower_emb', + 'item_tower_emb', ] elif self._loss_type == LossType.L2_LOSS: return ['y', 'user_emb', 'item_emb', 'user_tower_emb', 'item_tower_emb'] @@ -140,36 +117,24 @@ def get_outputs(self): def build_output_dict(self): output_dict = super(DSSM, self).build_output_dict() - output_dict['user_tower_feature'] = tf.reduce_join( - tf.as_string(self.user_tower_feature), axis=-1, separator=',' - ) - output_dict['item_tower_feature'] = tf.reduce_join( - tf.as_string(self.item_tower_feature), axis=-1, separator=',' - ) + output_dict['user_tower_feature'] = tf.reduce_join(tf.as_string(self.user_tower_feature), axis=-1, separator=',') + output_dict['item_tower_feature'] = tf.reduce_join(tf.as_string(self.item_tower_feature), axis=-1, separator=',') return output_dict def build_rtp_output_dict(self): output_dict = super(DSSM, self).build_rtp_output_dict() if 'user_tower_emb' not in self._prediction_dict: - raise ValueError( - 'User tower embedding does not exist. Please checking predict graph.' - ) + raise ValueError('User tower embedding does not exist. Please checking predict graph.') output_dict['user_embedding_output'] = tf.identity( self._prediction_dict['user_tower_emb'], name='user_embedding_output' ) if 'item_tower_emb' not in self._prediction_dict: - raise ValueError( - 'Item tower embedding does not exist. Please checking predict graph.' - ) + raise ValueError('Item tower embedding does not exist. Please checking predict graph.') output_dict['item_embedding_output'] = tf.identity( self._prediction_dict['item_tower_emb'], name='item_embedding_output' ) if self._loss_type == LossType.CLASSIFICATION: if 'probs' not in self._prediction_dict: - raise ValueError( - 'Probs output does not exist. Please checking predict graph.' - ) - output_dict['rank_predict'] = tf.identity( - self._prediction_dict['probs'], name='rank_predict' - ) + raise ValueError('Probs output does not exist. Please checking predict graph.') + output_dict['rank_predict'] = tf.identity(self._prediction_dict['probs'], name='rank_predict') return output_dict diff --git a/easy_rec/python/model/dssm_senet.py b/easy_rec/python/model/dssm_senet.py index 626b4f02c..3737c897e 100644 --- a/easy_rec/python/model/dssm_senet.py +++ b/easy_rec/python/model/dssm_senet.py @@ -5,51 +5,44 @@ from easy_rec.python.layers import dnn, senet from easy_rec.python.model.dssm import DSSM from easy_rec.python.model.match_model import MatchModel +from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.protos.simi_pb2 import Similarity from easy_rec.python.utils.proto_util import copy_obj -from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses class DSSM_SENet(DSSM): + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + MatchModel.__init__(self, model_config, feature_configs, features, labels, is_training) - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - - MatchModel.__init__( - self, model_config, feature_configs, features, labels, is_training + assert self._model_config.WhichOneof('model') == 'dssm_senet', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model') ) - - assert self._model_config.WhichOneof('model') == 'dssm_senet', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.dssm_senet assert isinstance(self._model_config, DSSM_SENet_Config) # copy_obj so that any modification will not affect original config self.user_tower = copy_obj(self._model_config.user_tower) - self.user_seq_features, self.user_plain_features, self.user_feature_list = self._input_layer( - self._feature_dict, 'user', is_combine=False - ) + ( + self.user_seq_features, + self.user_plain_features, + self.user_feature_list, + ) = self._input_layer(self._feature_dict, 'user', is_combine=False) self.user_num_fields = len(self.user_feature_list) # copy_obj so that any modification will not affect original config self.item_tower = copy_obj(self._model_config.item_tower) - self.item_seq_features, self.item_plain_features, self.item_feature_list = self._input_layer( - self._feature_dict, 'item', is_combine=False - ) + ( + self.item_seq_features, + self.item_plain_features, + self.item_feature_list, + ) = self._input_layer(self._feature_dict, 'item', is_combine=False) self.item_num_fields = len(self.item_feature_list) self._user_tower_emb = None @@ -61,22 +54,20 @@ def build_predict_graph(self): num_squeeze_group=self.user_tower.senet.num_squeeze_group, reduction_ratio=self.user_tower.senet.reduction_ratio, l2_reg=self._l2_reg, - name='user_senet' + name='user_senet', ) user_senet_output_list = user_senet(self.user_feature_list) user_senet_output = tf.concat(user_senet_output_list, axis=-1) num_user_dnn_layer = len(self.user_tower.dnn.hidden_units) last_user_hidden = self.user_tower.dnn.hidden_units.pop() - user_dnn = dnn.DNN( - self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training - ) + user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training) user_tower_emb = user_dnn(user_senet_output) user_tower_emb = tf.layers.dense( inputs=user_tower_emb, units=last_user_hidden, kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1) + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), ) item_senet = senet.SENet( @@ -84,7 +75,7 @@ def build_predict_graph(self): num_squeeze_group=self.item_tower.senet.num_squeeze_group, reduction_ratio=self.item_tower.senet.reduction_ratio, l2_reg=self._l2_reg, - name='item_senet' + name='item_senet', ) item_senet_output_list = item_senet(self.item_feature_list) @@ -92,15 +83,13 @@ def build_predict_graph(self): num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() - item_dnn = dnn.DNN( - self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training - ) + item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training) item_tower_emb = item_dnn(item_senet_output) item_tower_emb = tf.layers.dense( inputs=item_tower_emb, units=last_item_hidden, kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1) + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), ) if self._model_config.simi_func == Similarity.COSINE: @@ -112,18 +101,8 @@ def build_predict_graph(self): user_item_sim = self.sim(user_tower_emb, item_tower_emb) / temperature if self._model_config.scale_simi: - sim_w = tf.get_variable( - 'sim_w', - dtype=tf.float32, - shape=(1), - initializer=tf.ones_initializer() - ) - sim_b = tf.get_variable( - 'sim_b', - dtype=tf.float32, - shape=(1), - initializer=tf.zeros_initializer() - ) + sim_w = tf.get_variable('sim_w', dtype=tf.float32, shape=(1), initializer=tf.ones_initializer()) + sim_b = tf.get_variable('sim_b', dtype=tf.float32, shape=(1), initializer=tf.zeros_initializer()) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -143,12 +122,8 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb - self._prediction_dict['user_emb'] = tf.reduce_join( - tf.as_string(user_tower_emb), axis=-1, separator=',' - ) - self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_tower_emb), axis=-1, separator=',' - ) + self._prediction_dict['user_emb'] = tf.reduce_join(tf.as_string(user_tower_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_tower_emb), axis=-1, separator=',') return self._prediction_dict def build_output_dict(self): diff --git a/easy_rec/python/model/dummy_model.py b/easy_rec/python/model/dummy_model.py index ca8b97305..9e34c82a3 100644 --- a/easy_rec/python/model/dummy_model.py +++ b/easy_rec/python/model/dummy_model.py @@ -7,18 +7,8 @@ class DummyModel(EasyRecModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(DummyModel, self).__init__( - model_config, feature_configs, features, labels, is_training - ) + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(DummyModel, self).__init__(model_config, feature_configs, features, labels, is_training) if self._labels is not None: self._labels = list(self._labels.values()) @@ -38,12 +28,7 @@ def build_predict_graph(self): return self._prediction_dict def build_loss_graph(self): - return { - 'cross_ent': - tf.reduce_sum( - tf.square(self._prediction_dict['output'] - self._labels[0]) - ) - } + return {'cross_ent': tf.reduce_sum(tf.square(self._prediction_dict['output'] - self._labels[0]))} def get_outputs(self): return ['output'] diff --git a/easy_rec/python/model/easy_rec_estimator.py b/easy_rec/python/model/easy_rec_estimator.py index a9d20d137..5305f08c1 100644 --- a/easy_rec/python/model/easy_rec_estimator.py +++ b/easy_rec/python/model/easy_rec_estimator.py @@ -20,17 +20,31 @@ from easy_rec.python.builders import optimizer_builder from easy_rec.python.compat import optimizers, sync_replicas_optimizer +from easy_rec.python.compat.early_stopping import ( # NOQA + custom_early_stop_hook, + deadline_stop_hook, + find_early_stop_var, + oss_stop_hook, + stop_if_no_decrease_hook, + stop_if_no_increase_hook, +) +from easy_rec.python.compat.embedding_parallel_saver import ( # NOQA + EmbeddingParallelSaver, +) from easy_rec.python.compat.ops import GraphKeys from easy_rec.python.input.input import Input from easy_rec.python.layers.utils import _tensor_to_tensorinfo from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig from easy_rec.python.protos.train_pb2 import DistributionStrategy +from easy_rec.python.utils import ( # NOQA + constant, + embedding_utils, + estimator_utils, + hvd_utils, + pai_util, +) from easy_rec.python.utils.multi_optimizer import MultiOptimizer -from easy_rec.python.compat.early_stopping import custom_early_stop_hook, deadline_stop_hook, find_early_stop_var, oss_stop_hook, stop_if_no_decrease_hook, stop_if_no_increase_hook # NOQA -from easy_rec.python.compat.embedding_parallel_saver import EmbeddingParallelSaver # NOQA -from easy_rec.python.utils import constant, embedding_utils, estimator_utils, hvd_utils, pai_util # NOQA - try: import horovod.tensorflow as hvd except Exception: @@ -50,7 +64,6 @@ class EasyRecEstimator(tf.estimator.Estimator): - def __init__(self, pipeline_config, model_cls, run_config, params): self._pipeline_config = pipeline_config self._model_cls = model_cls @@ -60,33 +73,22 @@ def __init__(self, pipeline_config, model_cls, run_config, params): model_fn=self._model_fn, model_dir=pipeline_config.model_dir, config=run_config, - params=params + params=params, ) - def evaluate( - self, input_fn, steps=None, hooks=None, checkpoint_path=None, name=None - ): + def evaluate(self, input_fn, steps=None, hooks=None, checkpoint_path=None, name=None): # support for datahub/kafka offset restore input_fn.input_creator.restore(checkpoint_path) - return super(EasyRecEstimator, - self).evaluate(input_fn, steps, hooks, checkpoint_path, name) + return super(EasyRecEstimator, self).evaluate(input_fn, steps, hooks, checkpoint_path, name) - def train( - self, - input_fn, - hooks=None, - steps=None, - max_steps=None, - saving_listeners=None - ): + def train(self, input_fn, hooks=None, steps=None, max_steps=None, saving_listeners=None): # support for datahub/kafka offset restore checkpoint_path = estimator_utils.latest_checkpoint(self.model_dir) if checkpoint_path is not None: input_fn.input_creator.restore(checkpoint_path) elif self.train_config.HasField('fine_tune_checkpoint'): fine_tune_ckpt = self.train_config.fine_tune_checkpoint - if fine_tune_ckpt.endswith('/' - ) or gfile.IsDirectory(fine_tune_ckpt + '/'): + if fine_tune_ckpt.endswith('/') or gfile.IsDirectory(fine_tune_ckpt + '/'): fine_tune_ckpt = estimator_utils.latest_checkpoint(fine_tune_ckpt) print( 'fine_tune_checkpoint[%s] is directory, will use the latest checkpoint: %s' @@ -94,16 +96,13 @@ def train( ) self.train_config.fine_tune_checkpoint = fine_tune_ckpt input_fn.input_creator.restore(fine_tune_ckpt) - return super(EasyRecEstimator, self - ).train(input_fn, hooks, steps, max_steps, saving_listeners) + return super(EasyRecEstimator, self).train(input_fn, hooks, steps, max_steps, saving_listeners) @property def feature_configs(self): if len(self._pipeline_config.feature_configs) > 0: return self._pipeline_config.feature_configs - elif self._pipeline_config.feature_config and len( - self._pipeline_config.feature_config.features - ) > 0: + elif self._pipeline_config.feature_config and len(self._pipeline_config.feature_config.features) > 0: return self._pipeline_config.feature_config.features else: assert False, 'One of feature_configs and feature_config.features must be configured.' @@ -122,9 +121,7 @@ def train_config(self): @property def incr_save_config(self): - return self.train_config.incr_save_config if self.train_config.HasField( - 'incr_save_config' - ) else None + return self.train_config.incr_save_config if self.train_config.HasField('incr_save_config') else None @property def export_config(self): @@ -134,7 +131,7 @@ def export_config(self): def embedding_parallel(self): return self.train_config.train_distribute in ( DistributionStrategy.SokStrategy, - DistributionStrategy.EmbeddingParallelStrategy + DistributionStrategy.EmbeddingParallelStrategy, ) @property @@ -148,34 +145,21 @@ def saver_cls(self): def _train_model_fn(self, features, labels, run_config): tf.keras.backend.set_learning_phase(1) - model = self._model_cls( - self.model_config, - self.feature_configs, - features, - labels, - is_training=True - ) + model = self._model_cls(self.model_config, self.feature_configs, features, labels, is_training=True) predict_dict = model.build_predict_graph() loss_dict = model.build_loss_graph() - regularization_losses = tf.get_collection( - tf.GraphKeys.REGULARIZATION_LOSSES - ) + regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) if regularization_losses: regularization_losses = [ - reg_loss.get() if hasattr(reg_loss, 'get') else reg_loss - for reg_loss in regularization_losses + reg_loss.get() if hasattr(reg_loss, 'get') else reg_loss for reg_loss in regularization_losses ] - regularization_losses = tf.add_n( - regularization_losses, name='regularization_loss' - ) + regularization_losses = tf.add_n(regularization_losses, name='regularization_loss') loss_dict['regularization_loss'] = regularization_losses variational_dropout_loss = tf.get_collection('variational_dropout_loss') if variational_dropout_loss: - variational_dropout_loss = tf.add_n( - variational_dropout_loss, name='variational_dropout_loss' - ) + variational_dropout_loss = tf.add_n(variational_dropout_loss, name='variational_dropout_loss') loss_dict['variational_dropout_loss'] = variational_dropout_loss loss = tf.add_n(list(loss_dict.values())) @@ -190,11 +174,9 @@ def _train_model_fn(self, features, labels, run_config): dtype=tf.string, shape=[task_num], collections=[tf.GraphKeys.GLOBAL_VARIABLES, Input.DATA_OFFSET], - trainable=False - ) - update_offset = tf.assign( - data_offset_var[task_index], features[Input.DATA_OFFSET] + trainable=False, ) + update_offset = tf.assign(data_offset_var[task_index], features[Input.DATA_OFFSET]) ops.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_offset) else: data_offset_var = None @@ -206,9 +188,7 @@ def _train_model_fn(self, features, labels, run_config): global_vars = {x.name: x for x in tf.global_variables()} for x in update_ops: if isinstance(x, ops.Operation) and x.inputs[0].name in global_vars: - ops.add_to_collection( - constant.DENSE_UPDATE_VARIABLES, global_vars[x.inputs[0].name] - ) + ops.add_to_collection(constant.DENSE_UPDATE_VARIABLES, global_vars[x.inputs[0].name]) update_op = tf.group(*update_ops, name='update_barrier') with tf.control_dependencies([update_op]): loss = tf.identity(loss, name='total_loss') @@ -227,9 +207,13 @@ def _train_model_fn(self, features, labels, run_config): tf.summary.scalar('learning_rate', learning_rate[0]) all_opts.append(opt) grouped_vars = model.get_grouped_vars(len(all_opts)) - assert len(grouped_vars) == len(optimizer_config), \ - 'the number of var group(%d) != the number of optimizers(%d)' \ - % (len(grouped_vars), len(optimizer_config)) + assert len(grouped_vars) == len(optimizer_config), ( + 'the number of var group(%d) != the number of optimizers(%d)' + % ( + len(grouped_vars), + len(optimizer_config), + ) + ) optimizer = MultiOptimizer(all_opts, grouped_vars) if self.train_config.train_distribute == DistributionStrategy.SokStrategy: @@ -237,42 +221,31 @@ def _train_model_fn(self, features, labels, run_config): hooks = [] if estimator_utils.has_hvd(): - assert not self.train_config.sync_replicas, \ - 'sync_replicas should not be set when using horovod' + assert not self.train_config.sync_replicas, 'sync_replicas should not be set when using horovod' bcast_hook = hvd_utils.BroadcastGlobalVariablesHook(0) hooks.append(bcast_hook) # for distributed and synced training if self.train_config.sync_replicas and run_config.num_worker_replicas > 1: - logging.info( - 'sync_replicas: num_worker_replias = %d' % - run_config.num_worker_replicas - ) + logging.info('sync_replicas: num_worker_replias = %d' % run_config.num_worker_replicas) if pai_util.is_on_pai(): optimizer = tf.train.SyncReplicasOptimizer( optimizer, replicas_to_aggregate=run_config.num_worker_replicas, total_num_replicas=run_config.num_worker_replicas, - sparse_accumulator_type=self.train_config.sparse_accumulator_type + sparse_accumulator_type=self.train_config.sparse_accumulator_type, ) else: optimizer = sync_replicas_optimizer.SyncReplicasOptimizer( optimizer, replicas_to_aggregate=run_config.num_worker_replicas, - total_num_replicas=run_config.num_worker_replicas + total_num_replicas=run_config.num_worker_replicas, ) - hooks.append( - optimizer.make_session_run_hook(run_config.is_chief, num_tokens=0) - ) + hooks.append(optimizer.make_session_run_hook(run_config.is_chief, num_tokens=0)) # add barrier for no strategy case - if run_config.num_worker_replicas > 1 and \ - self.train_config.train_distribute == DistributionStrategy.NoStrategy: - hooks.append( - estimator_utils.ExitBarrierHook( - run_config.num_worker_replicas, run_config.is_chief, self.model_dir - ) - ) + if run_config.num_worker_replicas > 1 and self.train_config.train_distribute == DistributionStrategy.NoStrategy: + hooks.append(estimator_utils.ExitBarrierHook(run_config.num_worker_replicas, run_config.is_chief, self.model_dir)) if self.export_config.enable_early_stop: eval_dir = os.path.join(self._model_dir, 'eval_val') @@ -283,7 +256,7 @@ def _train_model_fn(self, features, labels, run_config): self, eval_dir=eval_dir, custom_stop_func=self.export_config.early_stop_func, - custom_stop_func_params=self.export_config.early_stop_params + custom_stop_func_params=self.export_config.early_stop_params, ) ) elif self.export_config.metric_bigger: @@ -292,7 +265,7 @@ def _train_model_fn(self, features, labels, run_config): self, self.export_config.best_exporter_metric, self.export_config.max_check_steps, - eval_dir=eval_dir + eval_dir=eval_dir, ) ) else: @@ -301,7 +274,7 @@ def _train_model_fn(self, features, labels, run_config): self, self.export_config.best_exporter_metric, self.export_config.max_check_steps, - eval_dir=eval_dir + eval_dir=eval_dir, ) ) @@ -320,14 +293,11 @@ def _train_model_fn(self, features, labels, run_config): gradient_clipping_by_norm = None gradient_multipliers = None - if self.train_config.optimizer_config[0].HasField( - 'embedding_learning_rate_multiplier' - ): + if self.train_config.optimizer_config[0].HasField('embedding_learning_rate_multiplier'): gradient_multipliers = { - var: self.train_config.optimizer_config[0]. - embedding_learning_rate_multiplier - for var in tf.trainable_variables() if 'embedding_weights:' in var.name - or '/embedding_weights/part_' in var.name + var: self.train_config.optimizer_config[0].embedding_learning_rate_multiplier + for var in tf.trainable_variables() + if 'embedding_weights:' in var.name or '/embedding_weights/part_' in var.name } # optimize loss @@ -360,11 +330,10 @@ def _train_model_fn(self, features, labels, run_config): variables=all_train_vars, summaries=summaries, colocate_gradients_with_ops=True, - not_apply_grad_after_first_step=run_config.is_chief - and self._pipeline_config.data_config.chief_redundant, + not_apply_grad_after_first_step=run_config.is_chief and self._pipeline_config.data_config.chief_redundant, name='', # Preventing scope prefix on all variables. incr_save=(self.incr_save_config is not None), - embedding_parallel=self.embedding_parallel + embedding_parallel=self.embedding_parallel, ) # online evaluation @@ -378,11 +347,7 @@ def _train_model_fn(self, features, labels, run_config): tf.summary.scalar('%s/batch' % k, v[1]) train_op = tf.group([train_op] + list(metric_update_op_dict.values())) if estimator_utils.is_chief(): - hooks.append( - estimator_utils.OnlineEvaluationHook( - metric_dict=metric_dict, output_dir=self.model_dir - ) - ) + hooks.append(estimator_utils.OnlineEvaluationHook(metric_dict=metric_dict, output_dir=self.model_dir)) if self.train_config.HasField('fine_tune_checkpoint'): fine_tune_ckpt = self.train_config.fine_tune_checkpoint @@ -393,7 +358,7 @@ def _train_model_fn(self, features, labels, run_config): fine_tune_ckpt, include_global_step=False, ckpt_var_map_path=fine_tune_ckpt_var_map, - force_restore_shape_compatible=force_restore + force_restore_shape_compatible=force_restore, ) if restore_hook is not None: hooks.append(restore_hook) @@ -410,23 +375,20 @@ def _train_model_fn(self, features, labels, run_config): logging_hook = basic_session_run_hooks.LoggingTensorHook( logging_dict, every_n_iter=log_step_count_steps, - formatter=estimator_utils.tensor_log_format_func + formatter=estimator_utils.tensor_log_format_func, ) hooks.append(logging_hook) if self.train_config.train_distribute in [ DistributionStrategy.CollectiveAllReduceStrategy, DistributionStrategy.MirroredStrategy, - DistributionStrategy.MultiWorkerMirroredStrategy + DistributionStrategy.MultiWorkerMirroredStrategy, ]: # for multi worker strategy, we could not replace the # inner CheckpointSaverHook, so just use it. scaffold = tf.train.Scaffold() else: - var_list = ( - tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) + - tf.get_collection(tf.GraphKeys.SAVEABLE_OBJECTS) - ) + var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) + tf.get_collection(tf.GraphKeys.SAVEABLE_OBJECTS) # exclude data_offset_var var_list = [x for x in var_list if x != data_offset_var] @@ -435,9 +397,7 @@ def _train_model_fn(self, features, labels, run_config): early_stop_var = find_early_stop_var(var_list) var_list = [x for x in var_list if x != early_stop_var] - initialize_var_list = [ - x for x in var_list if 'WorkQueue' not in str(type(x)) - ] + initialize_var_list = [x for x in var_list if 'WorkQueue' not in str(type(x))] # incompatiable shape restore will not be saved in checkpoint # but must be able to restore from checkpoint @@ -449,21 +409,17 @@ def _train_model_fn(self, features, labels, run_config): if early_stop_var is not None and estimator_utils.is_chief(): local_init_ops.append(tf.initializers.variables([early_stop_var])) if len(incompatiable_shape_restore) > 0: - local_init_ops.append( - tf.initializers.variables(incompatiable_shape_restore) - ) + local_init_ops.append(tf.initializers.variables(incompatiable_shape_restore)) scaffold = tf.train.Scaffold( saver=self.saver_cls( var_list=var_list, sharded=True, max_to_keep=self.train_config.keep_checkpoint_max, - save_relative_paths=True + save_relative_paths=True, ), local_init_op=tf.group(local_init_ops), - ready_for_local_init_op=tf.report_uninitialized_variables( - var_list=initialize_var_list - ) + ready_for_local_init_op=tf.report_uninitialized_variables(var_list=initialize_var_list), ) # saver hook saver_hook = estimator_utils.CheckpointSaverHook( @@ -473,22 +429,18 @@ def _train_model_fn(self, features, labels, run_config): scaffold=scaffold, write_graph=self.train_config.write_graph, data_offset_var=data_offset_var, - increment_save_config=self.incr_save_config + increment_save_config=self.incr_save_config, ) if estimator_utils.is_chief() or self.embedding_parallel: hooks.append(saver_hook) if estimator_utils.is_chief(): hooks.append( - basic_session_run_hooks.StepCounterHook( - every_n_steps=log_step_count_steps, output_dir=self.model_dir - ) + basic_session_run_hooks.StepCounterHook(every_n_steps=log_step_count_steps, output_dir=self.model_dir) ) # profiling hook if self.train_config.is_profiling and estimator_utils.is_chief(): - profile_hook = tf.train.ProfilerHook( - save_steps=log_step_count_steps, output_dir=self.model_dir - ) + profile_hook = tf.train.ProfilerHook(save_steps=log_step_count_steps, output_dir=self.model_dir) hooks.append(profile_hook) return tf.estimator.EstimatorSpec( @@ -497,19 +449,13 @@ def _train_model_fn(self, features, labels, run_config): predictions=predict_dict, train_op=train_op, scaffold=scaffold, - training_hooks=hooks + training_hooks=hooks, ) def _eval_model_fn(self, features, labels, run_config): tf.keras.backend.set_learning_phase(0) start = time.time() - model = self._model_cls( - self.model_config, - self.feature_configs, - features, - labels, - is_training=False - ) + model = self._model_cls(self.model_config, self.feature_configs, features, labels, is_training=False) predict_dict = model.build_predict_graph() loss_dict = model.build_loss_graph() loss = tf.add_n(list(loss_dict.values())) @@ -522,41 +468,28 @@ def _eval_model_fn(self, features, labels, run_config): metric_dict['loss/loss/' + loss_key] = tf.metrics.mean(loss_tensor) tf.logging.info('metric_dict keys: %s' % metric_dict.keys()) - var_list = ( - ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + - ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS) - ) + var_list = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS) metric_variables = ops.get_collection(ops.GraphKeys.METRIC_VARIABLES) model_ready_for_local_init_op = tf.variables_initializer(metric_variables) scaffold = tf.train.Scaffold( - saver=self.saver_cls( - var_list=var_list, sharded=True, save_relative_paths=True - ), - ready_for_local_init_op=model_ready_for_local_init_op + saver=self.saver_cls(var_list=var_list, sharded=True, save_relative_paths=True), + ready_for_local_init_op=model_ready_for_local_init_op, ) end = time.time() - tf.logging.info( - 'eval graph construct finished. Time %.3fs' % (end - start) - ) + tf.logging.info('eval graph construct finished. Time %.3fs' % (end - start)) return tf.estimator.EstimatorSpec( mode=tf.estimator.ModeKeys.EVAL, loss=loss, scaffold=scaffold, predictions=predict_dict, - eval_metric_ops=metric_dict + eval_metric_ops=metric_dict, ) def _distribute_eval_model_fn(self, features, labels, run_config): tf.keras.backend.set_learning_phase(0) start = time.time() - model = self._model_cls( - self.model_config, - self.feature_configs, - features, - labels, - is_training=False - ) + model = self._model_cls(self.model_config, self.feature_configs, features, labels, is_training=False) predict_dict = model.build_predict_graph() loss_dict = model.build_loss_graph() loss = tf.add_n(list(loss_dict.values())) @@ -569,9 +502,7 @@ def _distribute_eval_model_fn(self, features, labels, run_config): tf.logging.info('metric_dict keys: %s' % metric_dict.keys()) end = time.time() - tf.logging.info( - 'eval graph construct finished. Time %.3fs' % (end - start) - ) + tf.logging.info('eval graph construct finished. Time %.3fs' % (end - start)) metric_name_list = [] for metric_i in self.eval_config.metrics_set: metric_name_list.append(metric_i.WhichOneof('metric')) @@ -591,19 +522,15 @@ def _distribute_eval_model_fn(self, features, labels, run_config): global_variables = tf.global_variables() metric_variables = tf.get_collection(tf.GraphKeys.METRIC_VARIABLES) model_ready_for_local_init_op = tf.variables_initializer(metric_variables) - remain_variables = list( - set(global_variables).difference(set(metric_variables)) - ) + remain_variables = list(set(global_variables).difference(set(metric_variables))) cur_saver = tf.train.Saver(var_list=remain_variables, sharded=True) - scaffold = tf.train.Scaffold( - saver=cur_saver, ready_for_local_init_op=model_ready_for_local_init_op - ) + scaffold = tf.train.Scaffold(saver=cur_saver, ready_for_local_init_op=model_ready_for_local_init_op) return tf.estimator.EstimatorSpec( mode=tf.estimator.ModeKeys.EVAL, loss=loss, predictions=predict_dict, eval_metric_ops=metric_dict, - scaffold=scaffold + scaffold=scaffold, ) def _export_model_fn(self, features, labels, run_config, params): @@ -613,7 +540,7 @@ def _export_model_fn(self, features, labels, run_config, params): self.feature_configs, features, labels=None, - is_training=False + is_training=False, ) model.build_predict_graph() @@ -629,64 +556,50 @@ def _export_model_fn(self, features, labels, run_config, params): outputs.update(model.build_rtp_output_dict()) for out in outputs: - tf.logging.info( - 'output %s shape: %s type: %s' % - (out, outputs[out].get_shape().as_list(), outputs[out].dtype) - ) - export_outputs = { - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: - tf.estimator.export.PredictOutput(outputs) - } + tf.logging.info('output %s shape: %s type: %s' % (out, outputs[out].get_shape().as_list(), outputs[out].dtype)) + export_outputs = {signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: tf.estimator.export.PredictOutput(outputs)} # save train pipeline.config for debug purpose pipeline_path = os.path.join(self._model_dir, 'pipeline.config') if gfile.Exists(pipeline_path): ops.add_to_collection( tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(pipeline_path, dtype=tf.string, name='pipeline.config') + tf.constant(pipeline_path, dtype=tf.string, name='pipeline.config'), ) else: print('train pipeline_path(%s) does not exist' % pipeline_path) # restore DENSE_UPDATE_VARIABLES collection - dense_train_var_path = os.path.join( - self.model_dir, constant.DENSE_UPDATE_VARIABLES - ) + dense_train_var_path = os.path.join(self.model_dir, constant.DENSE_UPDATE_VARIABLES) if gfile.Exists(dense_train_var_path): with gfile.GFile(dense_train_var_path, 'r') as fin: var_name_to_id_map = json.load(fin) - var_name_id_lst = [ - (x, var_name_to_id_map[x]) for x in var_name_to_id_map - ] + var_name_id_lst = [(x, var_name_to_id_map[x]) for x in var_name_to_id_map] var_name_id_lst.sort(key=lambda x: x[1]) all_vars = {x.op.name: x for x in tf.global_variables()} for var_name, var_id in var_name_id_lst: assert var_name in all_vars, 'dense_train_var[%s] is not found' % var_name - ops.add_to_collection( - constant.DENSE_UPDATE_VARIABLES, all_vars[var_name] - ) + ops.add_to_collection(constant.DENSE_UPDATE_VARIABLES, all_vars[var_name]) # add more asset files if len(export_config.asset_files) > 0: for asset_file in export_config.asset_files: if asset_file.startswith('!'): asset_file = asset_file[1:] - if ':' not in asset_file or asset_file.startswith( - 'oss:' - ) or asset_file.startswith('hdfs:'): + if ':' not in asset_file or asset_file.startswith('oss:') or asset_file.startswith('hdfs:'): _, asset_name = os.path.split(asset_file) else: asset_name, asset_file = asset_file.split(':', 1) ops.add_to_collection( ops.GraphKeys.ASSET_FILEPATHS, - tf.constant(asset_file, dtype=tf.string, name=asset_name) + tf.constant(asset_file, dtype=tf.string, name=asset_name), ) elif 'asset_files' in params: for asset_name in params['asset_files']: asset_file = params['asset_files'][asset_name] ops.add_to_collection( tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(asset_file, dtype=tf.string, name=asset_name) + tf.constant(asset_file, dtype=tf.string, name=asset_name), ) if self._pipeline_config.HasField('fg_json_path'): @@ -695,25 +608,19 @@ def _export_model_fn(self, features, labels, run_config, params): fg_path = fg_path[1:] ops.add_to_collection( tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(fg_path, dtype=tf.string, name='fg.json') + tf.constant(fg_path, dtype=tf.string, name='fg.json'), ) - var_list = ( - ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + - ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS) - ) + var_list = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS) - scaffold = tf.train.Scaffold( - saver=self. - saver_cls(var_list=var_list, sharded=True, save_relative_paths=True) - ) + scaffold = tf.train.Scaffold(saver=self.saver_cls(var_list=var_list, sharded=True, save_relative_paths=True)) return tf.estimator.EstimatorSpec( mode=tf.estimator.ModeKeys.PREDICT, loss=None, scaffold=scaffold, predictions=outputs, - export_outputs=export_outputs + export_outputs=export_outputs, ) def _model_fn(self, features, labels, mode, config, params): @@ -722,9 +629,7 @@ def _model_fn(self, features, labels, mode, config, params): if self._pipeline_config.feature_config.embedding_on_cpu: os.environ['place_embedding_on_cpu'] = 'True' if self._pipeline_config.fg_json_path: - EasyRecEstimator._write_rtp_fg_config_to_col( - fg_config_path=self._pipeline_config.fg_json_path - ) + EasyRecEstimator._write_rtp_fg_config_to_col(fg_config_path=self._pipeline_config.fg_json_path) EasyRecEstimator._write_rtp_inputs_to_col(features) if self.embedding_parallel: @@ -778,27 +683,23 @@ def export_checkpoint( export_path=None, serving_input_receiver_fn=None, checkpoint_path=None, - mode=tf.estimator.ModeKeys.PREDICT + mode=tf.estimator.ModeKeys.PREDICT, ): with context.graph_mode(): if not checkpoint_path: # Locate the latest checkpoint checkpoint_path = estimator_utils.latest_checkpoint(self._model_dir) if not checkpoint_path: - raise ValueError( - "Couldn't find trained model at %s." % self._model_dir - ) + raise ValueError("Couldn't find trained model at %s." % self._model_dir) with ops.Graph().as_default(): input_receiver = serving_input_receiver_fn() estimator_spec = self._call_model_fn( features=input_receiver.features, labels=getattr(input_receiver, 'labels', None), mode=mode, - config=self.config + config=self.config, ) with tf_session.Session(config=self._session_config) as session: - graph_saver = estimator_spec.scaffold.saver or saver.Saver( - sharded=True - ) + graph_saver = estimator_spec.scaffold.saver or saver.Saver(sharded=True) graph_saver.restore(session, checkpoint_path) graph_saver.save(session, export_path) diff --git a/easy_rec/python/model/easy_rec_model.py b/easy_rec/python/model/easy_rec_model.py index df48e9371..4ea3dd77b 100644 --- a/easy_rec/python/model/easy_rec_model.py +++ b/easy_rec/python/model/easy_rec_model.py @@ -30,6 +30,7 @@ from tensorflow.python.framework.load_library import load_op_library import easy_rec + load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) except Exception as ex: @@ -40,21 +41,11 @@ tf = tf.compat.v1 _EASY_REC_MODEL_CLASS_MAP = {} -_meta_type = get_register_class_meta( - _EASY_REC_MODEL_CLASS_MAP, have_abstract_class=True -) +_meta_type = get_register_class_meta(_EASY_REC_MODEL_CLASS_MAP, have_abstract_class=True) class EasyRecModel(six.with_metaclass(_meta_type, object)): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): self._base_model_config = model_config self._model_config = model_config self._is_training = is_training @@ -67,9 +58,7 @@ def __init__( self._global_ev_params = model_config.ev_params if self.embedding_regularization > 0: - self._emb_reg = regularizers.l2_regularizer( - self.embedding_regularization - ) + self._emb_reg = regularizers.l2_regularizer(self.embedding_regularization) else: self._emb_reg = None @@ -108,7 +97,7 @@ def build_backbone_network(self): self._base_model_config.backbone, self._feature_dict, input_layer=self._input_layer, - l2_reg=self._l2_reg + l2_reg=self._l2_reg, ) return None @@ -126,7 +115,7 @@ def backbone(self): 'metric_dict': self._metric_dict, 'prediction_dict': self._prediction_dict, 'labels': self._labels, - constant.SAMPLE_WEIGHT: self._sample_weight + constant.SAMPLE_WEIGHT: self._sample_weight, } return self._backbone_net(self._is_training, **kwargs) return None @@ -145,16 +134,11 @@ def feature_groups(self): @property def l2_regularization(self): - model_config = getattr( - self._base_model_config, self._base_model_config.WhichOneof('model') - ) + model_config = getattr(self._base_model_config, self._base_model_config.WhichOneof('model')) l2_regularization = 0.0 - if hasattr(model_config, 'dense_regularization') and \ - model_config.HasField('dense_regularization'): + if hasattr(model_config, 'dense_regularization') and model_config.HasField('dense_regularization'): # backward compatibility - logging.warn( - 'dense_regularization is deprecated, please use l2_regularization' - ) + logging.warn('dense_regularization is deprecated, please use l2_regularization') l2_regularization = model_config.dense_regularization elif hasattr(model_config, 'l2_regularization'): l2_regularization = model_config.l2_regularization @@ -168,10 +152,11 @@ def build_input_layer(self, model_config, feature_configs): ev_params=self._global_ev_params, embedding_regularizer=self._emb_reg, kernel_regularizer=self._l2_reg, - variational_dropout_config=model_config.variational_dropout - if model_config.HasField('variational_dropout') else None, + variational_dropout_config=( + model_config.variational_dropout if model_config.HasField('variational_dropout') else None + ), is_training=self._is_training, - is_predicting=self._is_predicting + is_predicting=self._is_predicting, ) @abstractmethod @@ -194,10 +179,7 @@ def build_output_dict(self): outputs = {} for name in self.get_outputs(): if name not in self._prediction_dict: - raise KeyError( - 'output node {} not in prediction_dict, can not be exported'. - format(name) - ) + raise KeyError('output node {} not in prediction_dict, can not be exported'.format(name)) outputs[name] = self._prediction_dict[name] return outputs @@ -211,9 +193,7 @@ def build_feature_output_dict(self): sparse_values = feature_value.values if sparse_values.dtype != tf.string: sparse_values = tf.as_string(sparse_values) - feature_value = tf.sparse_to_dense( - feature_value.indices, feature_value.dense_shape, sparse_values, '' - ) + feature_value = tf.sparse_to_dense(feature_value.indices, feature_value.dense_shape, sparse_values, '') elif feature_value.dtype != tf.string: feature_value = tf.as_string(feature_value) feature_value = tf.reduce_join(feature_value, axis=-1, separator=',') @@ -229,7 +209,7 @@ def restore( ckpt_path, include_global_step=False, ckpt_var_map_path='', - force_restore_shape_compatible=False + force_restore_shape_compatible=False, ): """Restore variables from ckpt_path. @@ -276,20 +256,18 @@ def restore( var_shape, variable[0].dtype, variable, - partitions=[len(variable)] + [1] * (len(var_shape) - 1) + partitions=[len(variable)] + [1] * (len(var_shape) - 1), ) else: var_shape = variable.shape.as_list() if ckpt_var_shape == var_shape: - vars_in_ckpt[variable_name] = list(variable) if isinstance( - variable, variables.PartitionedVariable - ) else variable + vars_in_ckpt[variable_name] = ( + list(variable) if isinstance(variable, variables.PartitionedVariable) else variable + ) elif len(ckpt_var_shape) == len(var_shape): if force_restore_shape_compatible: # create a variable compatible with checkpoint to restore - dtype = variable[0].dtype if isinstance( - variable, list - ) else variable.dtype + dtype = variable[0].dtype if isinstance(variable, list) else variable.dtype with tf.variable_scope('incompatible_shape_restore'): tmp_var = tf.get_variable( name=variable_name + '_T_E_M_P', @@ -298,29 +276,27 @@ def restore( # add to a special collection for easy reference # by tf.get_collection('T_E_M_P_RESTROE') collections=['T_E_M_P_RESTROE'], - dtype=dtype + dtype=dtype, ) vars_in_ckpt[variable_name] = tmp_var incompatible_shape_var_map[variable] = tmp_var - print( - 'incompatible restore %s[%s, %s]' % - (variable_name, str(var_shape), str(ckpt_var_shape)) - ) + print('incompatible restore %s[%s, %s]' % (variable_name, str(var_shape), str(ckpt_var_shape))) else: logging.warning( - 'Variable [%s] is available in checkpoint, but ' - 'incompatible shape with model variable.', variable_name + 'Variable [%s] is available in checkpoint, but ' 'incompatible shape with model variable.', + variable_name, ) else: logging.warning( - 'Variable [%s] is available in checkpoint, but ' - 'incompatible shape dims with model variable.', variable_name + 'Variable [%s] is available in checkpoint, but ' 'incompatible shape dims with model variable.', + variable_name, ) elif 'EmbeddingVariable' in str(type(variable)): if '%s-keys' % variable_name not in ckpt_var2shape_map: continue print('restore embedding_variable %s' % variable_name) from tensorflow.python.training import saver + names_to_saveables = saver.BaseSaverBuilder.OpListToDict([variable]) saveable_objects = [] for name, op in names_to_saveables.items(): @@ -328,13 +304,12 @@ def restore( saveable_objects.append(s) init_op = saveable_objects[0].restore([ckpt_path], None) variable._initializer_op = init_op - elif type(variable) == list and 'EmbeddingVariable' in str( - type(variable[0]) - ): + elif type(variable) == list and 'EmbeddingVariable' in str(type(variable[0])): if '%s/part_0-keys' % variable_name not in ckpt_var2shape_map: continue print('restore partitioned embedding_variable %s' % variable_name) from tensorflow.python.training import saver + for part_var in variable: names_to_saveables = saver.BaseSaverBuilder.OpListToDict([part_var]) saveable_objects = [] @@ -350,26 +325,20 @@ def restore( task_num=hvd.size(), embed_dim=variable._dimension, var_name='embed-' + variable.name.replace('/', '__'), - ckpt_path=ckpt_path + ckpt_path=ckpt_path, ) with ops.control_dependencies([variable._initializer_op]): - variable._initializer_op = dynamic_variable_ops.dummy_var_assign( - variable.handle, keys, vals - ) + variable._initializer_op = dynamic_variable_ops.dummy_var_assign(variable.handle, keys, vals) else: fail_restore_vars.append(variable_name) for variable_name in fail_restore_vars: if 'Momentum' not in variable_name: - logging.warning( - 'Variable [%s] is not available in checkpoint', variable_name - ) + logging.warning('Variable [%s] is not available in checkpoint', variable_name) tf.train.init_from_checkpoint(ckpt_path, vars_in_ckpt) if force_restore_shape_compatible: - return estimator_utils.IncompatibleShapeRestoreHook( - incompatible_shape_var_map - ) + return estimator_utils.IncompatibleShapeRestoreHook(incompatible_shape_var_map) else: return None @@ -392,9 +361,7 @@ def _get_restore_vars(self, ckpt_var_map_path): name2var = {} for one_var in all_vars: var_name = re.sub(VAR_SUFIX_PATTERN, '', one_var.name) - if re.search( - PARTITION_PATTERN, var_name - ) and one_var._save_slice_info is not None: + if re.search(PARTITION_PATTERN, var_name) and one_var._save_slice_info is not None: var_name = re.sub(PARTITION_PATTERN, '', var_name) is_part = True else: @@ -434,16 +401,10 @@ def _get_restore_vars(self, ckpt_var_map_path): else: var_filter, scope_update = self.get_restore_filter() if var_filter is not None: - name2var = { - var_name: name2var[var_name] - for var in name2var if var_filter.keep(var.name) - } + name2var = {var_name: name2var[var_name] for var in name2var if var_filter.keep(var.name)} # drop scope prefix if necessary if scope_update is not None: - name2var = { - scope_update(var_name): name2var[var_name] - for var_name in name2var - } + name2var = {scope_update(var_name): name2var[var_name] for var_name in name2var} return name2var def get_restore_filter(self): @@ -459,14 +420,12 @@ def get_restore_filter(self): for x in self._base_model_config.restore_filters: logging.info('restore will filter out pattern %s' % x) - all_filters = [ - restore_filter.KeywordFilter(x, True) - for x in self._base_model_config.restore_filters - ] + all_filters = [restore_filter.KeywordFilter(x, True) for x in self._base_model_config.restore_filters] - return restore_filter.CombineFilter( - all_filters, restore_filter.Logical.AND - ), None + return ( + restore_filter.CombineFilter(all_filters, restore_filter.Logical.AND), + None, + ) def get_grouped_vars(self, opt_num): """Group the vars into different optimization groups. @@ -484,9 +443,7 @@ def get_grouped_vars(self, opt_num): embedding_vars = [] deep_vars = [] for tmp_var in variables.trainable_variables(): - if tmp_var.name.startswith( - 'input_layer' - ) or '/embedding_weights' in tmp_var.name: + if tmp_var.name.startswith('input_layer') or '/embedding_weights' in tmp_var.name: embedding_vars.append(tmp_var) else: deep_vars.append(tmp_var) diff --git a/easy_rec/python/model/esmm.py b/easy_rec/python/model/esmm.py index 262b08b48..62e338ca6 100644 --- a/easy_rec/python/model/esmm.py +++ b/easy_rec/python/model/esmm.py @@ -15,20 +15,11 @@ class ESMM(MultiTaskModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(ESMM, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(ESMM, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'esmm', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model' ) - assert self._model_config.WhichOneof('model') == 'esmm', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.esmm assert isinstance(self._model_config, ESMMConfig) @@ -53,8 +44,7 @@ def __init__( self._ctr_tower_cfg = self._model_config.ctr_tower self._init_towers([self._cvr_tower_cfg, self._ctr_tower_cfg]) - assert self._model_config.ctr_tower.loss_type == LossType.CLASSIFICATION, \ - 'ctr tower must be binary classification.' + assert self._model_config.ctr_tower.loss_type == LossType.CLASSIFICATION, 'ctr tower must be binary classification.' for task_tower_cfg in self._task_towers: assert task_tower_cfg.num_class == 1, 'Does not support multiclass classification problem' @@ -69,38 +59,27 @@ def build_loss_graph(self): cvr_label_name = self._label_name_dict[cvr_tower_name] ctr_label_name = self._label_name_dict[ctr_tower_name] if self._cvr_tower_cfg.loss_type == LossType.CLASSIFICATION: - ctcvr_label = tf.cast( - self._labels[cvr_label_name] * self._labels[ctr_label_name], tf.float32 - ) - cvr_losses = tf.keras.backend.binary_crossentropy( - ctcvr_label, self._prediction_dict['probs_ctcvr'] - ) + ctcvr_label = tf.cast(self._labels[cvr_label_name] * self._labels[ctr_label_name], tf.float32) + cvr_losses = tf.keras.backend.binary_crossentropy(ctcvr_label, self._prediction_dict['probs_ctcvr']) cvr_loss = tf.reduce_sum(cvr_losses, name='ctcvr_loss') # The weight defaults to 1. - self._loss_dict['weighted_cross_entropy_loss_%s' % cvr_tower_name - ] = self._cvr_tower_cfg.weight * cvr_loss + self._loss_dict['weighted_cross_entropy_loss_%s' % cvr_tower_name] = self._cvr_tower_cfg.weight * cvr_loss elif self._cvr_tower_cfg.loss_type == LossType.L2_LOSS: logging.info('l2 loss is used') cvr_dtype = self._labels[cvr_label_name].dtype - ctcvr_label = self._labels[cvr_label_name] * tf.cast( - self._labels[ctr_label_name], cvr_dtype - ) + ctcvr_label = self._labels[cvr_label_name] * tf.cast(self._labels[ctr_label_name], cvr_dtype) cvr_loss = tf.losses.mean_squared_error( labels=ctcvr_label, predictions=self._prediction_dict['y_ctcvr'], - weights=self._sample_weight + weights=self._sample_weight, ) - self._loss_dict['weighted_l2_loss_%s' % cvr_tower_name - ] = self._cvr_tower_cfg.weight * cvr_loss + self._loss_dict['weighted_l2_loss_%s' % cvr_tower_name] = self._cvr_tower_cfg.weight * cvr_loss _labels = tf.cast(self._labels[ctr_label_name], tf.float32) _logits = self._prediction_dict['logits_%s' % ctr_tower_name] - cross = tf.nn.sigmoid_cross_entropy_with_logits( - labels=_labels, logits=_logits, name='ctr_loss' - ) + cross = tf.nn.sigmoid_cross_entropy_with_logits(labels=_labels, logits=_logits, name='ctr_loss') ctr_loss = tf.reduce_sum(cross) - self._loss_dict['weighted_cross_entropy_loss_%s' % ctr_tower_name - ] = self._ctr_tower_cfg.weight * ctr_loss + self._loss_dict['weighted_cross_entropy_loss_%s' % ctr_tower_name] = self._ctr_tower_cfg.weight * ctr_loss return self._loss_dict def build_metric_graph(self, eval_config): @@ -122,37 +101,31 @@ def build_metric_graph(self, eval_config): # CTCVR metric ctcvr_label_name = cvr_label_name + '_ctcvr' cvr_dtype = self._labels[cvr_label_name].dtype - self._labels[ctcvr_label_name] = self._labels[cvr_label_name] * tf.cast( - self._labels[ctr_label_name], cvr_dtype - ) + self._labels[ctcvr_label_name] = self._labels[cvr_label_name] * tf.cast(self._labels[ctr_label_name], cvr_dtype) metric_dict.update( self._build_metric_impl( metric, loss_type=self._cvr_tower_cfg.loss_type, label_name=ctcvr_label_name, num_class=self._cvr_tower_cfg.num_class, - suffix='_ctcvr' + suffix='_ctcvr', ) ) # CVR metric cvr_label_masked_name = cvr_label_name + '_masked' ctr_mask = self._labels[ctr_label_name] > 0 - self._labels[cvr_label_masked_name] = tf.boolean_mask( - self._labels[cvr_label_name], ctr_mask - ) + self._labels[cvr_label_masked_name] = tf.boolean_mask(self._labels[cvr_label_name], ctr_mask) pred_prefix = 'probs' if self._cvr_tower_cfg.loss_type == LossType.CLASSIFICATION else 'y' pred_name = '%s_%s' % (pred_prefix, cvr_tower_name) - self._prediction_dict[pred_name + '_masked'] = tf.boolean_mask( - self._prediction_dict[pred_name], ctr_mask - ) + self._prediction_dict[pred_name + '_masked'] = tf.boolean_mask(self._prediction_dict[pred_name], ctr_mask) metric_dict.update( self._build_metric_impl( metric, loss_type=self._cvr_tower_cfg.loss_type, label_name=cvr_label_masked_name, num_class=self._cvr_tower_cfg.num_class, - suffix='_%s_masked' % cvr_tower_name + suffix='_%s_masked' % cvr_tower_name, ) ) @@ -164,7 +137,7 @@ def build_metric_graph(self, eval_config): loss_type=self._ctr_tower_cfg.loss_type, label_name=ctr_label_name, num_class=self._ctr_tower_cfg.num_class, - suffix='_%s' % ctr_tower_name + suffix='_%s' % ctr_tower_name, ) ) return metric_dict @@ -174,7 +147,7 @@ def _add_to_prediction_dict(self, output): if self._cvr_tower_cfg.loss_type == LossType.CLASSIFICATION: prob = tf.multiply( self._prediction_dict['probs_%s' % self._cvr_tower_cfg.tower_name], - self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name] + self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name], ) # pctcvr = pctr * pcvr self._prediction_dict['probs_ctcvr'] = prob @@ -182,7 +155,7 @@ def _add_to_prediction_dict(self, output): else: prob = tf.multiply( self._prediction_dict['y_%s' % self._cvr_tower_cfg.tower_name], - self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name] + self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name], ) # pctcvr = pctr * pcvr self._prediction_dict['y_ctcvr'] = prob @@ -202,9 +175,7 @@ def build_predict_graph(self): group_fea = self._group_features[group_id] group = self._model_config.groups[group_id] group_name = group.input - dnn_model = dnn.DNN( - group.dnn, self._l2_reg, group_name, self._is_training - ) + dnn_model = dnn.DNN(group.dnn, self._l2_reg, group_name, self._is_training) group_fea = dnn_model(group_fea) group_fea_arr.append(group_fea) all_fea = tf.concat(group_fea_arr, axis=1) @@ -216,14 +187,14 @@ def build_predict_graph(self): self._cvr_tower_cfg.dnn, self._l2_reg, name=cvr_tower_name, - is_training=self._is_training + is_training=self._is_training, ) cvr_tower_output = dnn_model(all_fea) cvr_tower_output = tf.layers.dense( inputs=cvr_tower_output, units=1, kernel_regularizer=self._l2_reg, - name='%s/dnn_output' % cvr_tower_name + name='%s/dnn_output' % cvr_tower_name, ) ctr_tower_name = self._ctr_tower_cfg.tower_name @@ -231,19 +202,19 @@ def build_predict_graph(self): self._ctr_tower_cfg.dnn, self._l2_reg, name=ctr_tower_name, - is_training=self._is_training + is_training=self._is_training, ) ctr_tower_output = dnn_model(all_fea) ctr_tower_output = tf.layers.dense( inputs=ctr_tower_output, units=1, kernel_regularizer=self._l2_reg, - name='%s/dnn_output' % ctr_tower_name + name='%s/dnn_output' % ctr_tower_name, ) tower_outputs = { cvr_tower_name: cvr_tower_output, - ctr_tower_name: ctr_tower_output + ctr_tower_name: ctr_tower_output, } self._add_to_prediction_dict(tower_outputs) return self._prediction_dict @@ -260,7 +231,5 @@ def get_outputs(self): elif self._cvr_tower_cfg.loss_type == LossType.L2_LOSS: outputs.append('y_ctcvr') else: - raise ValueError( - 'invalid cvr_tower loss type: %s' % str(self._cvr_tower_cfg.loss_type) - ) + raise ValueError('invalid cvr_tower loss type: %s' % str(self._cvr_tower_cfg.loss_type)) return outputs diff --git a/easy_rec/python/model/fm.py b/easy_rec/python/model/fm.py index 5ee3c216c..6f92e6e95 100644 --- a/easy_rec/python/model/fm.py +++ b/easy_rec/python/model/fm.py @@ -13,27 +13,16 @@ class FM(RankModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(FM, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(FM, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'fm', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model' ) - assert self._model_config.WhichOneof('model') == 'fm', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.fm assert isinstance(self._model_config, FMConfig) self._wide_features, _ = self._input_layer(self._feature_dict, 'wide') - self._deep_features, self._fm_features = self._input_layer( - self._feature_dict, 'deep' - ) + self._deep_features, self._fm_features = self._input_layer(self._feature_dict, 'deep') def build_input_layer(self, model_config, feature_configs): # overwrite create input_layer to support wide_output_dim @@ -41,9 +30,7 @@ def build_input_layer(self, model_config, feature_configs): super(FM, self).build_input_layer(model_config, feature_configs) def build_predict_graph(self): - wide_fea = tf.reduce_sum( - self._wide_features, axis=1, keepdims=True, name='wide_feature' - ) + wide_fea = tf.reduce_sum(self._wide_features, axis=1, keepdims=True, name='wide_feature') fm_fea = fm.FM(name='fm_feature')(self._fm_features) @@ -52,15 +39,16 @@ def build_predict_graph(self): fm_fea, self._num_class, kernel_regularizer=self._l2_reg, - name='fm_logits' + name='fm_logits', ) else: fm_fea = tf.reduce_sum(fm_fea, 1, keepdims=True) bias = tf.get_variable( - 'fm_bias', [self._num_class], + 'fm_bias', + [self._num_class], initializer=tf.zeros_initializer(), - trainable=True + trainable=True, ) output = wide_fea + fm_fea diff --git a/easy_rec/python/model/match_model.py b/easy_rec/python/model/match_model.py index 3b8bd9ebe..a28c82453 100644 --- a/easy_rec/python/model/match_model.py +++ b/easy_rec/python/model/match_model.py @@ -16,18 +16,8 @@ class MatchModel(EasyRecModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(MatchModel, self).__init__( - model_config, feature_configs, features, labels, is_training - ) + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(MatchModel, self).__init__(model_config, feature_configs, features, labels, is_training) self._loss_type = self._model_config.loss_type self._num_class = self._model_config.num_class @@ -53,25 +43,24 @@ def __init__( def _mask_in_batch(self, logits): batch_size = tf.shape(logits)[0] if getattr(self._model_config, 'ignore_in_batch_neg_sam', False): - in_batch = logits[:, :batch_size] - ( - 1 - tf.diag(tf.ones([batch_size], dtype=tf.float32)) - ) * 1e32 + in_batch = logits[:, :batch_size] - (1 - tf.diag(tf.ones([batch_size], dtype=tf.float32))) * 1e32 return tf.concat([in_batch, logits[:, batch_size:]], axis=1) else: if self._item_ids is not None: mask_in_batch_neg = tf.to_float( tf.equal( - self._item_ids[None, :batch_size], self._item_ids[:batch_size, - None] + self._item_ids[None, :batch_size], + self._item_ids[:batch_size, None], ) ) - tf.diag(tf.ones([batch_size], dtype=tf.float32)) - tf.summary.scalar( - 'in_batch_neg_conflict', tf.reduce_sum(mask_in_batch_neg) - ) - return tf.concat([ - logits[:, :batch_size] - mask_in_batch_neg * 1e32, - logits[:, batch_size:]], - axis=1) # yapf: disable + tf.summary.scalar('in_batch_neg_conflict', tf.reduce_sum(mask_in_batch_neg)) + return tf.concat( + [ + logits[:, :batch_size] - mask_in_batch_neg * 1e32, + logits[:, batch_size:], + ], + axis=1, + ) # yapf: disable else: return logits @@ -84,9 +73,7 @@ def _list_wise_sim(self, user_emb, item_emb): noclk_size = tf.shape(hard_neg_indices)[0] # pos_item_emb, neg_item_emb, hard_neg_item_emb = tf.split( # item_emb, [batch_size, -1, noclk_size], axis=0) - simple_item_emb, hard_neg_item_emb = tf.split( - item_emb, [-1, noclk_size], axis=0 - ) + simple_item_emb, hard_neg_item_emb = tf.split(item_emb, [-1, noclk_size], axis=0) else: # pos_item_emb = item_emb[:batch_size] # neg_item_emb = item_emb[batch_size:] @@ -99,9 +86,7 @@ def _list_wise_sim(self, user_emb, item_emb): _mode = os.environ['tf.estimator.mode'] if _mode == tf.estimator.ModeKeys.PREDICT: - simple_user_item_sim = tf.reduce_sum( - tf.multiply(user_emb, simple_item_emb), axis=1, keep_dims=True - ) + simple_user_item_sim = tf.reduce_sum(tf.multiply(user_emb, simple_item_emb), axis=1, keep_dims=True) else: simple_user_item_sim = tf.matmul(user_emb, tf.transpose(simple_item_emb)) @@ -109,18 +94,14 @@ def _list_wise_sim(self, user_emb, item_emb): return simple_user_item_sim else: user_emb_expand = tf.gather(user_emb, hard_neg_indices[:, 0]) - hard_neg_user_item_sim = tf.reduce_sum( - tf.multiply(user_emb_expand, hard_neg_item_emb), axis=1 - ) + hard_neg_user_item_sim = tf.reduce_sum(tf.multiply(user_emb_expand, hard_neg_item_emb), axis=1) max_num_neg = tf.reduce_max(hard_neg_indices[:, 1]) + 1 hard_neg_shape = tf.stack([tf.to_int64(batch_size), max_num_neg]) - hard_neg_sim = tf.scatter_nd( - hard_neg_indices, hard_neg_user_item_sim, hard_neg_shape - ) + hard_neg_sim = tf.scatter_nd(hard_neg_indices, hard_neg_user_item_sim, hard_neg_shape) hard_neg_mask = tf.scatter_nd( hard_neg_indices, tf.ones_like(hard_neg_user_item_sim, dtype=tf.float32), - shape=hard_neg_shape + shape=hard_neg_shape, ) # set tail positions to -1e32, so that after exp(x), will be zero hard_neg_user_item_sim = hard_neg_sim - (1 - hard_neg_mask) * 1e32 @@ -133,9 +114,7 @@ def _list_wise_sim(self, user_emb, item_emb): return tf.concat([simple_user_item_sim, hard_neg_user_item_sim], axis=1) def _point_wise_sim(self, user_emb, item_emb): - user_item_sim = tf.reduce_sum( - tf.multiply(user_emb, item_emb), axis=1, keep_dims=True - ) + user_item_sim = tf.reduce_sum(tf.multiply(user_emb, item_emb), axis=1, keep_dims=True) return user_item_sim def sim(self, user_emb, item_emb): @@ -157,9 +136,7 @@ def norm(self, fea): def build_predict_graph(self): if not self.has_backbone: - raise NotImplementedError( - 'method `build_predict_graph` must be implemented when you donot use backbone network' - ) + raise NotImplementedError('method `build_predict_graph` must be implemented when you donot use backbone network') model = self._model_config.WhichOneof('model') assert model == 'model_params', '`model_params` must be configured' model_params = self._model_config.model_params @@ -181,18 +158,8 @@ def build_predict_graph(self): user_item_sim = self.sim(user_tower_emb, item_tower_emb) / temperature if model_params.scale_simi: - sim_w = tf.get_variable( - 'sim_w', - dtype=tf.float32, - shape=(1), - initializer=tf.ones_initializer() - ) - sim_b = tf.get_variable( - 'sim_b', - dtype=tf.float32, - shape=(1), - initializer=tf.zeros_initializer() - ) + sim_w = tf.get_variable('sim_w', dtype=tf.float32, shape=(1), initializer=tf.ones_initializer()) + sim_b = tf.get_variable('sim_b', dtype=tf.float32, shape=(1), initializer=tf.zeros_initializer()) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -212,12 +179,8 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb - self._prediction_dict['user_emb'] = tf.reduce_join( - tf.as_string(user_tower_emb), axis=-1, separator=',' - ) - self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_tower_emb), axis=-1, separator=',' - ) + self._prediction_dict['user_emb'] = tf.reduce_join(tf.as_string(user_tower_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_tower_emb), axis=-1, separator=',') return self._prediction_dict @@ -232,9 +195,7 @@ def _build_list_wise_loss_graph(self): batch_size = tf.shape(self._prediction_dict['probs'])[0] indices = tf.range(batch_size) indices = tf.concat([indices[:, None], indices[:, None]], axis=1) - hit_prob = tf.gather_nd( - self._prediction_dict['probs'][:batch_size, :batch_size], indices - ) + hit_prob = tf.gather_nd(self._prediction_dict['probs'][:batch_size, :batch_size], indices) sample_weights = tf.cast(tf.squeeze(self._sample_weight), tf.float32) self._loss_dict['cross_entropy_loss'] = -tf.reduce_mean( @@ -248,31 +209,36 @@ def _build_list_wise_loss_graph(self): pos_simi = tf.reduce_sum(user_features * pos_item_emb, axis=1) # if pos_simi < 0, produce loss reg_pos_loss = tf.nn.relu(-pos_simi) - self._loss_dict['reg_pos_loss'] = tf.reduce_mean( - reg_pos_loss * sample_weights - ) / tf.reduce_mean(sample_weights) + self._loss_dict['reg_pos_loss'] = tf.reduce_mean(reg_pos_loss * sample_weights) / tf.reduce_mean(sample_weights) # the AMM loss for DAT model if all( [ - k in self._prediction_dict.keys() for k in - ['augmented_p_u', 'augmented_p_i', 'augmented_a_u', 'augmented_a_i'] + k in self._prediction_dict.keys() + for k in [ + 'augmented_p_u', + 'augmented_p_i', + 'augmented_a_u', + 'augmented_a_i', + ] ] ): - self._loss_dict['amm_loss_u' - ] = self._model_config.amm_u_weight * tf.reduce_mean( - tf.square( - self._prediction_dict['augmented_a_u'] - - self._prediction_dict['augmented_p_i'][:batch_size] - ) * sample_weights - ) / tf.reduce_mean(sample_weights) - self._loss_dict[ - 'amm_loss_i'] = self._model_config.amm_i_weight * tf.reduce_mean( - tf.square( - self._prediction_dict['augmented_a_i'][:batch_size] - - self._prediction_dict['augmented_p_u'] - ) * sample_weights - ) / tf.reduce_mean(sample_weights) + self._loss_dict['amm_loss_u'] = ( + self._model_config.amm_u_weight + * tf.reduce_mean( + tf.square(self._prediction_dict['augmented_a_u'] - self._prediction_dict['augmented_p_i'][:batch_size]) + * sample_weights + ) + / tf.reduce_mean(sample_weights) + ) + self._loss_dict['amm_loss_i'] = ( + self._model_config.amm_i_weight + * tf.reduce_mean( + tf.square(self._prediction_dict['augmented_a_i'][:batch_size] - self._prediction_dict['augmented_p_u']) + * sample_weights + ) + / tf.reduce_mean(sample_weights) + ) else: raise ValueError('invalid loss type: %s' % str(self._loss_type)) @@ -295,13 +261,11 @@ def _build_point_wise_loss_graph(self): label=label, pred=pred, loss_weight=self._sample_weight, - **kwargs + **kwargs, ) # build kd loss - kd_loss_dict = loss_builder.build_kd_loss( - self.kd, self._prediction_dict, self._labels, self._feature_dict - ) + kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, self._labels, self._feature_dict) self._loss_dict.update(kd_loss_dict) return self._loss_dict @@ -313,6 +277,7 @@ def build_metric_graph(self, eval_config): def _build_list_wise_metric_graph(self, eval_config): from easy_rec.python.core.easyrec_metrics import metrics_tf + logits = self._prediction_dict['logits'] # label = tf.zeros_like(logits[:, :1], dtype=tf.int64) batch_size = tf.shape(logits)[0] @@ -324,37 +289,32 @@ def _build_list_wise_metric_graph(self, eval_config): metric_dict = {} for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'recall_at_topk': - metric_dict[ - 'recall@%d' % metric.recall_at_topk.topk - ] = metrics_tf.recall_at_k(label, logits, metric.recall_at_topk.topk) - - logits_v2 = tf.concat( - [pos_item_sim[:, None], logits[:, batch_size:]], axis=1 + metric_dict['recall@%d' % metric.recall_at_topk.topk] = metrics_tf.recall_at_k( + label, logits, metric.recall_at_topk.topk ) + + logits_v2 = tf.concat([pos_item_sim[:, None], logits[:, batch_size:]], axis=1) labels_v2 = tf.zeros_like(logits_v2[:, :1], dtype=tf.int64) - metric_dict['recall_neg_sam@%d' % metric.recall_at_topk.topk - ] = metrics_tf.recall_at_k( - labels_v2, logits_v2, metric.recall_at_topk.topk - ) - - metric_dict['recall_in_batch@%d' % metric.recall_at_topk.topk - ] = metrics_tf.recall_at_k( - label, logits[:, :batch_size], metric.recall_at_topk.topk - ) + metric_dict['recall_neg_sam@%d' % metric.recall_at_topk.topk] = metrics_tf.recall_at_k( + labels_v2, logits_v2, metric.recall_at_topk.topk + ) + + metric_dict['recall_in_batch@%d' % metric.recall_at_topk.topk] = metrics_tf.recall_at_k( + label, logits[:, :batch_size], metric.recall_at_topk.topk + ) else: raise ValueError('invalid metric type: %s' % str(metric)) return metric_dict def _build_point_wise_metric_graph(self, eval_config): from easy_rec.python.core.easyrec_metrics import metrics_tf + metric_dict = {} label = list(self._labels.values())[0] for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'auc': assert self._loss_type == LossType.CLASSIFICATION - metric_dict['auc'] = metrics_tf.auc( - label, self._prediction_dict['probs'] - ) + metric_dict['auc'] = metrics_tf.auc(label, self._prediction_dict['probs']) elif metric.WhichOneof('metric') == 'mean_absolute_error': assert self._loss_type == LossType.L2_LOSS metric_dict['mean_absolute_error'] = metrics_tf.mean_absolute_error( @@ -366,24 +326,26 @@ def _build_point_wise_metric_graph(self, eval_config): def get_outputs(self): if not self.has_backbone: - raise NotImplementedError( - 'could not call get_outputs on abstract class MatchModel' - ) + raise NotImplementedError('could not call get_outputs on abstract class MatchModel') if self._loss_type == LossType.CLASSIFICATION: return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_tower_emb', + 'item_tower_emb', ] elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: - self._prediction_dict['logits'] = tf.squeeze( - self._prediction_dict['logits'], axis=-1 - ) - self._prediction_dict['probs'] = tf.nn.sigmoid( - self._prediction_dict['logits'] - ) + self._prediction_dict['logits'] = tf.squeeze(self._prediction_dict['logits'], axis=-1) + self._prediction_dict['probs'] = tf.nn.sigmoid(self._prediction_dict['logits']) return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_tower_emb', + 'item_tower_emb', ] elif self._loss_type == LossType.L2_LOSS: return ['y', 'user_emb', 'item_emb', 'user_tower_emb', 'item_tower_emb'] diff --git a/easy_rec/python/model/mind.py b/easy_rec/python/model/mind.py index b86babb6d..8b43ca8b1 100644 --- a/easy_rec/python/model/mind.py +++ b/easy_rec/python/model/mind.py @@ -19,25 +19,14 @@ class MIND(MatchModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(MIND, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(MIND, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'mind', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model' ) - assert self._model_config.WhichOneof('model') == 'mind', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.mind - self._hist_seq_features, _, _ = self._input_layer( - self._feature_dict, 'hist', is_combine=False - ) + self._hist_seq_features, _, _ = self._input_layer(self._feature_dict, 'hist', is_combine=False) self._user_features, _ = self._input_layer(self._feature_dict, 'user') self._item_features, _ = self._input_layer(self._feature_dict, 'item') @@ -48,33 +37,20 @@ def __init__( # copy obj so that any modification will not affect original config self.concat_dnn = copy_obj(self._model_config.concat_dnn) - self._l2_reg = regularizers.l2_regularizer( - self._model_config.l2_regularization - ) + self._l2_reg = regularizers.l2_regularizer(self._model_config.l2_regularization) def build_predict_graph(self): - capsule_layer = CapsuleLayer( - self._model_config.capsule_config, self._is_training - ) + capsule_layer = CapsuleLayer(self._model_config.capsule_config, self._is_training) if self._model_config.time_id_fea: - time_id_fea = [ - x[0] for x in self._hist_seq_features - if self._model_config.time_id_fea in x[0].name - ] - logging.info( - 'time_id_fea is set(%s), find num: %d' % - (self._model_config.time_id_fea, len(time_id_fea)) - ) + time_id_fea = [x[0] for x in self._hist_seq_features if self._model_config.time_id_fea in x[0].name] + logging.info('time_id_fea is set(%s), find num: %d' % (self._model_config.time_id_fea, len(time_id_fea))) else: time_id_fea = [] time_id_fea = time_id_fea[0] if len(time_id_fea) > 0 else None if time_id_fea is not None: - hist_seq_feas = [ - x[0] for x in self._hist_seq_features - if self._model_config.time_id_fea not in x[0].name - ] + hist_seq_feas = [x[0] for x in self._hist_seq_features if self._model_config.time_id_fea not in x[0].name] else: hist_seq_feas = [x[0] for x in self._hist_seq_features] @@ -85,24 +61,24 @@ def build_predict_graph(self): # sum pooling over the features hist_embed_dims = [x.get_shape()[-1] for x in hist_seq_feas] for i in range(1, len(hist_embed_dims)): - assert hist_embed_dims[i] == hist_embed_dims[0], \ - 'all hist seq must have the same embedding shape, but: %s' \ - % str(hist_embed_dims) + assert hist_embed_dims[i] == hist_embed_dims[0], ( + 'all hist seq must have the same embedding shape, but: %s' % str(hist_embed_dims) + ) hist_seq_feas = tf.add_n(hist_seq_feas) / len(hist_seq_feas) else: hist_seq_feas = tf.concat(hist_seq_feas, axis=2) - if self._model_config.HasField('pre_capsule_dnn') and \ - len(self._model_config.pre_capsule_dnn.hidden_units) > 0: + if self._model_config.HasField('pre_capsule_dnn') and len(self._model_config.pre_capsule_dnn.hidden_units) > 0: pre_dnn_layer = dnn.DNN( - self._model_config.pre_capsule_dnn, self._l2_reg, 'pre_capsule_dnn', - self._is_training + self._model_config.pre_capsule_dnn, + self._l2_reg, + 'pre_capsule_dnn', + self._is_training, ) hist_seq_feas = pre_dnn_layer(hist_seq_feas) if time_id_fea is not None: - assert time_id_fea.get_shape( - )[-1] == 1, 'time_id must have only embedding_size of 1' + assert time_id_fea.get_shape()[-1] == 1, 'time_id must have only embedding_size of 1' time_id_mask = tf.sequence_mask(hist_seq_len, tf.shape(time_id_fea)[1]) time_id_mask = (tf.cast(time_id_mask, tf.float32) * 2 - 1) * 1e32 time_id_fea = tf.minimum(time_id_fea, time_id_mask[:, :, None]) @@ -111,9 +87,7 @@ def build_predict_graph(self): tf.summary.histogram('hist_seq_len', hist_seq_len) # batch_size x max_k x high_capsule_dim - high_capsules, num_high_capsules = capsule_layer( - hist_seq_feas, hist_seq_len - ) + high_capsules, num_high_capsules = capsule_layer(hist_seq_feas, hist_seq_len) tf.summary.histogram('num_high_capsules', num_high_capsules) @@ -122,63 +96,49 @@ def build_predict_graph(self): # trainable=True, name='capsule_bn') # high_capsules = high_capsules * 0.1 - tf.summary.scalar( - 'high_capsules_norm', tf.reduce_mean(tf.norm(high_capsules, axis=-1)) - ) - tf.summary.scalar( - 'num_high_capsules', tf.reduce_mean(tf.to_float(num_high_capsules)) - ) + tf.summary.scalar('high_capsules_norm', tf.reduce_mean(tf.norm(high_capsules, axis=-1))) + tf.summary.scalar('num_high_capsules', tf.reduce_mean(tf.to_float(num_high_capsules))) user_features = tf.layers.batch_normalization( self._user_features, training=self._is_training, trainable=True, - name='user_fea_bn' - ) - user_dnn = dnn.DNN( - self.user_dnn, self._l2_reg, 'user_dnn', self._is_training + name='user_fea_bn', ) + user_dnn = dnn.DNN(self.user_dnn, self._l2_reg, 'user_dnn', self._is_training) user_features = user_dnn(user_features) - tf.summary.scalar( - 'user_features_norm', - tf.reduce_mean(tf.norm(self._user_features, axis=-1)) - ) + tf.summary.scalar('user_features_norm', tf.reduce_mean(tf.norm(self._user_features, axis=-1))) # concatenate with user features - user_features_tile = tf.tile( - user_features[:, None, :], [1, tf.shape(high_capsules)[1], 1] - ) + user_features_tile = tf.tile(user_features[:, None, :], [1, tf.shape(high_capsules)[1], 1]) user_interests = tf.concat([high_capsules, user_features_tile], axis=2) num_concat_dnn_layer = len(self.concat_dnn.hidden_units) last_hidden = self.concat_dnn.hidden_units.pop() - concat_dnn = dnn.DNN( - self.concat_dnn, self._l2_reg, 'concat_dnn', self._is_training - ) + concat_dnn = dnn.DNN(self.concat_dnn, self._l2_reg, 'concat_dnn', self._is_training) user_interests = concat_dnn(user_interests) user_interests = tf.layers.dense( inputs=user_interests, units=last_hidden, kernel_regularizer=self._l2_reg, - name='concat_dnn/dnn_%d' % (num_concat_dnn_layer - 1) + name='concat_dnn/dnn_%d' % (num_concat_dnn_layer - 1), ) num_item_dnn_layer = len(self.item_dnn.hidden_units) last_item_hidden = self.item_dnn.hidden_units.pop() - item_dnn = dnn.DNN( - self.item_dnn, self._l2_reg, 'item_dnn', self._is_training - ) + item_dnn = dnn.DNN(self.item_dnn, self._l2_reg, 'item_dnn', self._is_training) item_tower_emb = item_dnn(self._item_features) item_tower_emb = tf.layers.dense( inputs=item_tower_emb, units=last_item_hidden, kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1) + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), ) assert self._model_config.simi_func in [ - Similarity.COSINE, Similarity.INNER_PRODUCT + Similarity.COSINE, + Similarity.INNER_PRODUCT, ] if self._model_config.simi_func == Similarity.COSINE: @@ -190,19 +150,13 @@ def build_predict_graph(self): batch_size = tf.shape(user_interests)[0] pos_item_fea = item_tower_emb[:batch_size] simi = tf.einsum('bhe,be->bh', user_interests, pos_item_fea) - tf.summary.histogram( - 'interest_item_simi/pre_scale', tf.reduce_max(simi, axis=1) - ) + tf.summary.histogram('interest_item_simi/pre_scale', tf.reduce_max(simi, axis=1)) # simi = tf.Print(simi, [tf.reduce_max(simi, axis=1), tf.reduce_min(simi, axis=1)], message='simi_max_0') # simi = tf.pow(simi, self._model_config.simi_pow) simi = simi * self._model_config.simi_pow - tf.summary.histogram( - 'interest_item_simi/scaled', tf.reduce_max(simi, axis=1) - ) + tf.summary.histogram('interest_item_simi/scaled', tf.reduce_max(simi, axis=1)) # simi = tf.Print(simi, [tf.reduce_max(simi, axis=1), tf.reduce_min(simi, axis=1)], message='simi_max') - simi_mask = tf.sequence_mask( - num_high_capsules, self._model_config.capsule_config.max_k - ) + simi_mask = tf.sequence_mask(num_high_capsules, self._model_config.capsule_config.max_k) user_interests = user_interests * tf.to_float(simi_mask[:, :, None]) self._prediction_dict['user_interests'] = user_interests @@ -210,9 +164,7 @@ def build_predict_graph(self): max_thresh = (tf.cast(simi_mask, tf.float32) * 2 - 1) * 1e32 simi = tf.minimum(simi, max_thresh) simi = tf.nn.softmax(simi, axis=1) - tf.summary.histogram( - 'interest_item_simi/softmax', tf.reduce_max(simi, axis=1) - ) + tf.summary.histogram('interest_item_simi/softmax', tf.reduce_max(simi, axis=1)) if self._model_config.simi_pow >= 100: logging.info( @@ -227,18 +179,8 @@ def build_predict_graph(self): # calculate similarity between user_tower_emb and item_tower_emb user_item_sim = self.sim(user_tower_emb, item_tower_emb) if self._model_config.scale_simi: - sim_w = tf.get_variable( - 'sim_w', - dtype=tf.float32, - shape=(1), - initializer=tf.ones_initializer() - ) - sim_b = tf.get_variable( - 'sim_b', - dtype=tf.float32, - shape=(1), - initializer=tf.zeros_initializer() - ) + sim_w = tf.get_variable('sim_w', dtype=tf.float32, shape=(1), initializer=tf.ones_initializer()) + sim_b = tf.get_variable('sim_b', dtype=tf.float32, shape=(1), initializer=tf.zeros_initializer()) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -263,12 +205,10 @@ def build_predict_graph(self): self._prediction_dict['user_emb'] = tf.reduce_join( tf.reduce_join(tf.as_string(user_interests), axis=-1, separator=','), axis=-1, - separator='|' + separator='|', ) self._prediction_dict['user_emb_num'] = num_high_capsules - self._prediction_dict['item_emb'] = tf.reduce_join( - tf.as_string(item_tower_emb), axis=-1, separator=',' - ) + self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_tower_emb), axis=-1, separator=',') if self._labels is not None: # for summary purpose @@ -282,16 +222,13 @@ def build_loss_graph(self): loss_dict = super(MIND, self).build_loss_graph() if self._model_config.max_interests_simi < 1.0: loss_dict['reg_interest_simi'] = tf.nn.relu( - self._prediction_dict['interests_simi'] - - self._model_config.max_interests_simi + self._prediction_dict['interests_simi'] - self._model_config.max_interests_simi ) return loss_dict def _build_interest_simi(self): user_emb_num = self._prediction_dict['user_emb_num'] - high_capsule_mask = tf.sequence_mask( - user_emb_num, self._model_config.capsule_config.max_k - ) + high_capsule_mask = tf.sequence_mask(user_emb_num, self._model_config.capsule_config.max_k) user_interests = self._prediction_dict['user_interests'] high_capsule_mask = tf.to_float(high_capsule_mask[:, :, None]) @@ -308,24 +245,18 @@ def _build_interest_simi(self): high_capsule_simi = high_capsule_sum_sqr - high_capsule_sqr_sum # normalize by interest number - interest_div = tf.maximum( - tf.to_float(user_emb_num * (user_emb_num - 1)), 1.0 - ) + interest_div = tf.maximum(tf.to_float(user_emb_num * (user_emb_num - 1)), 1.0) interest_simi = tf.reduce_sum(interest_simi, axis=1) / interest_div high_capsule_simi = tf.reduce_sum(high_capsule_simi, axis=1) / interest_div # normalize by batch_size multi_interest = tf.to_float(user_emb_num > 1) - sum_interest_simi = tf.reduce_sum( - (interest_simi + 1) * multi_interest - ) / 2.0 + sum_interest_simi = tf.reduce_sum((interest_simi + 1) * multi_interest) / 2.0 sum_div = tf.maximum(tf.reduce_sum(multi_interest), 1.0) avg_interest_simi = sum_interest_simi / sum_div - sum_capsule_simi = tf.reduce_sum( - (high_capsule_simi + 1) * multi_interest - ) / 2.0 + sum_capsule_simi = tf.reduce_sum((high_capsule_simi + 1) * multi_interest) / 2.0 avg_capsule_simi = sum_capsule_simi / sum_div tf.summary.scalar('interest_similarity', avg_interest_simi) @@ -339,7 +270,7 @@ def build_metric_graph(self, eval_config): interest_simi, capsule_simi = self._build_interest_simi() metric_dict = { 'interest_similarity': metrics.mean(interest_simi), - 'capsule_similarity': metrics.mean(capsule_simi) + 'capsule_similarity': metrics.mean(capsule_simi), } if self._is_point_wise: metric_dict.update(self._build_point_wise_metric_graph(eval_config)) @@ -349,7 +280,8 @@ def build_metric_graph(self, eval_config): for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'recall_at_topk': assert self._loss_type in [ - LossType.CLASSIFICATION, LossType.SOFTMAX_CROSS_ENTROPY + LossType.CLASSIFICATION, + LossType.SOFTMAX_CROSS_ENTROPY, ] if metric.recall_at_topk.topk not in recall_at_topks: recall_at_topks.append(metric.recall_at_topk.topk) @@ -367,9 +299,7 @@ def build_metric_graph(self, eval_config): if hard_neg_indices is not None: logging.info('With hard negative examples') noclk_size = tf.shape(hard_neg_indices)[0] - simple_item_emb, hard_neg_item_emb = tf.split( - item_tower_emb, [-1, noclk_size], axis=0 - ) + simple_item_emb, hard_neg_item_emb = tf.split(item_tower_emb, [-1, noclk_size], axis=0) else: simple_item_emb = item_tower_emb hard_neg_item_emb = None @@ -382,16 +312,10 @@ def build_metric_graph(self, eval_config): # labels = tf.zeros_like(logits[:, :1], dtype=tf.int64) pos_indices = tf.range(batch_size) - pos_indices = tf.concat( - [pos_indices[:, None], pos_indices[:, None]], axis=1 - ) - pos_item_sim = tf.gather_nd( - simple_item_sim[:batch_size, :batch_size], pos_indices - ) + pos_indices = tf.concat([pos_indices[:, None], pos_indices[:, None]], axis=1) + pos_item_sim = tf.gather_nd(simple_item_sim[:batch_size, :batch_size], pos_indices) - simple_item_sim_v2 = tf.concat( - [pos_item_sim[:, None], simple_item_sim[:, batch_size:]], axis=1 - ) + simple_item_sim_v2 = tf.concat([pos_item_sim[:, None], simple_item_sim[:, batch_size:]], axis=1) simple_lbls_v2 = tf.zeros_like(simple_item_sim_v2[:, :1], dtype=tf.int64) for topk in recall_at_topks: @@ -399,22 +323,18 @@ def build_metric_graph(self, eval_config): labels=simple_lbls, predictions=simple_item_sim, k=topk, - name='interests_recall_at_%d' % topk + name='interests_recall_at_%d' % topk, ) metric_dict['interests_neg_sam_recall@%d' % topk] = metrics.recall_at_k( labels=simple_lbls_v2, predictions=simple_item_sim_v2, k=topk, - name='interests_recall_neg_sam_at_%d' % topk + name='interests_recall_neg_sam_at_%d' % topk, ) logits = self._prediction_dict['logits'] - pos_item_logits = tf.gather_nd( - logits[:batch_size, :batch_size], pos_indices - ) - logits_v2 = tf.concat( - [pos_item_logits[:, None], logits[:, batch_size:]], axis=1 - ) + pos_item_logits = tf.gather_nd(logits[:batch_size, :batch_size], pos_indices) + logits_v2 = tf.concat([pos_item_logits[:, None], logits[:, batch_size:]], axis=1) labels_v2 = tf.zeros_like(logits_v2[:, :1], dtype=tf.int64) for topk in recall_at_topks: @@ -422,78 +342,85 @@ def build_metric_graph(self, eval_config): labels=simple_lbls, predictions=logits, k=topk, - name='recall_at_%d' % topk + name='recall_at_%d' % topk, ) metric_dict['recall_neg_sam@%d' % topk] = metrics.recall_at_k( labels=labels_v2, predictions=logits_v2, k=topk, - name='recall_neg_sam_at_%d' % topk + name='recall_neg_sam_at_%d' % topk, ) eval_logits = logits[:, :batch_size] eval_logits = tf.cond( - batch_size < topk, lambda: tf.pad( - eval_logits, [[0, 0], [0, topk - batch_size]], + batch_size < topk, + lambda: tf.pad( + eval_logits, + [[0, 0], [0, topk - batch_size]], mode='CONSTANT', constant_values=-1e32, - name='pad_eval_logits' - ), lambda: eval_logits + name='pad_eval_logits', + ), + lambda: eval_logits, ) metric_dict['recall_in_batch@%d' % topk] = metrics.recall_at_k( labels=simple_lbls, predictions=eval_logits, k=topk, - name='recall_in_batch_at_%d' % topk + name='recall_in_batch_at_%d' % topk, ) # batch_size num_interest if hard_neg_indices is not None: hard_neg_user_emb = tf.gather(user_interests, hard_neg_indices[:, 0]) - hard_neg_sim = tf.einsum( - 'nhe,ne->nh', hard_neg_user_emb, hard_neg_item_emb - ) + hard_neg_sim = tf.einsum('nhe,ne->nh', hard_neg_user_emb, hard_neg_item_emb) hard_neg_sim = tf.reduce_max(hard_neg_sim, axis=1) max_num_neg = tf.reduce_max(hard_neg_indices[:, 1]) + 1 hard_neg_shape = tf.stack([tf.to_int64(batch_size), max_num_neg]) hard_neg_mask = tf.scatter_nd( hard_neg_indices, tf.ones_like(hard_neg_sim, dtype=tf.float32), - shape=hard_neg_shape - ) - hard_neg_sim = tf.scatter_nd( - hard_neg_indices, hard_neg_sim, hard_neg_shape + shape=hard_neg_shape, ) + hard_neg_sim = tf.scatter_nd(hard_neg_indices, hard_neg_sim, hard_neg_shape) hard_neg_sim = hard_neg_sim - (1 - hard_neg_mask) * 1e32 hard_logits = tf.concat([pos_item_logits[:, None], hard_neg_sim], axis=1) hard_lbls = tf.zeros_like(hard_logits[:, :1], dtype=tf.int64) - metric_dict['hard_neg_acc'] = metrics.accuracy( - hard_lbls, tf.argmax(hard_logits, axis=1) - ) + metric_dict['hard_neg_acc'] = metrics.accuracy(hard_lbls, tf.argmax(hard_logits, axis=1)) return metric_dict def get_outputs(self): if self._loss_type == LossType.CLASSIFICATION: return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_emb_num', - 'user_interests', 'item_tower_emb' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_emb_num', + 'user_interests', + 'item_tower_emb', ] elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: - self._prediction_dict['logits'] = tf.squeeze( - self._prediction_dict['logits'], axis=-1 - ) - self._prediction_dict['probs'] = tf.nn.sigmoid( - self._prediction_dict['logits'] - ) + self._prediction_dict['logits'] = tf.squeeze(self._prediction_dict['logits'], axis=-1) + self._prediction_dict['probs'] = tf.nn.sigmoid(self._prediction_dict['logits']) return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_emb_num', - 'user_interests', 'item_tower_emb' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_emb_num', + 'user_interests', + 'item_tower_emb', ] elif self._loss_type == LossType.L2_LOSS: return [ - 'y', 'user_emb', 'item_emb', 'user_emb_num', 'user_interests', - 'item_tower_emb' + 'y', + 'user_emb', + 'item_emb', + 'user_emb_num', + 'user_interests', + 'item_tower_emb', ] else: raise ValueError('invalid loss type: %s' % str(self._loss_type)) diff --git a/easy_rec/python/model/mmoe.py b/easy_rec/python/model/mmoe.py index f432eacf7..66e8185d6 100644 --- a/easy_rec/python/model/mmoe.py +++ b/easy_rec/python/model/mmoe.py @@ -11,20 +11,11 @@ class MMoE(MultiTaskModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(MMoE, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(MMoE, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'mmoe', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model' ) - assert self._model_config.WhichOneof('model') == 'mmoe', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.mmoe assert isinstance(self._model_config, MMoEConfig) @@ -40,14 +31,14 @@ def build_predict_graph(self): self._model_config.expert_dnn, l2_reg=self._l2_reg, num_task=self._task_num, - num_expert=self._model_config.num_expert + num_expert=self._model_config.num_expert, ) else: # For backward compatibility with original mmoe layer config mmoe_layer = mmoe.MMOE( [x.dnn for x in self._model_config.experts], l2_reg=self._l2_reg, - num_task=self._task_num + num_task=self._task_num, ) task_input_list = mmoe_layer(self._features) @@ -60,7 +51,7 @@ def build_predict_graph(self): task_tower_cfg.dnn, self._l2_reg, name=tower_name, - is_training=self._is_training + is_training=self._is_training, ) tower_output = tower_dnn(task_input_list[i]) else: @@ -69,7 +60,7 @@ def build_predict_graph(self): inputs=tower_output, units=task_tower_cfg.num_class, kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i + name='dnn_output_%d' % i, ) tower_outputs[tower_name] = tower_output diff --git a/easy_rec/python/model/multi_task_model.py b/easy_rec/python/model/multi_task_model.py index 07a54a6ba..e09ab9cd1 100644 --- a/easy_rec/python/model/multi_task_model.py +++ b/easy_rec/python/model/multi_task_model.py @@ -17,27 +17,15 @@ class MultiTaskModel(RankModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(MultiTaskModel, self).__init__( - model_config, feature_configs, features, labels, is_training - ) + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(MultiTaskModel, self).__init__(model_config, feature_configs, features, labels, is_training) self._task_towers = [] self._task_num = None self._label_name_dict = {} def build_predict_graph(self): if not self.has_backbone: - raise NotImplementedError( - 'method `build_predict_graph` must be implemented when backbone network do not exists' - ) + raise NotImplementedError('method `build_predict_graph` must be implemented when backbone network do not exists') model = self._model_config.WhichOneof('model') assert model == 'model_params', '`model_params` must be configured' config = self._model_config.model_params @@ -49,9 +37,7 @@ def build_predict_graph(self): backbone = self.backbone if type(backbone) in (list, tuple): if len(backbone) != len(config.task_towers): - raise ValueError( - 'The number of backbone outputs and task towers must be equal' - ) + raise ValueError('The number of backbone outputs and task towers must be equal') task_input_list = backbone else: task_input_list = [backbone] * len(config.task_towers) @@ -65,7 +51,7 @@ def build_predict_graph(self): task_tower_cfg.dnn, self._l2_reg, name=tower_name, - is_training=self._is_training + is_training=self._is_training, ) tower_output = tower_dnn(task_input_list[i]) else: @@ -83,14 +69,12 @@ def build_predict_graph(self): task_tower_cfg.relation_dnn, self._l2_reg, name=tower_name + '/relation_dnn', - is_training=self._is_training + is_training=self._is_training, ) tower_inputs = [tower_features[tower_name]] for relation_tower_name in task_tower_cfg.relation_tower_names: tower_inputs.append(relation_features[relation_tower_name]) - relation_input = tf.concat( - tower_inputs, axis=-1, name=tower_name + '/relation_input' - ) + relation_input = tf.concat(tower_inputs, axis=-1, name=tower_name + '/relation_input') relation_fea = relation_dnn(relation_input) relation_features[tower_name] = relation_fea else: @@ -100,7 +84,7 @@ def build_predict_graph(self): relation_fea, task_tower_cfg.num_class, kernel_regularizer=self._l2_reg, - name=tower_name + '/output' + name=tower_name + '/output', ) tower_outputs[tower_name] = output_logits @@ -112,9 +96,9 @@ def _init_towers(self, task_tower_configs): self._task_towers = task_tower_configs self._task_num = len(task_tower_configs) for i, task_tower_config in enumerate(task_tower_configs): - assert isinstance(task_tower_config, tower_pb2.TaskTower) or \ - isinstance(task_tower_config, tower_pb2.BayesTaskTower), \ - 'task_tower_config must be a instance of tower_pb2.TaskTower or tower_pb2.BayesTaskTower' + assert isinstance(task_tower_config, tower_pb2.TaskTower) or isinstance( + task_tower_config, tower_pb2.BayesTaskTower + ), 'task_tower_config must be a instance of tower_pb2.TaskTower or tower_pb2.BayesTaskTower' tower_name = task_tower_config.tower_name # For label backward compatibility with list @@ -124,9 +108,7 @@ def _init_towers(self, task_tower_configs): else: # If label name is not specified, task_tower and label will be matched by order label_name = list(self._labels.keys())[i] - logging.info( - 'Task Tower [%s] use label [%s]' % (tower_name, label_name) - ) + logging.info('Task Tower [%s] use label [%s]' % (tower_name, label_name)) assert label_name in self._labels, 'label [%s] must exists in labels' % label_name self._label_name_dict[tower_name] = label_name @@ -139,7 +121,7 @@ def _add_to_prediction_dict(self, output): output[tower_name], loss_type=task_tower_cfg.loss_type, num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name + suffix='_%s' % tower_name, ) ) else: @@ -149,7 +131,7 @@ def _add_to_prediction_dict(self, output): output[tower_name], loss_type=loss.loss_type, num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name + suffix='_%s' % tower_name, ) ) @@ -167,7 +149,7 @@ def build_metric_graph(self, eval_config): loss_type=loss_types, label_name=self._label_name_dict[tower_name], num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name + suffix='_%s' % tower_name, ) ) return self._metric_dict @@ -180,9 +162,7 @@ def build_loss_weight(self): losses = task_tower_cfg.losses n = len(losses) if n > 0: - loss_weights[tower_name] = [ - loss.weight * task_tower_cfg.weight for loss in losses - ] + loss_weights[tower_name] = [loss.weight * task_tower_cfg.weight for loss in losses] num_loss += n else: loss_weights[tower_name] = [task_tower_cfg.weight] @@ -195,16 +175,14 @@ def build_loss_weight(self): i = 0 for k, v in loss_weights.items(): n = len(v) - loss_weights[k] = weights[i:i + n] + loss_weights[k] = weights[i : i + n] i += n return loss_weights def get_learnt_loss(self, loss_type, name, value): strategy = self._base_model_config.loss_weight_strategy if strategy == self._base_model_config.Uncertainty: - uncertainty = tf.Variable( - 0, name='%s_loss_weight' % name, dtype=tf.float32 - ) + uncertainty = tf.Variable(0, name='%s_loss_weight' % name, dtype=tf.float32) tf.summary.scalar('loss/%s_uncertainty' % name, uncertainty) if loss_type in {LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS}: return 0.5 * tf.exp(-uncertainty) * value + 0.5 * uncertainty @@ -223,27 +201,25 @@ def build_loss_graph(self): if task_tower_cfg.use_sample_weight: loss_weight *= self._sample_weight - if hasattr(task_tower_cfg, 'task_space_indicator_label') and \ - task_tower_cfg.HasField('task_space_indicator_label'): - in_task_space = tf.to_float( - self._labels[task_tower_cfg.task_space_indicator_label] > 0 - ) + if hasattr(task_tower_cfg, 'task_space_indicator_label') and task_tower_cfg.HasField( + 'task_space_indicator_label' + ): + in_task_space = tf.to_float(self._labels[task_tower_cfg.task_space_indicator_label] > 0) loss_weight = loss_weight * ( - task_tower_cfg.in_task_space_weight * in_task_space + - task_tower_cfg.out_task_space_weight * (1 - in_task_space) + task_tower_cfg.in_task_space_weight * in_task_space + + task_tower_cfg.out_task_space_weight * (1 - in_task_space) ) - if task_tower_cfg.HasField('task_space_indicator_name') and \ - task_tower_cfg.HasField('task_space_indicator_value'): + if task_tower_cfg.HasField('task_space_indicator_name') and task_tower_cfg.HasField('task_space_indicator_value'): in_task_space = tf.to_float( tf.equal( self._feature_dict[task_tower_cfg.task_space_indicator_name], - task_tower_cfg.task_space_indicator_value + task_tower_cfg.task_space_indicator_value, ) ) loss_weight = loss_weight * ( - task_tower_cfg.in_task_space_weight * in_task_space + - task_tower_cfg.out_task_space_weight * (1 - in_task_space) + task_tower_cfg.in_task_space_weight * in_task_space + + task_tower_cfg.out_task_space_weight * (1 - in_task_space) ) task_loss_weight = task_loss_weights[tower_name] @@ -255,7 +231,7 @@ def build_loss_graph(self): label_name=self._label_name_dict[tower_name], loss_weight=loss_weight, num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name + suffix='_%s' % tower_name, ) for loss_name in loss_dict.keys(): loss_dict[loss_name] = loss_dict[loss_name] * task_loss_weight[0] @@ -268,9 +244,7 @@ def build_loss_graph(self): y_rt = self._prediction_dict['probs_%s' % relation_tower_name] cali_loss = tf.reduce_mean(tf.nn.relu(y_t - y_rt)) calibrate_loss.append(cali_loss * loss.weight) - logging.info( - 'calibrate loss: %s -> %s' % (relation_tower_name, tower_name) - ) + logging.info('calibrate loss: %s -> %s' % (relation_tower_name, tower_name)) continue loss_param = loss.WhichOneof('loss_param') if loss_param is not None: @@ -282,14 +256,12 @@ def build_loss_graph(self): num_class=task_tower_cfg.num_class, suffix='_%s' % tower_name, loss_name=loss.loss_name, - loss_param=loss_param + loss_param=loss_param, ) for i, loss_name in enumerate(loss_ops): loss_value = loss_ops[loss_name] if loss.learn_loss_weight: - loss_dict[loss_name] = self.get_learnt_loss( - loss.loss_type, loss_name, loss_value - ) + loss_dict[loss_name] = self.get_learnt_loss(loss.loss_type, loss_name, loss_value) else: loss_dict[loss_name] = loss_value * task_loss_weight[i] if calibrate_loss: @@ -298,9 +270,7 @@ def build_loss_graph(self): tf.summary.scalar('loss/order_calibrate_loss', cali_loss) self._loss_dict.update(loss_dict) - kd_loss_dict = loss_builder.build_kd_loss( - self.kd, self._prediction_dict, self._labels, self._feature_dict - ) + kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, self._labels, self._feature_dict) self._loss_dict.update(kd_loss_dict) return self._loss_dict @@ -316,7 +286,7 @@ def get_outputs(self): self._get_outputs_impl( task_tower_cfg.loss_type, task_tower_cfg.num_class, - suffix='_%s' % tower_name + suffix='_%s' % tower_name, ) ) else: @@ -327,7 +297,7 @@ def get_outputs(self): self._get_outputs_impl( loss.loss_type, task_tower_cfg.num_class, - suffix='_%s' % tower_name + suffix='_%s' % tower_name, ) ) return list(set(outputs)) diff --git a/easy_rec/python/model/multi_tower.py b/easy_rec/python/model/multi_tower.py index aa4c7ecd0..648ed0774 100644 --- a/easy_rec/python/model/multi_tower.py +++ b/easy_rec/python/model/multi_tower.py @@ -5,7 +5,6 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': @@ -13,18 +12,8 @@ class MultiTower(RankModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(MultiTower, self).__init__( - model_config, feature_configs, features, labels, is_training - ) + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(MultiTower, self).__init__(model_config, feature_configs, features, labels, is_training) assert self._model_config.WhichOneof('model') == 'multi_tower', ( 'invalid model config: %s' % self._model_config.WhichOneof('model') ) @@ -48,20 +37,15 @@ def build_predict_graph(self): tower_fea, training=self._is_training, trainable=True, - name='%s_fea_bn' % tower_name + name='%s_fea_bn' % tower_name, ) - tower_dnn_layer = dnn.DNN( - tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training - ) + tower_dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training) tower_fea = tower_dnn_layer(tower_fea) tower_fea_arr.append(tower_fea) all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn_layer = dnn.DNN( - self._model_config.final_dnn, self._l2_reg, 'final_dnn', - self._is_training - ) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/multi_tower_bst.py b/easy_rec/python/model/multi_tower_bst.py index 67f620341..99dc612cf 100644 --- a/easy_rec/python/model/multi_tower_bst.py +++ b/easy_rec/python/model/multi_tower_bst.py @@ -8,7 +8,6 @@ from easy_rec.python.compat import regularizers from easy_rec.python.layers import dnn, layer_norm, seq_input_layer from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': @@ -16,26 +15,17 @@ class MultiTowerBST(RankModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(MultiTowerBST, self).__init__( - model_config, feature_configs, features, labels, is_training - ) + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(MultiTowerBST, self).__init__(model_config, feature_configs, features, labels, is_training) self._seq_input_layer = seq_input_layer.SeqInputLayer( feature_configs, model_config.seq_att_groups, embedding_regularizer=self._emb_reg, - ev_params=self._global_ev_params + ev_params=self._global_ev_params, + ) + assert self._model_config.WhichOneof('model') == 'multi_tower', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model') ) - assert self._model_config.WhichOneof('model') == 'multi_tower', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.multi_tower assert isinstance(self._model_config, MultiTowerConfig) @@ -49,20 +39,14 @@ def __init__( self._bst_tower_features = [] self._bst_tower_num = len(self._model_config.bst_towers) - logging.info( - 'all tower num: {0}'.format(self._tower_num + self._bst_tower_num) - ) + logging.info('all tower num: {0}'.format(self._tower_num + self._bst_tower_num)) logging.info('bst tower num: {0}'.format(self._bst_tower_num)) for tower_id in range(self._bst_tower_num): tower = self._model_config.bst_towers[tower_id] tower_feature = self._seq_input_layer(self._feature_dict, tower.input) - regularizers.apply_regularization( - self._emb_reg, weights_list=[tower_feature['key']] - ) - regularizers.apply_regularization( - self._emb_reg, weights_list=[tower_feature['hist_seq_emb']] - ) + regularizers.apply_regularization(self._emb_reg, weights_list=[tower_feature['key']]) + regularizers.apply_regularization(self._emb_reg, weights_list=[tower_feature['hist_seq_emb']]) self._bst_tower_features.append(tower_feature) def dnn_net(self, net, dnn_units, name): @@ -74,33 +58,23 @@ def dnn_net(self, net, dnn_units, name): net, units=units, activation=tf.nn.relu, - name='%s_%d' % (name, idx) + name='%s_%d' % (name, idx), ) else: - net = tf.layers.dense( - net, units=units, activation=None, name='%s_%d' % (name, idx) - ) + net = tf.layers.dense(net, units=units, activation=None, name='%s_%d' % (name, idx)) return net def attention_net(self, net, dim, cur_seq_len, seq_size, name): query_net = self.dnn_net(net, [dim], name + '_query') # B, seq_len，dim key_net = self.dnn_net(net, [dim], name + '_key') value_net = self.dnn_net(net, [dim], name + '_value') - scores = tf.matmul( - query_net, key_net, transpose_b=True - ) # [B, seq_size, seq_size] - - hist_mask = tf.sequence_mask( - cur_seq_len, maxlen=seq_size - 1 - ) # [B, seq_size-1] - cur_id_mask = tf.ones( - tf.stack([tf.shape(hist_mask)[0], 1]), dtype=tf.bool - ) # [B, 1] + scores = tf.matmul(query_net, key_net, transpose_b=True) # [B, seq_size, seq_size] + + hist_mask = tf.sequence_mask(cur_seq_len, maxlen=seq_size - 1) # [B, seq_size-1] + cur_id_mask = tf.ones(tf.stack([tf.shape(hist_mask)[0], 1]), dtype=tf.bool) # [B, 1] mask = tf.concat([hist_mask, cur_id_mask], axis=1) # [B, seq_size] - masks = tf.reshape( - tf.tile(mask, [1, seq_size]), (-1, seq_size, seq_size) - ) # [B, seq_size, seq_size] - padding = tf.ones_like(scores) * (-2**32 + 1) + masks = tf.reshape(tf.tile(mask, [1, seq_size]), (-1, seq_size, seq_size)) # [B, seq_size, seq_size] + padding = tf.ones_like(scores) * (-(2**32) + 1) scores = tf.where(masks, scores, padding) # [B, seq_size, seq_size] # Scale @@ -108,29 +82,23 @@ def attention_net(self, net, dim, cur_seq_len, seq_size, name): att_res_net = tf.matmul(scores, value_net) # [B, seq_size, emb_dim] return att_res_net - def multi_head_att_net( - self, id_cols, head_count, emb_dim, seq_len, seq_size - ): + def multi_head_att_net(self, id_cols, head_count, emb_dim, seq_len, seq_size): multi_head_attention_res = [] part_cols_emd_dim = int(math.ceil(emb_dim / head_count)) for start_idx in range(0, emb_dim, part_cols_emd_dim): if start_idx + part_cols_emd_dim > emb_dim: part_cols_emd_dim = emb_dim - start_idx - part_id_col = tf.slice( - id_cols, [0, 0, start_idx], [-1, -1, part_cols_emd_dim] - ) + part_id_col = tf.slice(id_cols, [0, 0, start_idx], [-1, -1, part_cols_emd_dim]) part_attention_net = self.attention_net( part_id_col, part_cols_emd_dim, seq_len, seq_size, - name='multi_head_%d' % start_idx + name='multi_head_%d' % start_idx, ) multi_head_attention_res.append(part_attention_net) multi_head_attention_res_net = tf.concat(multi_head_attention_res, axis=2) - multi_head_attention_res_net = self.dnn_net( - multi_head_attention_res_net, [emb_dim], name='multi_head_attention' - ) + multi_head_attention_res_net = self.dnn_net(multi_head_attention_res_net, [emb_dim], name='multi_head_attention') return multi_head_attention_res_net def add_and_norm(self, net_1, net_2, emb_dim, name): @@ -141,33 +109,31 @@ def add_and_norm(self, net_1, net_2, emb_dim, name): return net def bst(self, bst_fea, seq_size, head_count, name): - cur_id, hist_id_col, seq_len = bst_fea['key'], bst_fea[ - 'hist_seq_emb'], bst_fea['hist_seq_len'] + cur_id, hist_id_col, seq_len = ( + bst_fea['key'], + bst_fea['hist_seq_emb'], + bst_fea['hist_seq_len'], + ) cur_batch_max_seq_len = tf.shape(hist_id_col)[1] hist_id_col = tf.cond( - tf.constant(seq_size) > cur_batch_max_seq_len, lambda: tf.pad( + tf.constant(seq_size) > cur_batch_max_seq_len, + lambda: tf.pad( hist_id_col, - [[0, 0], [0, seq_size - cur_batch_max_seq_len - 1], [0, 0]], 'CONSTANT' - ), lambda: tf.slice(hist_id_col, [0, 0, 0], [-1, seq_size - 1, -1]) + [[0, 0], [0, seq_size - cur_batch_max_seq_len - 1], [0, 0]], + 'CONSTANT', + ), + lambda: tf.slice(hist_id_col, [0, 0, 0], [-1, seq_size - 1, -1]), ) - all_ids = tf.concat( - [hist_id_col, tf.expand_dims(cur_id, 1)], axis=1 - ) # b, seq_size, emb_dim + all_ids = tf.concat([hist_id_col, tf.expand_dims(cur_id, 1)], axis=1) # b, seq_size, emb_dim emb_dim = int(all_ids.shape[2]) - attention_net = self.multi_head_att_net( - all_ids, head_count, emb_dim, seq_len, seq_size - ) + attention_net = self.multi_head_att_net(all_ids, head_count, emb_dim, seq_len, seq_size) - tmp_net = self.add_and_norm( - all_ids, attention_net, emb_dim, name='add_and_norm_1' - ) + tmp_net = self.add_and_norm(all_ids, attention_net, emb_dim, name='add_and_norm_1') feed_forward_net = self.dnn_net(tmp_net, [emb_dim], 'feed_forward_net') - net = self.add_and_norm( - tmp_net, feed_forward_net, emb_dim, name='add_and_norm_2' - ) + net = self.add_and_norm(tmp_net, feed_forward_net, emb_dim, name='add_and_norm_2') bst_output = tf.reshape(net, [-1, seq_size * emb_dim]) return bst_output @@ -181,11 +147,9 @@ def build_predict_graph(self): tower_fea, training=self._is_training, trainable=True, - name='%s_fea_bn' % tower_name - ) - tower_dnn = dnn.DNN( - tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training + name='%s_fea_bn' % tower_name, ) + tower_dnn = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training) tower_fea = tower_dnn(tower_fea) tower_fea_arr.append(tower_fea) @@ -199,15 +163,12 @@ def build_predict_graph(self): tower_fea, seq_size=tower_seq_len, head_count=tower_multi_head_size, - name=tower_name + name=tower_name, ) tower_fea_arr.append(tower_fea) all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn = dnn.DNN( - self._model_config.final_dnn, self._l2_reg, 'final_dnn', - self._is_training - ) + final_dnn = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) all_fea = final_dnn(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/multi_tower_din.py b/easy_rec/python/model/multi_tower_din.py index 9736caf93..1a92f1d94 100644 --- a/easy_rec/python/model/multi_tower_din.py +++ b/easy_rec/python/model/multi_tower_din.py @@ -7,7 +7,6 @@ from easy_rec.python.compat import regularizers from easy_rec.python.layers import dnn, seq_input_layer from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': @@ -15,26 +14,17 @@ class MultiTowerDIN(RankModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(MultiTowerDIN, self).__init__( - model_config, feature_configs, features, labels, is_training - ) + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(MultiTowerDIN, self).__init__(model_config, feature_configs, features, labels, is_training) self._seq_input_layer = seq_input_layer.SeqInputLayer( feature_configs, model_config.seq_att_groups, embedding_regularizer=self._emb_reg, - ev_params=self._global_ev_params + ev_params=self._global_ev_params, + ) + assert self._model_config.WhichOneof('model') == 'multi_tower', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model') ) - assert self._model_config.WhichOneof('model') == 'multi_tower', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.multi_tower assert isinstance(self._model_config, MultiTowerConfig) @@ -48,9 +38,7 @@ def __init__( self._din_tower_features = [] self._din_tower_num = len(self._model_config.din_towers) - logging.info( - 'all tower num: {0}'.format(self._tower_num + self._din_tower_num) - ) + logging.info('all tower num: {0}'.format(self._tower_num + self._din_tower_num)) logging.info('din tower num: {0}'.format(self._din_tower_num)) for tower_id in range(self._din_tower_num): @@ -59,26 +47,25 @@ def __init__( # apply regularization for sequence feature key in seq_input_layer. - regularizers.apply_regularization( - self._emb_reg, weights_list=[tower_feature['hist_seq_emb']] - ) + regularizers.apply_regularization(self._emb_reg, weights_list=[tower_feature['hist_seq_emb']]) self._din_tower_features.append(tower_feature) def din(self, dnn_config, deep_fea, name): - cur_id, hist_id_col, seq_len = deep_fea['key'], deep_fea[ - 'hist_seq_emb'], deep_fea['hist_seq_len'] + cur_id, hist_id_col, seq_len = ( + deep_fea['key'], + deep_fea['hist_seq_emb'], + deep_fea['hist_seq_len'], + ) seq_max_len = tf.shape(hist_id_col)[1] emb_dim = hist_id_col.shape[2] cur_ids = tf.tile(cur_id, [1, seq_max_len]) - cur_ids = tf.reshape( - cur_ids, tf.shape(hist_id_col) - ) # (B, seq_max_len, emb_dim) + cur_ids = tf.reshape(cur_ids, tf.shape(hist_id_col)) # (B, seq_max_len, emb_dim) din_net = tf.concat( [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1 + axis=-1, ) # (B, seq_max_len, emb_dim*4) din_layer = dnn.DNN( @@ -87,14 +74,14 @@ def din(self, dnn_config, deep_fea, name): name, self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True + last_layer_no_batch_norm=True, ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) seq_len = tf.expand_dims(seq_len, 1) mask = tf.sequence_mask(seq_len) - padding = tf.ones_like(scores) * (-2**32 + 1) + padding = tf.ones_like(scores) * (-(2**32) + 1) scores = tf.where(mask, scores, padding) # [B, 1, seq_max_len] # Scale @@ -114,11 +101,9 @@ def build_predict_graph(self): tower_fea, training=self._is_training, trainable=True, - name='%s_fea_bn' % tower_name - ) - dnn_layer = dnn.DNN( - tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training + name='%s_fea_bn' % tower_name, ) + dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training) tower_fea = dnn_layer(tower_fea) tower_fea_arr.append(tower_fea) @@ -130,10 +115,7 @@ def build_predict_graph(self): tower_fea_arr.append(tower_fea) all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn_layer = dnn.DNN( - self._model_config.final_dnn, self._l2_reg, 'final_dnn', - self._is_training - ) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/multi_tower_recall.py b/easy_rec/python/model/multi_tower_recall.py index bc12b4fc3..733b689ee 100644 --- a/easy_rec/python/model/multi_tower_recall.py +++ b/easy_rec/python/model/multi_tower_recall.py @@ -5,7 +5,6 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.multi_tower_recall_pb2 import MultiTowerRecall as MultiTowerRecallConfig # NOQA if tf.__version__ >= '2.0': @@ -13,18 +12,8 @@ class MultiTowerRecall(RankModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(MultiTowerRecall, self).__init__( - model_config, feature_configs, features, labels, is_training - ) + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(MultiTowerRecall, self).__init__(model_config, feature_configs, features, labels, is_training) assert self._model_config.WhichOneof('model') == 'multi_tower_recall', ( 'invalid model config: %s' % self._model_config.WhichOneof('model') ) @@ -35,7 +24,6 @@ def __init__( self.item_tower_feature, _ = self._input_layer(self._feature_dict, 'item') def build_predict_graph(self): - user_tower_feature = self.user_tower_feature batch_size = tf.shape(user_tower_feature)[0] pos_item_feature = self.item_tower_feature[:batch_size] @@ -43,22 +31,24 @@ def build_predict_graph(self): item_tower_feature = tf.concat( [ pos_item_feature[:, tf.newaxis, :], - tf.tile( - neg_item_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1] - ) + tf.tile(neg_item_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1]), ], - axis=1 + axis=1, ) # noqa: E126 user_dnn = dnn.DNN( - self._model_config.user_tower.dnn, self._l2_reg, 'user_dnn', - self._is_training + self._model_config.user_tower.dnn, + self._l2_reg, + 'user_dnn', + self._is_training, ) user_tower_emb = user_dnn(user_tower_feature) item_dnn = dnn.DNN( - self._model_config.item_tower.dnn, self._l2_reg, 'item_dnn', - self._is_training + self._model_config.item_tower.dnn, + self._l2_reg, + 'item_dnn', + self._is_training, ) item_tower_emb = item_dnn(item_tower_feature) item_tower_emb = tf.reshape(item_tower_emb, tf.shape(user_tower_emb)) @@ -68,10 +58,7 @@ def build_predict_graph(self): tower_fea_arr.append(item_tower_emb) all_fea = tf.concat(tower_fea_arr, axis=-1) - final_dnn_layer = dnn.DNN( - self._model_config.final_dnn, self._l2_reg, 'final_dnn', - self._is_training - ) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, 1, name='output') output = output[:, 0] diff --git a/easy_rec/python/model/pdn.py b/easy_rec/python/model/pdn.py index 16ffde39a..279a840fc 100644 --- a/easy_rec/python/model/pdn.py +++ b/easy_rec/python/model/pdn.py @@ -14,20 +14,11 @@ class PDN(MatchModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(PDN, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(PDN, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'pdn', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model' ) - assert self._model_config.WhichOneof('model') == 'pdn', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.pdn self._user_features, _ = self._input_layer(self._feature_dict, 'user') @@ -47,10 +38,7 @@ def build_predict_graph(self): sim_out = self._build_similarity_net() logits = tf.multiply(sim_out, trigger_out) - seq_mask = tf.to_float( - tf.sequence_mask(self._seq_len, - tf.shape(sim_out)[1]) - ) + seq_mask = tf.to_float(tf.sequence_mask(self._seq_len, tf.shape(sim_out)[1])) logits = tf.reduce_sum(logits * seq_mask[:, :, None], axis=1) direct_logits = self._build_direct_net() @@ -65,9 +53,7 @@ def build_predict_graph(self): probs = 1 - tf.exp(-logits) # map [0, inf) to [0, 1) self._prediction_dict['probs'] = probs - self._prediction_dict['logits'] = tf.log( - tf.clip_by_value(probs, 1e-8, 1 - 1e-8) - ) + self._prediction_dict['logits'] = tf.log(tf.clip_by_value(probs, 1e-8, 1 - 1e-8)) return self._prediction_dict def _get_seq_features(self, name): @@ -77,15 +63,11 @@ def _get_seq_features(self, name): return seq, seq_len def _build_trigger_net(self): - user_dnn_layer = dnn.DNN( - self._model_config.user_dnn, self._l2_reg, 'user_dnn', self._is_training - ) + user_dnn_layer = dnn.DNN(self._model_config.user_dnn, self._l2_reg, 'user_dnn', self._is_training) user_fea = user_dnn_layer(self._user_features) trigger_seq = tf.concat([self._u2i_seq, self._i_seq], axis=2) - u2i_dnn_layer = dnn.DNN( - self._model_config.u2i_dnn, self._l2_reg, 'u2i_dnn', self._is_training - ) + u2i_dnn_layer = dnn.DNN(self._model_config.u2i_dnn, self._l2_reg, 'u2i_dnn', self._is_training) trigger_seq_fea = u2i_dnn_layer(trigger_seq) trigger_merge_fea = trigger_seq_fea + user_fea[:, None, :] @@ -95,7 +77,7 @@ def _build_trigger_net(self): 'trigger_dnn', self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True + last_layer_no_batch_norm=True, ) # output: N x seq_len x d, d is usually set to 1 @@ -107,40 +89,32 @@ def _build_trigger_net(self): tf.reduce_join( tf.as_string(trigger_out, precision=4, shortest=True), axis=2, - separator=',' + separator=',', ), axis=1, - separator=';' + separator=';', ) return trigger_out def _build_similarity_net(self): - item_dnn_layer = dnn.DNN( - self._model_config.item_dnn, self._l2_reg, 'item_dnn', self._is_training - ) + item_dnn_layer = dnn.DNN(self._model_config.item_dnn, self._l2_reg, 'item_dnn', self._is_training) item_fea = item_dnn_layer(self._item_features) - sim_side_dnn_layer = dnn.DNN( - self._model_config.i2i_dnn, self._l2_reg, 'i2i_dnn', self._is_training - ) + sim_side_dnn_layer = dnn.DNN(self._model_config.i2i_dnn, self._l2_reg, 'i2i_dnn', self._is_training) sim_seq_fea = sim_side_dnn_layer(self._i_seq) sim_seq_cross = sim_seq_fea * item_fea[:, None, :] - item_fea_tile = tf.tile( - item_fea[:, None, :], [1, tf.shape(sim_seq_fea)[1], 1] - ) + item_fea_tile = tf.tile(item_fea[:, None, :], [1, tf.shape(sim_seq_fea)[1], 1]) - sim_seq_concat = tf.concat( - [sim_seq_cross, sim_seq_cross, self._i2i_seq, item_fea_tile], axis=2 - ) + sim_seq_concat = tf.concat([sim_seq_cross, sim_seq_cross, self._i2i_seq, item_fea_tile], axis=2) sim_dnn_layer = dnn.DNN( self._model_config.sim_dnn, self._l2_reg, 'sim_dnn', self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True + last_layer_no_batch_norm=True, ) # output: N x seq_len x 1 sim_out = sim_dnn_layer(sim_seq_concat) @@ -148,25 +122,20 @@ def _build_similarity_net(self): sim_out = tf.exp(sim_out) self._prediction_dict['sim_out'] = tf.reduce_join( - tf.reduce_join( - tf.as_string(sim_out, precision=4, shortest=True), - axis=2, - separator=',' - ), + tf.reduce_join(tf.as_string(sim_out, precision=4, shortest=True), axis=2, separator=','), axis=1, - separator=';' + separator=';', ) return sim_out def _build_direct_net(self): - if self._model_config.HasField('direct_user_dnn') and \ - self._model_config.HasField('direct_item_dnn'): + if self._model_config.HasField('direct_user_dnn') and self._model_config.HasField('direct_item_dnn'): direct_user_layer = dnn.DNN( self._model_config.direct_user_dnn, 'direct_user_dnn', self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True + last_layer_no_batch_norm=True, ) direct_user_out = direct_user_layer(self._user_features) direct_item_layer = dnn.DNN( @@ -174,7 +143,7 @@ def _build_direct_net(self): 'direct_item_dnn', self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True + last_layer_no_batch_norm=True, ) direct_item_out = direct_item_layer(self._item_features) @@ -191,13 +160,13 @@ def _build_direct_net(self): 'direct_net/sim_w', dtype=tf.float32, shape=(1), - initializer=tf.ones_initializer() + initializer=tf.ones_initializer(), ) sim_b = tf.get_variable( 'direct_net/sim_b', dtype=tf.float32, shape=(1), - initializer=tf.zeros_initializer() + initializer=tf.zeros_initializer(), ) direct_logits = direct_logits * tf.abs(sim_w) + sim_b @@ -214,7 +183,7 @@ def _build_bias_net(self): 'bias_dnn', self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True + last_layer_no_batch_norm=True, ) bias_logits = bias_dnn_layer(self._bias_features) return tf.nn.softplus(bias_logits) diff --git a/easy_rec/python/model/ple.py b/easy_rec/python/model/ple.py index 942d52eee..f04d9a463 100644 --- a/easy_rec/python/model/ple.py +++ b/easy_rec/python/model/ple.py @@ -11,20 +11,11 @@ class PLE(MultiTaskModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(PLE, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(PLE, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'ple', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model' ) - assert self._model_config.WhichOneof('model') == 'ple', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.ple assert isinstance(self._model_config, PLEConfig) @@ -43,7 +34,7 @@ def gate(self, selector_fea, vec_feas, name): units=len(vec_feas), kernel_regularizer=self._l2_reg, activation=None, - name=name + '_gate/dnn' + name=name + '_gate/dnn', ) gate = tf.nn.softmax(gate, axis=1) gate = tf.expand_dims(gate, -1) @@ -58,20 +49,25 @@ def experts_layer(self, deep_fea, expert_num, experts_cfg, name): experts_cfg, self._l2_reg, name=name + '_expert_%d/dnn' % expert_id, - is_training=self._is_training + is_training=self._is_training, ) tower_output = tower_dnn(deep_fea) tower_outputs.append(tower_output) return tower_outputs def CGC_layer( - self, extraction_networks_cfg, extraction_network_fea, shared_expert_fea, - final_flag + self, + extraction_networks_cfg, + extraction_network_fea, + shared_expert_fea, + final_flag, ): layer_name = extraction_networks_cfg.network_name expert_shared_out = self.experts_layer( - shared_expert_fea, extraction_networks_cfg.share_num, - extraction_networks_cfg.share_expert_net, layer_name + '_share/dnn' + shared_expert_fea, + extraction_networks_cfg.share_num, + extraction_networks_cfg.share_expert_net, + layer_name + '_share/dnn', ) experts_outs = [] @@ -81,11 +77,10 @@ def CGC_layer( experts_out = self.experts_layer( extraction_network_fea[task_idx], extraction_networks_cfg.expert_num_per_task, - extraction_networks_cfg.task_expert_net, name - ) - cgc_layer_out = self.gate( - extraction_network_fea[task_idx], experts_out + expert_shared_out, name + extraction_networks_cfg.task_expert_net, + name, ) + cgc_layer_out = self.gate(extraction_network_fea[task_idx], experts_out + expert_shared_out, name) experts_outs.extend(experts_out) cgc_layer_outs.append(cgc_layer_out) @@ -93,8 +88,9 @@ def CGC_layer( shared_layer_out = None else: shared_layer_out = self.gate( - shared_expert_fea, experts_outs + expert_shared_out, - layer_name + '_share' + shared_expert_fea, + experts_outs + expert_shared_out, + layer_name + '_share', ) return cgc_layer_outs, shared_layer_out @@ -107,8 +103,10 @@ def build_predict_graph(self): if idx == len(self._model_config.extraction_networks) - 1: final_flag = True extraction_network_fea, shared_expert_fea = self.CGC_layer( - extraction_network, extraction_network_fea, shared_expert_fea, - final_flag + extraction_network, + extraction_network_fea, + shared_expert_fea, + final_flag, ) tower_outputs = {} @@ -118,7 +116,7 @@ def build_predict_graph(self): task_tower_cfg.dnn, self._l2_reg, name=tower_name, - is_training=self._is_training + is_training=self._is_training, ) tower_output = tower_dnn(extraction_network_fea[i]) @@ -126,7 +124,7 @@ def build_predict_graph(self): inputs=tower_output, units=task_tower_cfg.num_class, kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i + name='dnn_output_%d' % i, ) tower_outputs[tower_name] = tower_output diff --git a/easy_rec/python/model/rank_model.py b/easy_rec/python/model/rank_model.py index a5194ba5b..bab2b9ec9 100644 --- a/easy_rec/python/model/rank_model.py +++ b/easy_rec/python/model/rank_model.py @@ -6,28 +6,19 @@ from tensorflow.python.ops import math_ops from easy_rec.python.builders import loss_builder +from easy_rec.python.loss.zero_inflated_lognormal import ( # NOQA + zero_inflated_lognormal_pred, +) from easy_rec.python.model.easy_rec_model import EasyRecModel from easy_rec.python.protos.loss_pb2 import LossType -from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_pred # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 class RankModel(EasyRecModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(RankModel, self).__init__( - model_config, feature_configs, features, labels, is_training - ) + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(RankModel, self).__init__(model_config, feature_configs, features, labels, is_training) self._loss_type = self._model_config.loss_type self._num_class = self._model_config.num_class self._losses = self._model_config.losses @@ -40,9 +31,7 @@ def __init__( def build_predict_graph(self): if not self.has_backbone: - raise NotImplementedError( - 'method `build_predict_graph` must be implemented when backbone network do not exits' - ) + raise NotImplementedError('method `build_predict_graph` must be implemented when backbone network do not exits') model = self._model_config.WhichOneof('model') assert model == 'model_params', '`model_params` must be configured' config = self._model_config.model_params @@ -56,16 +45,18 @@ def build_predict_graph(self): self._add_to_prediction_dict(output) return self._prediction_dict - def _output_to_prediction_impl( - self, output, loss_type, num_class=1, suffix='', **kwargs - ): + def _output_to_prediction_impl(self, output, loss_type, num_class=1, suffix='', **kwargs): prediction_dict = {} binary_loss_type = { - LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, LossType.BINARY_CROSS_ENTROPY_LOSS, - LossType.LISTWISE_DISTILL_LOSS + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.BINARY_CROSS_ENTROPY_LOSS, + LossType.LISTWISE_DISTILL_LOSS, } if loss_type in binary_loss_type: assert num_class == 1, 'num_class must be 1 when loss type is %s' % loss_type.name @@ -90,7 +81,7 @@ def _output_to_prediction_impl( output, max_sigma=max_sigma, max_log_clip=max_log_clip_val, - return_log=return_log + return_log=return_log, ) tf.summary.scalar('prediction/probs', tf.reduce_mean(probs)) tf.summary.scalar('prediction/y', tf.reduce_mean(preds)) @@ -110,10 +101,8 @@ def _output_to_prediction_impl( prediction_dict['logits' + suffix + '_1'] = output[:, 1] prediction_dict['probs' + suffix] = probs prediction_dict['probs' + suffix + '_1'] = probs[:, 1] - prediction_dict['logits' + suffix + - '_y'] = math_ops.reduce_max(output, axis=1) - prediction_dict['probs' + suffix + - '_y'] = math_ops.reduce_max(probs, axis=1) + prediction_dict['logits' + suffix + '_y'] = math_ops.reduce_max(output, axis=1) + prediction_dict['probs' + suffix + '_y'] = math_ops.reduce_max(probs, axis=1) prediction_dict['y' + suffix] = tf.argmax(output, axis=1) elif loss_type == LossType.L2_LOSS: output = tf.squeeze(output, axis=1) @@ -125,9 +114,7 @@ def _output_to_prediction_impl( def _add_to_prediction_dict(self, output): if len(self._losses) == 0: - prediction_dict = self._output_to_prediction_impl( - output, loss_type=self._loss_type, num_class=self._num_class - ) + prediction_dict = self._output_to_prediction_impl(output, loss_type=self._loss_type, num_class=self._num_class) self._prediction_dict.update(prediction_dict) else: for loss in self._losses: @@ -143,7 +130,7 @@ def _add_to_prediction_dict(self, output): output, loss_type=loss.loss_type, num_class=self._num_class, - **kwargs + **kwargs, ) self._prediction_dict.update(prediction_dict) @@ -156,10 +143,9 @@ def build_rtp_output_dict(self): op = tf.get_default_graph().get_operation_by_name('rank_predict') if len(op.outputs) != 1: raise ValueError( - ( - 'failed to build RTP rank_predict output: op {}[{}] has output ' + - 'size {}, however 1 is expected.' - ).format(op.name, op.type, len(op.outputs)) + ('failed to build RTP rank_predict output: op {}[{}] has output ' + 'size {}, however 1 is expected.').format( + op.name, op.type, len(op.outputs) + ) ) rank_predict = op.outputs[0] except KeyError: @@ -168,38 +154,40 @@ def build_rtp_output_dict(self): if len(self._losses) > 0: loss_types = {loss.loss_type for loss in self._losses} binary_loss_set = { - LossType.CLASSIFICATION, LossType.F1_REWEIGHTED_LOSS, - LossType.PAIR_WISE_LOSS, LossType.BINARY_FOCAL_LOSS, - LossType.PAIRWISE_FOCAL_LOSS, LossType.PAIRWISE_LOGISTIC_LOSS, - LossType.JRC_LOSS, LossType.LISTWISE_DISTILL_LOSS, - LossType.LISTWISE_RANK_LOSS + LossType.CLASSIFICATION, + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.JRC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, + LossType.LISTWISE_RANK_LOSS, } if loss_types & binary_loss_set: if 'probs' in self._prediction_dict: forwarded = self._prediction_dict['probs'] else: raise ValueError( - 'failed to build RTP rank_predict output: classification model ' + - "expect 'probs' prediction, which is not found. Please check if" + - ' build_predict_graph() is called.' + 'failed to build RTP rank_predict output: classification model ' + + "expect 'probs' prediction, which is not found. Please check if" + + ' build_predict_graph() is called.' ) elif loss_types & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: if 'y' in self._prediction_dict: forwarded = self._prediction_dict['y'] else: raise ValueError( 'failed to build RTP rank_predict output: regression model expect' - + - "'y' prediction, which is not found. Please check if build_predic" + + "'y' prediction, which is not found. Please check if build_predic" + 't_graph() is called.' ) else: - logging.warning( - 'failed to build RTP rank_predict: unsupported loss type {}'. - format(loss_types) - ) + logging.warning('failed to build RTP rank_predict: unsupported loss type {}'.format(loss_types)) if forwarded is not None: rank_predict = tf.identity(forwarded, name='rank_predict') if rank_predict is not None: @@ -214,19 +202,22 @@ def _build_loss_impl( num_class=1, suffix='', loss_name='', - loss_param=None + loss_param=None, ): loss_dict = {} binary_loss_type = { - LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, LossType.JRC_LOSS, - LossType.LISTWISE_DISTILL_LOSS, LossType.ZILN_LOSS + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.JRC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, + LossType.ZILN_LOSS, } - if loss_type in { - LossType.CLASSIFICATION, LossType.BINARY_CROSS_ENTROPY_LOSS - }: + if loss_type in {LossType.CLASSIFICATION, LossType.BINARY_CROSS_ENTROPY_LOSS}: loss_name = loss_name if loss_name else 'cross_entropy_loss' + suffix pred = self._prediction_dict['logits' + suffix] elif loss_type in binary_loss_type: @@ -243,7 +234,7 @@ def _build_loss_impl( tf.summary.scalar( 'labels/%s' % label_name, - tf.reduce_mean(tf.to_float(self._labels[label_name])) + tf.reduce_mean(tf.to_float(self._labels[label_name])), ) kwargs = {'loss_name': loss_name} if loss_param is not None: @@ -256,7 +247,7 @@ def _build_loss_impl( loss_weight, num_class, loss_param=loss_param, - **kwargs + **kwargs, ) return loss_dict @@ -268,14 +259,12 @@ def build_loss_graph(self): self._loss_type, label_name=self._label_name, loss_weight=self._sample_weight, - num_class=self._num_class + num_class=self._num_class, ) else: strategy = self._base_model_config.loss_weight_strategy loss_weight = [1.0] - if strategy == self._base_model_config.Random and len( - self._losses - ) > 1: + if strategy == self._base_model_config.Random and len(self._losses) > 1: weights = tf.random_normal([len(self._losses)]) loss_weight = tf.nn.softmax(weights) for i, loss in enumerate(self._losses): @@ -288,7 +277,7 @@ def build_loss_graph(self): loss_weight=self._sample_weight, num_class=self._num_class, loss_name=loss.loss_name, - loss_param=loss_param + loss_param=loss_param, ) for loss_name, loss_value in loss_ops.items(): if strategy == self._base_model_config.Fixed: @@ -296,48 +285,47 @@ def build_loss_graph(self): elif strategy == self._base_model_config.Uncertainty: if loss.learn_loss_weight: uncertainty = tf.Variable( - 0, name='%s_loss_weight' % loss_name, dtype=tf.float32 + 0, + name='%s_loss_weight' % loss_name, + dtype=tf.float32, ) tf.summary.scalar('%s_uncertainty' % loss_name, uncertainty) if loss.loss_type in { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, }: - loss_dict[loss_name] = 0.5 * tf.exp( - -uncertainty - ) * loss_value + 0.5 * uncertainty + loss_dict[loss_name] = 0.5 * tf.exp(-uncertainty) * loss_value + 0.5 * uncertainty else: - loss_dict[ - loss_name - ] = tf.exp(-uncertainty) * loss_value + 0.5 * uncertainty + loss_dict[loss_name] = tf.exp(-uncertainty) * loss_value + 0.5 * uncertainty else: loss_dict[loss_name] = loss_value * loss.weight elif strategy == self._base_model_config.Random: loss_dict[loss_name] = loss_value * loss_weight[i] else: - raise ValueError( - 'Unsupported loss weight strategy: ' + strategy.Name - ) + raise ValueError('Unsupported loss weight strategy: ' + strategy.Name) self._loss_dict.update(loss_dict) # build kd loss - kd_loss_dict = loss_builder.build_kd_loss( - self.kd, self._prediction_dict, self._labels, self._feature_dict - ) + kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, self._labels, self._feature_dict) self._loss_dict.update(kd_loss_dict) return self._loss_dict - def _build_metric_impl( - self, metric, loss_type, label_name, num_class=1, suffix='' - ): + def _build_metric_impl(self, metric, loss_type, label_name, num_class=1, suffix=''): if not isinstance(loss_type, set): loss_type = {loss_type} from easy_rec.python.core import metrics as metrics_lib from easy_rec.python.core.easyrec_metrics import metrics_tf + binary_loss_set = { - LossType.CLASSIFICATION, LossType.F1_REWEIGHTED_LOSS, - LossType.PAIR_WISE_LOSS, LossType.BINARY_FOCAL_LOSS, - LossType.PAIRWISE_FOCAL_LOSS, LossType.PAIRWISE_LOGISTIC_LOSS, - LossType.JRC_LOSS, LossType.LISTWISE_DISTILL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.ZILN_LOSS + LossType.CLASSIFICATION, + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.JRC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.ZILN_LOSS, } metric_dict = {} if metric.WhichOneof('metric') == 'auc': @@ -347,14 +335,14 @@ def _build_metric_impl( metric_dict['auc' + suffix] = metrics_tf.auc( label, self._prediction_dict['probs' + suffix], - num_thresholds=metric.auc.num_thresholds + num_thresholds=metric.auc.num_thresholds, ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) metric_dict['auc' + suffix] = metrics_tf.auc( label, self._prediction_dict['probs' + suffix][:, 1], - num_thresholds=metric.auc.num_thresholds + num_thresholds=metric.auc.num_thresholds, ) else: raise ValueError('Wrong class number') @@ -364,15 +352,13 @@ def _build_metric_impl( label = tf.to_int64(self._labels[label_name]) uids = self._feature_dict[metric.gauc.uid_field] if isinstance(uids, tf.sparse.SparseTensor): - uids = tf.sparse_to_dense( - uids.indices, uids.dense_shape, uids.values, default_value='' - ) + uids = tf.sparse_to_dense(uids.indices, uids.dense_shape, uids.values, default_value='') uids = tf.reshape(uids, [-1]) metric_dict['gauc' + suffix] = metrics_lib.gauc( label, self._prediction_dict['probs' + suffix], uids=uids, - reduction=metric.gauc.reduction + reduction=metric.gauc.reduction, ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) @@ -380,7 +366,7 @@ def _build_metric_impl( label, self._prediction_dict['probs' + suffix][:, 1], uids=self._feature_dict[metric.gauc.uid_field], - reduction=metric.gauc.reduction + reduction=metric.gauc.reduction, ) else: raise ValueError('Wrong class number') @@ -392,7 +378,7 @@ def _build_metric_impl( label, self._prediction_dict['probs' + suffix], session_ids=self._feature_dict[metric.session_auc.session_id_field], - reduction=metric.session_auc.reduction + reduction=metric.session_auc.reduction, ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) @@ -400,7 +386,7 @@ def _build_metric_impl( label, self._prediction_dict['probs' + suffix][:, 1], session_ids=self._feature_dict[metric.session_auc.session_id_field], - reduction=metric.session_auc.reduction + reduction=metric.session_auc.reduction, ) else: raise ValueError('Wrong class number') @@ -408,14 +394,10 @@ def _build_metric_impl( assert loss_type & binary_loss_set if num_class == 1 or loss_type & {LossType.JRC_LOSS, LossType.ZILN_LOSS}: label = tf.to_int64(self._labels[label_name]) - metric_dict['max_f1' + suffix] = metrics_lib.max_f1( - label, self._prediction_dict['logits' + suffix] - ) + metric_dict['max_f1' + suffix] = metrics_lib.max_f1(label, self._prediction_dict['logits' + suffix]) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) - metric_dict['max_f1' + suffix] = metrics_lib.max_f1( - label, self._prediction_dict['logits' + suffix][:, 1] - ) + metric_dict['max_f1' + suffix] = metrics_lib.max_f1(label, self._prediction_dict['logits' + suffix][:, 1]) else: raise ValueError('Wrong class number') elif metric.WhichOneof('metric') == 'recall_at_topk': @@ -423,64 +405,63 @@ def _build_metric_impl( assert num_class > 1 label = tf.to_int64(self._labels[label_name]) metric_dict['recall_at_topk' + suffix] = metrics_tf.recall_at_k( - label, self._prediction_dict['logits' + suffix], - metric.recall_at_topk.topk + label, + self._prediction_dict['logits' + suffix], + metric.recall_at_topk.topk, ) elif metric.WhichOneof('metric') == 'mean_absolute_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: - metric_dict['mean_absolute_error' + - suffix] = metrics_tf.mean_absolute_error( - label, self._prediction_dict['y' + suffix] - ) + metric_dict['mean_absolute_error' + suffix] = metrics_tf.mean_absolute_error( + label, self._prediction_dict['y' + suffix] + ) elif loss_type & {LossType.CLASSIFICATION} and num_class == 1: - metric_dict['mean_absolute_error' + - suffix] = metrics_tf.mean_absolute_error( - label, self._prediction_dict['probs' + suffix] - ) + metric_dict['mean_absolute_error' + suffix] = metrics_tf.mean_absolute_error( + label, self._prediction_dict['probs' + suffix] + ) else: assert False, 'mean_absolute_error is not supported for this model' elif metric.WhichOneof('metric') == 'mean_squared_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: - metric_dict['mean_squared_error' + - suffix] = metrics_tf.mean_squared_error( - label, self._prediction_dict['y' + suffix] - ) + metric_dict['mean_squared_error' + suffix] = metrics_tf.mean_squared_error( + label, self._prediction_dict['y' + suffix] + ) elif num_class == 1 and loss_type & binary_loss_set: - metric_dict['mean_squared_error' + - suffix] = metrics_tf.mean_squared_error( - label, self._prediction_dict['probs' + suffix] - ) + metric_dict['mean_squared_error' + suffix] = metrics_tf.mean_squared_error( + label, self._prediction_dict['probs' + suffix] + ) else: assert False, 'mean_squared_error is not supported for this model' elif metric.WhichOneof('metric') == 'root_mean_squared_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: - metric_dict['root_mean_squared_error' + - suffix] = metrics_tf.root_mean_squared_error( - label, self._prediction_dict['y' + suffix] - ) + metric_dict['root_mean_squared_error' + suffix] = metrics_tf.root_mean_squared_error( + label, self._prediction_dict['y' + suffix] + ) elif loss_type & {LossType.CLASSIFICATION} and num_class == 1: - metric_dict['root_mean_squared_error' + - suffix] = metrics_tf.root_mean_squared_error( - label, self._prediction_dict['probs' + suffix] - ) + metric_dict['root_mean_squared_error' + suffix] = metrics_tf.root_mean_squared_error( + label, self._prediction_dict['probs' + suffix] + ) else: assert False, 'root_mean_squared_error is not supported for this model' elif metric.WhichOneof('metric') == 'accuracy': assert loss_type & {LossType.CLASSIFICATION} assert num_class > 1 label = tf.to_int64(self._labels[label_name]) - metric_dict['accuracy' + suffix] = metrics_tf.accuracy( - label, self._prediction_dict['y' + suffix] - ) + metric_dict['accuracy' + suffix] = metrics_tf.accuracy(label, self._prediction_dict['y' + suffix]) return metric_dict def build_metric_graph(self, eval_config): @@ -493,17 +474,21 @@ def build_metric_graph(self, eval_config): metric, loss_type=loss_types, label_name=self._label_name, - num_class=self._num_class + num_class=self._num_class, ) ) return self._metric_dict def _get_outputs_impl(self, loss_type, num_class=1, suffix=''): binary_loss_set = { - LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, LossType.LISTWISE_DISTILL_LOSS + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, } if loss_type in binary_loss_set: return ['probs' + suffix, 'logits' + suffix] @@ -516,9 +501,13 @@ def _get_outputs_impl(self, loss_type, num_class=1, suffix=''): return ['probs' + suffix, 'logits' + suffix] else: return [ - 'y' + suffix, 'probs' + suffix, 'logits' + suffix, - 'probs' + suffix + '_y', 'logits' + suffix + '_y', - 'probs' + suffix + '_1', 'logits' + suffix + '_1' + 'y' + suffix, + 'probs' + suffix, + 'logits' + suffix, + 'probs' + suffix + '_y', + 'logits' + suffix + '_y', + 'probs' + suffix + '_1', + 'logits' + suffix + '_1', ] elif loss_type in [LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS]: return ['y' + suffix] diff --git a/easy_rec/python/model/rocket_launching.py b/easy_rec/python/model/rocket_launching.py index 62f7cff46..f1bd18942 100755 --- a/easy_rec/python/model/rocket_launching.py +++ b/easy_rec/python/model/rocket_launching.py @@ -6,29 +6,19 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel from easy_rec.python.protos.loss_pb2 import LossType -from easy_rec.python.protos.simi_pb2 import Similarity - from easy_rec.python.protos.rocket_launching_pb2 import RocketLaunching as RocketLaunchingConfig # NOQA +from easy_rec.python.protos.simi_pb2 import Similarity if tf.__version__ >= '2.0': tf = tf.compat.v1 class RocketLaunching(RankModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(RocketLaunching, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(RocketLaunching, self).__init__(model_config, feature_configs, features, labels, is_training) + assert self._model_config.WhichOneof('model') == 'rocket_launching', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model') ) - assert self._model_config.WhichOneof('model') == 'rocket_launching', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.rocket_launching assert isinstance(self._model_config, RocketLaunchingConfig) if self._labels is not None: @@ -37,9 +27,7 @@ def __init__( self._features, _ = self._input_layer(self._feature_dict, 'all') def sim(self, feature_emb1, feature_emb2): - emb1_emb2_sim = tf.reduce_sum( - tf.multiply(feature_emb1, feature_emb2), axis=1, keepdims=True - ) + emb1_emb2_sim = tf.reduce_sum(tf.multiply(feature_emb1, feature_emb2), axis=1, keepdims=True) return emb1_emb2_sim def norm(self, fea): @@ -47,88 +35,70 @@ def norm(self, fea): return fea_norm def feature_based_sim(self, feature_based_distillation, i, j): - booster_feature_no_gradient = tf.stop_gradient( - self.booster_feature['hidden_layer' + str(j)] - ) + booster_feature_no_gradient = tf.stop_gradient(self.booster_feature['hidden_layer' + str(j)]) if feature_based_distillation == Similarity.COSINE: booster_feature_no_gradient_norm = self.norm(booster_feature_no_gradient) - light_feature_norm = self.norm( - self.light_feature['hidden_layer' + str(i)] - ) - sim_middle_layer = tf.reduce_mean( - self.sim(booster_feature_no_gradient_norm, light_feature_norm) - ) + light_feature_norm = self.norm(self.light_feature['hidden_layer' + str(i)]) + sim_middle_layer = tf.reduce_mean(self.sim(booster_feature_no_gradient_norm, light_feature_norm)) return sim_middle_layer else: return tf.sqrt( - tf.reduce_sum( - tf.square( - booster_feature_no_gradient - - self.light_feature['hidden_layer' + str(i)] - ) - ) + tf.reduce_sum(tf.square(booster_feature_no_gradient - self.light_feature['hidden_layer' + str(i)])) ) def build_predict_graph(self): self.hidden_layer_feature_output = self._model_config.feature_based_distillation if self._model_config.HasField('share_dnn'): share_dnn_layer = dnn.DNN( - self._model_config.share_dnn, self._l2_reg, 'share_dnn', - self._is_training + self._model_config.share_dnn, + self._l2_reg, + 'share_dnn', + self._is_training, ) share_feature = share_dnn_layer(self._features) booster_dnn_layer = dnn.DNN( - self._model_config.booster_dnn, self._l2_reg, 'booster_dnn', - self._is_training - ) - light_dnn_layer = dnn.DNN( - self._model_config.light_dnn, self._l2_reg, 'light_dnn', - self._is_training + self._model_config.booster_dnn, + self._l2_reg, + 'booster_dnn', + self._is_training, ) + light_dnn_layer = dnn.DNN(self._model_config.light_dnn, self._l2_reg, 'light_dnn', self._is_training) if self._model_config.HasField('share_dnn'): - self.booster_feature = booster_dnn_layer( - share_feature, self.hidden_layer_feature_output - ) + self.booster_feature = booster_dnn_layer(share_feature, self.hidden_layer_feature_output) input_embedding_stop_gradient = tf.stop_gradient(share_feature) - self.light_feature = light_dnn_layer( - input_embedding_stop_gradient, self.hidden_layer_feature_output - ) + self.light_feature = light_dnn_layer(input_embedding_stop_gradient, self.hidden_layer_feature_output) else: - self.booster_feature = booster_dnn_layer( - self._features, self.hidden_layer_feature_output - ) + self.booster_feature = booster_dnn_layer(self._features, self.hidden_layer_feature_output) input_embedding_stop_gradient = tf.stop_gradient(self._features) - self.light_feature = light_dnn_layer( - input_embedding_stop_gradient, self.hidden_layer_feature_output - ) + self.light_feature = light_dnn_layer(input_embedding_stop_gradient, self.hidden_layer_feature_output) if self._model_config.feature_based_distillation: booster_out = tf.layers.dense( self.booster_feature['hidden_layer_end'], self._num_class, kernel_regularizer=self._l2_reg, - name='booster_output' + name='booster_output', ) light_out = tf.layers.dense( self.light_feature['hidden_layer_end'], self._num_class, kernel_regularizer=self._l2_reg, - name='light_output' + name='light_output', ) else: booster_out = tf.layers.dense( self.booster_feature, self._num_class, kernel_regularizer=self._l2_reg, - name='booster_output' + name='booster_output', ) light_out = tf.layers.dense( self.light_feature, self._num_class, kernel_regularizer=self._l2_reg, - name='light_output' + name='light_output', ) self._prediction_dict.update( @@ -136,13 +106,11 @@ def build_predict_graph(self): booster_out, self._loss_type, num_class=self._num_class, - suffix='_booster' + suffix='_booster', ) ) self._prediction_dict.update( - self._output_to_prediction_impl( - light_out, self._loss_type, num_class=self._num_class, suffix='_light' - ) + self._output_to_prediction_impl(light_out, self._loss_type, num_class=self._num_class, suffix='_light') ) return self._prediction_dict @@ -161,10 +129,9 @@ def build_loss_graph(self): for i, unit_i in enumerate(light_hidden_units): for j, unit_j in enumerate(booster_hidden_units): if light_hidden_units[i] == booster_hidden_units[j]: - self._prediction_dict[ - 'similarity_' + str(count)] = self.feature_based_sim( - self._model_config.feature_based_distillation, i, j - ) + self._prediction_dict['similarity_' + str(count)] = self.feature_based_sim( + self._model_config.feature_based_distillation, i, j + ) count += 1 break @@ -174,7 +141,7 @@ def build_loss_graph(self): label_name=self._label_name, loss_weight=self._sample_weight, num_class=self._num_class, - suffix='_booster' + suffix='_booster', ) ) @@ -184,7 +151,7 @@ def build_loss_graph(self): label_name=self._label_name, loss_weight=self._sample_weight, num_class=self._num_class, - suffix='_light' + suffix='_light', ) ) @@ -194,7 +161,7 @@ def build_loss_graph(self): LossType.L2_LOSS, label=booster_logits_no_grad, pred=logits_light, - loss_weight=self._sample_weight + loss_weight=self._sample_weight, ) if self._model_config.feature_based_distillation: @@ -214,7 +181,7 @@ def build_metric_graph(self, eval_config): loss_type=LossType.CLASSIFICATION, label_name=self._label_name, num_class=self._num_class, - suffix='_light' + suffix='_light', ) ) metric_dict.update( @@ -223,21 +190,13 @@ def build_metric_graph(self, eval_config): loss_type=LossType.CLASSIFICATION, label_name=self._label_name, num_class=self._num_class, - suffix='_booster' + suffix='_booster', ) ) return metric_dict def get_outputs(self): outputs = [] - outputs.extend( - self._get_outputs_impl( - self._loss_type, self._num_class, suffix='_light' - ) - ) - outputs.extend( - self._get_outputs_impl( - self._loss_type, self._num_class, suffix='_booster' - ) - ) + outputs.extend(self._get_outputs_impl(self._loss_type, self._num_class, suffix='_light')) + outputs.extend(self._get_outputs_impl(self._loss_type, self._num_class, suffix='_booster')) return outputs diff --git a/easy_rec/python/model/simple_multi_task.py b/easy_rec/python/model/simple_multi_task.py index a02d294f4..00527b47a 100644 --- a/easy_rec/python/model/simple_multi_task.py +++ b/easy_rec/python/model/simple_multi_task.py @@ -4,7 +4,6 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.multi_task_model import MultiTaskModel - from easy_rec.python.protos.simple_multi_task_pb2 import SimpleMultiTask as SimpleMultiTaskConfig # NOQA if tf.__version__ >= '2.0': @@ -12,21 +11,12 @@ class SimpleMultiTask(MultiTaskModel): + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(SimpleMultiTask, self).__init__(model_config, feature_configs, features, labels, is_training) - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(SimpleMultiTask, self).__init__( - model_config, feature_configs, features, labels, is_training + assert self._model_config.WhichOneof('model') == 'simple_multi_task', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model') ) - - assert self._model_config.WhichOneof('model') == 'simple_multi_task', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.simple_multi_task assert isinstance(self._model_config, SimpleMultiTaskConfig) @@ -44,14 +34,14 @@ def build_predict_graph(self): task_tower_cfg.dnn, self._l2_reg, name=tower_name, - is_training=self._is_training + is_training=self._is_training, ) task_fea = task_dnn(self._features) task_output = tf.layers.dense( inputs=task_fea, units=task_tower_cfg.num_class, kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i + name='dnn_output_%d' % i, ) tower_outputs[tower_name] = task_output diff --git a/easy_rec/python/model/uniter.py b/easy_rec/python/model/uniter.py index a38179e22..3b8173979 100644 --- a/easy_rec/python/model/uniter.py +++ b/easy_rec/python/model/uniter.py @@ -4,7 +4,6 @@ from easy_rec.python.layers import dnn, uniter from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.uniter_pb2 import Uniter as UNITERConfig # NOQA if tf.__version__ >= '2.0': @@ -18,33 +17,24 @@ class Uniter(RankModel): https://arxiv.org/abs/1909.11740 """ - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(Uniter, self).__init__( - model_config, feature_configs, features, labels, is_training - ) + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(Uniter, self).__init__(model_config, feature_configs, features, labels, is_training) assert self._model_config.WhichOneof('model') == 'uniter', ( 'invalid model config: %s' % self._model_config.WhichOneof('model') ) self._uniter_layer = uniter.Uniter( - model_config, feature_configs, features, - self._model_config.uniter.config, self._input_layer + model_config, + feature_configs, + features, + self._model_config.uniter.config, + self._input_layer, ) self._model_config = self._model_config.uniter def build_predict_graph(self): hidden = self._uniter_layer(self._is_training, l2_reg=self._l2_reg) - final_dnn_layer = dnn.DNN( - self._model_config.final_dnn, self._l2_reg, 'final_dnn', - self._is_training - ) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) all_fea = final_dnn_layer(hidden) final = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/wide_and_deep.py b/easy_rec/python/model/wide_and_deep.py index 40f8d1e00..cc24a60c0 100755 --- a/easy_rec/python/model/wide_and_deep.py +++ b/easy_rec/python/model/wide_and_deep.py @@ -6,7 +6,6 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel - from easy_rec.python.protos.wide_and_deep_pb2 import WideAndDeep as WideAndDeepConfig # NOQA if tf.__version__ >= '2.0': @@ -14,20 +13,11 @@ class WideAndDeep(RankModel): - - def __init__( - self, - model_config, - feature_configs, - features, - labels=None, - is_training=False - ): - super(WideAndDeep, self).__init__( - model_config, feature_configs, features, labels, is_training + def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + super(WideAndDeep, self).__init__(model_config, feature_configs, features, labels, is_training) + assert model_config.WhichOneof('model') == 'wide_and_deep', 'invalid model config: %s' % model_config.WhichOneof( + 'model' ) - assert model_config.WhichOneof('model') == 'wide_and_deep', \ - 'invalid model config: %s' % model_config.WhichOneof('model') self._model_config = model_config.wide_and_deep assert isinstance(self._model_config, WideAndDeepConfig) assert self._input_layer.has_group('wide') @@ -49,39 +39,30 @@ def build_predict_graph(self): logging.info('wide features dimension: %d' % wide_fea.get_shape()[-1]) self._deep_features = tf.concat(self._deep_features, axis=1) - logging.info( - 'input deep features dimension: %d' % self._deep_features.get_shape()[-1] - ) + logging.info('input deep features dimension: %d' % self._deep_features.get_shape()[-1]) - deep_layer = dnn.DNN( - self._model_config.dnn, self._l2_reg, 'deep_feature', self._is_training - ) + deep_layer = dnn.DNN(self._model_config.dnn, self._l2_reg, 'deep_feature', self._is_training) deep_fea = deep_layer(self._deep_features) - logging.info( - 'output deep features dimension: %d' % deep_fea.get_shape()[-1] - ) + logging.info('output deep features dimension: %d' % deep_fea.get_shape()[-1]) has_final = len(self._model_config.final_dnn.hidden_units) > 0 print('wide_deep has_final_dnn layers = %d' % has_final) if has_final: all_fea = tf.concat([wide_fea, deep_fea], axis=1) final_layer = dnn.DNN( - self._model_config.final_dnn, self._l2_reg, 'final_dnn', - self._is_training + self._model_config.final_dnn, + self._l2_reg, + 'final_dnn', + self._is_training, ) all_fea = final_layer(all_fea) - output = tf.layers.dense( - all_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='output' - ) + output = tf.layers.dense(all_fea, self._num_class, kernel_regularizer=self._l2_reg, name='output') else: deep_out = tf.layers.dense( deep_fea, self._num_class, kernel_regularizer=self._l2_reg, - name='deep_out' + name='deep_out', ) output = deep_out + wide_fea @@ -100,17 +81,18 @@ def get_grouped_vars(self, opt_num): Return: list of list of variables. """ - assert opt_num <= 3, 'could only support 2 or 3 optimizers, ' + \ - 'if opt_num = 2, one for the wide , and one for the others, ' + \ - 'if opt_num = 3, one for the wide, second for the deep embeddings, ' + \ - 'and third for the other layers.' + assert opt_num <= 3, ( + 'could only support 2 or 3 optimizers, ' + + 'if opt_num = 2, one for the wide , and one for the others, ' + + 'if opt_num = 3, one for the wide, second for the deep embeddings, ' + + 'and third for the other layers.' + ) if opt_num == 2: wide_vars = [] deep_vars = [] for tmp_var in tf.trainable_variables(): - if tmp_var.name.startswith('input_layer') and \ - (not tmp_var.name.startswith('input_layer_1')): + if tmp_var.name.startswith('input_layer') and (not tmp_var.name.startswith('input_layer_1')): wide_vars.append(tmp_var) else: deep_vars.append(tmp_var) @@ -120,12 +102,9 @@ def get_grouped_vars(self, opt_num): embedding_vars = [] deep_vars = [] for tmp_var in tf.trainable_variables(): - if tmp_var.name.startswith('input_layer') and \ - (not tmp_var.name.startswith('input_layer_1')): + if tmp_var.name.startswith('input_layer') and (not tmp_var.name.startswith('input_layer_1')): wide_vars.append(tmp_var) - elif tmp_var.name.startswith( - 'input_layer' - ) or '/embedding_weights' in tmp_var.name: + elif tmp_var.name.startswith('input_layer') or '/embedding_weights' in tmp_var.name: embedding_vars.append(tmp_var) else: deep_vars.append(tmp_var) diff --git a/easy_rec/python/ops/gen_kafka_ops.py b/easy_rec/python/ops/gen_kafka_ops.py index bf2618e5a..a6bb716fd 100644 --- a/easy_rec/python/ops/gen_kafka_ops.py +++ b/easy_rec/python/ops/gen_kafka_ops.py @@ -14,6 +14,7 @@ from tensorflow.python.eager import context as _context from tensorflow.python.eager import core as _core from tensorflow.python.eager import execute as _execute + # Needed to trigger the call to _set_call_cpp_shape_fn. from tensorflow.python.framework import dtypes as _dtypes from tensorflow.python.framework import ops as _ops @@ -28,9 +29,7 @@ try: kafka_module = tf.load_op_library(kafka_ops_path) except Exception: - logging.warning( - 'load %s failed: %s' % (kafka_ops_path, traceback.format_exc()) - ) + logging.warning('load %s failed: %s' % (kafka_ops_path, traceback.format_exc())) @tf_export('io_kafka_dataset_v2') @@ -44,7 +43,7 @@ def io_kafka_dataset_v2( config_topic, message_key, message_offset, - name=None + name=None, ): """Creates a dataset that emits the messages of one or more Kafka topics. @@ -85,7 +84,7 @@ def io_kafka_dataset_v2( config_topic=config_topic, message_key=message_key, message_offset=message_offset, - name=name + name=name, ) @@ -100,7 +99,7 @@ def io_kafka_dataset_eager_fallback( message_key, message_offset, name=None, - ctx=None + ctx=None, ): """This is the slowpath function for Eager mode. @@ -117,22 +116,20 @@ def io_kafka_dataset_eager_fallback( message_key = _ops.convert_to_tensor(message_key, _dtypes.bool) message_offset = _ops.convert_to_tensor(message_offset, _dtypes.bool) _inputs_flat = [ - topics, servers, group, eof, timeout, config_global, config_topic, - message_key, message_offset + topics, + servers, + group, + eof, + timeout, + config_global, + config_topic, + message_key, + message_offset, ] _attrs = None - _result = _execute.execute( - b'IOKafkaDataset', - 1, - inputs=_inputs_flat, - attrs=_attrs, - ctx=_ctx, - name=name - ) - _execute.record_gradient( - 'IOKafkaDataset', _inputs_flat, _attrs, _result, name - ) - _result, = _result + _result = _execute.execute(b'IOKafkaDataset', 1, inputs=_inputs_flat, attrs=_attrs, ctx=_ctx, name=name) + _execute.record_gradient('IOKafkaDataset', _inputs_flat, _attrs, _result, name) + (_result,) = _result return _result @@ -151,29 +148,29 @@ def io_write_kafka_v2(message, topic, servers, name=None): """ _ctx = _context._context if _ctx is None or not _ctx._eager_context.is_eager: - _op = kafka_module.io_write_kafka_v2( - message=message, topic=topic, servers=servers, name=name - ) + _op = kafka_module.io_write_kafka_v2(message=message, topic=topic, servers=servers, name=name) _result = _op.outputs[:] _inputs_flat = _op.inputs _attrs = None - _execute.record_gradient( - 'IOWriteKafka', _inputs_flat, _attrs, _result, name - ) - _result, = _result + _execute.record_gradient('IOWriteKafka', _inputs_flat, _attrs, _result, name) + (_result,) = _result return _result else: try: _result = _pywrap_tensorflow.TFE_Py_FastPathExecute( - _ctx._context_handle, _ctx._eager_context.device_name, 'IOWriteKafka', - name, _ctx._post_execution_callbacks, message, topic, servers + _ctx._context_handle, + _ctx._eager_context.device_name, + 'IOWriteKafka', + name, + _ctx._post_execution_callbacks, + message, + topic, + servers, ) return _result except _core._FallbackException: - return io_write_kafka_eager_fallback( - message, topic, servers, name=name, ctx=_ctx - ) + return io_write_kafka_eager_fallback(message, topic, servers, name=name, ctx=_ctx) except _core._NotOkStatusException as e: if name is not None: message = e.message + ' name: ' + name @@ -182,9 +179,7 @@ def io_write_kafka_v2(message, topic, servers, name=None): _six.raise_from(_core._status_to_exception(e.code, message), None) -def io_write_kafka_eager_fallback( - message, topic, servers, name=None, ctx=None -): +def io_write_kafka_eager_fallback(message, topic, servers, name=None, ctx=None): """This is the slowpath function for Eager mode. This is for function io_write_kafka @@ -195,9 +190,7 @@ def io_write_kafka_eager_fallback( servers = _ops.convert_to_tensor(servers, _dtypes.string) _inputs_flat = [message, topic, servers] _attrs = None - _result = _execute.execute( - b'IOWriteKafka', 1, inputs=_inputs_flat, attrs=_attrs, ctx=_ctx, name=name - ) + _result = _execute.execute(b'IOWriteKafka', 1, inputs=_inputs_flat, attrs=_attrs, ctx=_ctx, name=name) _execute.record_gradient('IOWriteKafka', _inputs_flat, _attrs, _result, name) - _result, = _result + (_result,) = _result return _result diff --git a/easy_rec/python/ops/gen_str_avx_op.py b/easy_rec/python/ops/gen_str_avx_op.py index 2710bae02..f46d8df81 100644 --- a/easy_rec/python/ops/gen_str_avx_op.py +++ b/easy_rec/python/ops/gen_str_avx_op.py @@ -20,11 +20,10 @@ def str_split_by_chr(input_str, sep, skip_empty): if constant.has_avx_str_split() and str_avx_op is not None: - assert len(sep) == 1, \ - 'invalid data_config.separator(%s) len(%d) != 1' % ( - sep, len(sep)) - return str_avx_op.avx512_string_split( - input_str, sep, skip_empty=skip_empty + assert len(sep) == 1, 'invalid data_config.separator(%s) len(%d) != 1' % ( + sep, + len(sep), ) + return str_avx_op.avx512_string_split(input_str, sep, skip_empty=skip_empty) else: return string_ops.string_split(input_str, sep, skip_empty=skip_empty) diff --git a/easy_rec/python/ops/incr_record.py b/easy_rec/python/ops/incr_record.py index dd62a0a06..18688df86 100644 --- a/easy_rec/python/ops/incr_record.py +++ b/easy_rec/python/ops/incr_record.py @@ -20,13 +20,9 @@ get_sparse_indices = None set_sparse_indices = None kv_resource_incr_gather = None - logging.warning( - 'failed to import gen_io_ops.collect_sparse_indices: %s' % str(ex) - ) + logging.warning('failed to import gen_io_ops.collect_sparse_indices: %s' % str(ex)) except Exception as ex: get_sparse_indices = None set_sparse_indices = None kv_resource_incr_gather = None - logging.warning( - 'failed to import gen_io_ops.collect_sparse_indices: %s' % str(ex) - ) + logging.warning('failed to import gen_io_ops.collect_sparse_indices: %s' % str(ex)) diff --git a/easy_rec/python/predict.py b/easy_rec/python/predict.py index ae60e8c99..34ec7435b 100644 --- a/easy_rec/python/predict.py +++ b/easy_rec/python/predict.py @@ -9,6 +9,9 @@ from tensorflow.python.lib.io import file_io from easy_rec.python.inference.csv_predictor import CSVPredictor +from easy_rec.python.inference.hive_parquet_predictor import ( # NOQA + HiveParquetPredictor, +) from easy_rec.python.inference.hive_predictor import HivePredictor from easy_rec.python.inference.parquet_predictor import ParquetPredictor from easy_rec.python.inference.parquet_predictor_v2 import ParquetPredictorV2 @@ -17,47 +20,40 @@ from easy_rec.python.utils import config_util, numpy_utils from easy_rec.python.utils.hive_utils import HiveUtils -from easy_rec.python.inference.hive_parquet_predictor import HiveParquetPredictor # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO + level=logging.INFO, ) tf.app.flags.DEFINE_string('input_path', None, 'predict data path') tf.app.flags.DEFINE_string('output_path', None, 'path to save predict result') tf.app.flags.DEFINE_integer('batch_size', 1024, help='batch size') -tf.app.flags.DEFINE_bool( - 'with_header', False, 'whether the input csv file has header' -) +tf.app.flags.DEFINE_bool('with_header', False, 'whether the input csv file has header') # predict by checkpoint +tf.app.flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') tf.app.flags.DEFINE_string( - 'pipeline_config_path', None, 'Path to pipeline config ' - 'file.' -) -tf.app.flags.DEFINE_string( - 'checkpoint_path', None, 'checkpoint to be evaled ' - ' if not specified, use the latest checkpoint in ' - 'train_config.model_dir' + 'checkpoint_path', + None, + 'checkpoint to be evaled ' ' if not specified, use the latest checkpoint in ' 'train_config.model_dir', ) tf.app.flags.DEFINE_string('model_dir', None, help='will update the model_dir') # predict by saved_model tf.app.flags.DEFINE_string('saved_model_dir', None, help='save model dir') tf.app.flags.DEFINE_string( - 'reserved_cols', 'ALL_COLUMNS', - 'columns to keep from input table, they are separated with ,' -) -tf.app.flags.DEFINE_string( - 'output_cols', 'ALL_COLUMNS', - 'output columns, such as: score float. multiple columns are separated by ,' + 'reserved_cols', + 'ALL_COLUMNS', + 'columns to keep from input table, they are separated with ,', ) tf.app.flags.DEFINE_string( - 'output_sep', chr(1), 'separator of predict result file' + 'output_cols', + 'ALL_COLUMNS', + 'output columns, such as: score float. multiple columns are separated by ,', ) +tf.app.flags.DEFINE_string('output_sep', chr(1), 'separator of predict result file') tf.app.flags.DEFINE_string('selected_cols', None, '') tf.app.flags.DEFINE_string('fg_json_path', '', '') tf.app.flags.DEFINE_string('ds_vector_recall', '', '') @@ -75,24 +71,19 @@ def get_input_type(input_type, data_config): def main(argv): - if FLAGS.saved_model_dir: logging.info('Predict by saved_model.') if FLAGS.pipeline_config_path: pipeline_config_path = FLAGS.pipeline_config_path else: - pipeline_config_path = config_util.search_pipeline_config( - FLAGS.saved_model_dir - ) - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False - ) + pipeline_config_path = config_util.search_pipeline_config(FLAGS.saved_model_dir) + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) data_config = pipeline_config.data_config input_type = get_input_type(FLAGS.input_type, data_config) if input_type in [data_config.HiveParquetInput, data_config.HiveInput]: all_cols, all_col_types = HiveUtils( data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input + hive_config=pipeline_config.hive_train_input, ).get_all_cols(FLAGS.input_path) if input_type == DatasetConfig.HiveParquetInput: predictor = HiveParquetPredictor( @@ -102,7 +93,7 @@ def main(argv): hive_config=pipeline_config.hive_train_input, output_sep=FLAGS.output_sep, all_cols=all_cols, - all_col_types=all_col_types + all_col_types=all_col_types, ) else: predictor = HivePredictor( @@ -112,7 +103,7 @@ def main(argv): hive_config=pipeline_config.hive_train_input, output_sep=FLAGS.output_sep, all_cols=all_cols, - all_col_types=all_col_types + all_col_types=all_col_types, ) elif input_type in [data_config.ParquetInput, data_config.ParquetInputV2]: predictor_cls = ParquetPredictor @@ -125,7 +116,7 @@ def main(argv): fg_json_path=FLAGS.fg_json_path, selected_cols=FLAGS.selected_cols, output_sep=FLAGS.output_sep, - pipeline_config=pipeline_config + pipeline_config=pipeline_config, ) elif input_type == data_config.CSVInput: predictor = CSVPredictor( @@ -135,15 +126,12 @@ def main(argv): ds_vector_recall=FLAGS.ds_vector_recall, fg_json_path=FLAGS.fg_json_path, selected_cols=FLAGS.selected_cols, - output_sep=FLAGS.output_sep + output_sep=FLAGS.output_sep, ) else: assert False, 'invalid input type: %s' % input_class_map_r[input_type] - logging.info( - 'input_path = %s, output_path = %s' % - (FLAGS.input_path, FLAGS.output_path) - ) + logging.info('input_path = %s, output_path = %s' % (FLAGS.input_path, FLAGS.output_path)) if 'TF_CONFIG' in os.environ: tf_config = json.loads(os.environ['TF_CONFIG']) worker_num = len(tf_config['cluster']['worker']) @@ -158,7 +146,7 @@ def main(argv): output_cols=FLAGS.output_cols, batch_size=FLAGS.batch_size, slice_id=task_index, - slice_num=worker_num + slice_num=worker_num, ) else: logging.info('Predict by checkpoint_path.') @@ -166,17 +154,13 @@ def main(argv): if FLAGS.model_dir: pipeline_config_path = os.path.join(FLAGS.model_dir, 'pipeline.config') if file_io.file_exists(pipeline_config_path): - logging.info( - 'update pipeline_config_path to %s' % pipeline_config_path - ) + logging.info('update pipeline_config_path to %s' % pipeline_config_path) else: pipeline_config_path = FLAGS.pipeline_config_path else: pipeline_config_path = FLAGS.pipeline_config_path - pred_result = predict( - pipeline_config_path, FLAGS.checkpoint_path, FLAGS.input_path - ) + pred_result = predict(pipeline_config_path, FLAGS.checkpoint_path, FLAGS.input_path) if FLAGS.output_path is not None: logging.info('will save predict result to %s' % FLAGS.output_path) with tf.gfile.GFile(FLAGS.output_path, 'wb') as fout: diff --git a/easy_rec/python/test/csv_input_test.py b/easy_rec/python/test/csv_input_test.py index fc57b7331..90f615ac0 100644 --- a/easy_rec/python/test/csv_input_test.py +++ b/easy_rec/python/test/csv_input_test.py @@ -23,7 +23,6 @@ class CSVInputTest(tf.test.TestCase): - def __init__(self, methodName='CSVInputTest'): super(CSVInputTest, self).__init__(methodName=methodName) self._input_path = 'data/test/test.csv' @@ -71,9 +70,7 @@ def test_csv_data(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput( - dataset_config, feature_configs, self._input_path - ).create_input() + train_input_fn = CSVInput(dataset_config, feature_configs, self._input_path).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -83,7 +80,7 @@ def test_csv_data(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=False + log_device_placement=False, ) with self.test_session(config=session_config) as sess: sess.run(init_op) @@ -136,9 +133,7 @@ def test_csv_data_flt_to_str_exception(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput( - dataset_config, feature_configs, self._input_path - ).create_input() + train_input_fn = CSVInput(dataset_config, feature_configs, self._input_path).create_input() try: dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) # noqa: F841 passed = True @@ -194,9 +189,7 @@ def test_csv_data_flt_to_str(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput( - dataset_config, feature_configs, self._input_path - ).create_input() + train_input_fn = CSVInput(dataset_config, feature_configs, self._input_path).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) @@ -208,7 +201,7 @@ def test_csv_data_flt_to_str(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=False + log_device_placement=False, ) with self.test_session(config=session_config) as sess: sess.run(init_op) @@ -256,9 +249,7 @@ def test_csv_input_ex(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInputEx( - dataset_config, feature_configs, self._input_path_with_quote - ).create_input() + train_input_fn = CSVInputEx(dataset_config, feature_configs, self._input_path_with_quote).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -268,7 +259,7 @@ def test_csv_input_ex(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=False + log_device_placement=False, ) with self.test_session(config=session_config) as sess: sess.run(init_op) @@ -276,7 +267,7 @@ def test_csv_input_ex(self): @unittest.skipIf( 'AVX_TEST' not in os.environ, - 'Only execute when avx512 instructions are supported' + 'Only execute when avx512 instructions are supported', ) @RunAsSubprocess def test_csv_input_ex_avx(self): @@ -327,9 +318,7 @@ def test_csv_data_ignore_error(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput( - dataset_config, feature_configs, self._input_path_with_quote - ).create_input() + train_input_fn = CSVInput(dataset_config, feature_configs, self._input_path_with_quote).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -339,7 +328,7 @@ def test_csv_data_ignore_error(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=False + log_device_placement=False, ) with self.test_session(config=session_config) as sess: sess.run(init_op) diff --git a/easy_rec/python/test/dh_local_run.py b/easy_rec/python/test/dh_local_run.py index 14ead43d6..497b34c24 100644 --- a/easy_rec/python/test/dh_local_run.py +++ b/easy_rec/python/test/dh_local_run.py @@ -8,13 +8,14 @@ from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare +from easy_rec.python.test.odps_test_util import ( # NOQA + OdpsOSSConfig, + delete_oss_path, + get_oss_bucket, +) from easy_rec.python.utils import test_utils -from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA - -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') odps_oss_config = OdpsOSSConfig(script_path='./samples/dh_script') @@ -37,8 +38,9 @@ def test_datahub_train_eval(self): odps_cmd = OdpsCommand(odps_oss_config) self._success = test_utils.test_datahub_train_eval( - '%s/configs/deepfm.config' % odps_oss_config.temp_dir, odps_oss_config, - self._test_dir + '%s/configs/deepfm.config' % odps_oss_config.temp_dir, + odps_oss_config, + self._test_dir, ) odps_cmd.run_list(end) self.assertTrue(self._success) @@ -46,31 +48,14 @@ def test_datahub_train_eval(self): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( - '--odps_config', type=str, default=None, help='odps config path' - ) - parser.add_argument( - '--oss_config', type=str, default=None, help='ossutilconfig path' - ) - parser.add_argument( - '--bucket_name', type=str, default=None, help='test oss bucket name' - ) + parser.add_argument('--odps_config', type=str, default=None, help='odps config path') + parser.add_argument('--oss_config', type=str, default=None, help='ossutilconfig path') + parser.add_argument('--bucket_name', type=str, default=None, help='test oss bucket name') parser.add_argument('--arn', type=str, default=None, help='oss rolearn') - parser.add_argument( - '--odpscmd', type=str, default='odpscmd', help='odpscmd path' - ) - parser.add_argument( - '--algo_project', type=str, default=None, help='algo project name' - ) - parser.add_argument( - '--algo_res_project', - type=str, - default=None, - help='algo resource project name' - ) - parser.add_argument( - '--algo_version', type=str, default=None, help='algo version' - ) + parser.add_argument('--odpscmd', type=str, default='odpscmd', help='odpscmd path') + parser.add_argument('--algo_project', type=str, default=None, help='algo project name') + parser.add_argument('--algo_res_project', type=str, default=None, help='algo resource project name') + parser.add_argument('--algo_version', type=str, default=None, help='algo version') args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] @@ -97,7 +82,7 @@ def test_datahub_train_eval(self): prepare(odps_oss_config) start = [ 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql' + 'deep_fm/create_inner_deepfm_table.sql', ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) @@ -106,8 +91,10 @@ def test_datahub_train_eval(self): tf.test.main() # delete oss path bucket = get_oss_bucket( - odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp diff --git a/easy_rec/python/test/embed_test.py b/easy_rec/python/test/embed_test.py index 61b429bbc..8ef378d67 100644 --- a/easy_rec/python/test/embed_test.py +++ b/easy_rec/python/test/embed_test.py @@ -18,7 +18,6 @@ class EmbedTest(tf.test.TestCase): - def test_raw_embed(self): # embedding variable is: # [[1, 2 ], @@ -27,7 +26,7 @@ def test_raw_embed(self): # [7, 8 ], # [9, 10] # ] - feature_config_str = ''' + feature_config_str = """ input_names: 'field1' feature_type: RawFeature initializer { @@ -39,11 +38,11 @@ def test_raw_embed(self): raw_input_dim: 5 embedding_dim: 2 combiner: 'sum' - ''' + """ feature_config = FeatureConfig() text_format.Merge(feature_config_str, feature_config) - data_config_str = ''' + data_config_str = """ input_fields { input_name: 'clk' input_type: INT32 @@ -56,15 +55,13 @@ def test_raw_embed(self): } label_fields: 'clk' batch_size: 1 - ''' + """ data_config = DatasetConfig() text_format.Merge(data_config_str, data_config) feature_configs = [feature_config] features = {'field1': tf.constant(['0.1,0.2,0.3,0.4,0.5'])} - dummy_input = DummyInput( - data_config, feature_configs, '', input_vals=features - ) + dummy_input = DummyInput(data_config, feature_configs, '', input_vals=features) field_dict, _ = dummy_input._build(tf.estimator.ModeKeys.TRAIN, {}) wide_and_deep_dict = {'field1': WideOrDeep.WIDE_AND_DEEP} @@ -93,7 +90,7 @@ def test_seq_multi_embed(self): # [7, 8 ], # [9, 10] # ] - feature_config_str = ''' + feature_config_str = """ input_names: 'field1' feature_type: SequenceFeature initializer { @@ -106,11 +103,11 @@ def test_seq_multi_embed(self): embedding_dim: 2 num_buckets: 5 combiner: 'mean' - ''' + """ feature_config = FeatureConfig() text_format.Merge(feature_config_str, feature_config) - data_config_str = ''' + data_config_str = """ input_fields { input_name: 'clk' input_type: INT32 @@ -123,22 +120,19 @@ def test_seq_multi_embed(self): } label_fields: 'clk' batch_size: 1 - ''' + """ data_config = DatasetConfig() text_format.Merge(data_config_str, data_config) feature_configs = [feature_config] features = {'field1': tf.constant(['0112', '132430'])} - dummy_input = DummyInput( - data_config, feature_configs, '', input_vals=features - ) + dummy_input = DummyInput(data_config, feature_configs, '', input_vals=features) field_dict, _ = dummy_input._build(tf.estimator.ModeKeys.TRAIN, {}) wide_and_deep_dict = {'field1': WideOrDeep.DEEP} fc_parser = FeatureColumnParser(feature_configs, wide_and_deep_dict) builder = feature_column._LazyBuilder(field_dict) - hist_embedding, hist_seq_len = \ - fc_parser.sequence_columns['field1']._get_sequence_dense_tensor(builder) + hist_embedding, hist_seq_len = fc_parser.sequence_columns['field1']._get_sequence_dense_tensor(builder) init = tf.initialize_all_variables() with tf.Session() as sess: diff --git a/easy_rec/python/test/emr_run.py b/easy_rec/python/test/emr_run.py index f258d63f4..725a83b4e 100644 --- a/easy_rec/python/test/emr_run.py +++ b/easy_rec/python/test/emr_run.py @@ -11,13 +11,14 @@ from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare +from easy_rec.python.test.odps_test_util import ( # NOQA + OdpsOSSConfig, + delete_oss_path, + get_oss_bucket, +) from easy_rec.python.utils import test_utils -from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA - -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') odps_oss_config = OdpsOSSConfig(script_path='./samples/emr_script') @@ -27,9 +28,7 @@ class TestPipelineOnEmr(tf.test.TestCase): def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) - self._test_hdfs_dir = test_utils.get_hdfs_tmp_dir( - 'hdfs://emr-header-1:9000/user/easy_rec/emr_test' - ) + self._test_hdfs_dir = test_utils.get_hdfs_tmp_dir('hdfs://emr-header-1:9000/user/easy_rec/emr_test') self._success = True logging.info('test hdfs dir: %s' % self._test_hdfs_dir) @@ -41,7 +40,7 @@ def tearDown(self): def test_deepfm_train_eval_export(self): start = [ 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql' + 'deep_fm/create_inner_deepfm_table.sql', ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) @@ -49,21 +48,21 @@ def test_deepfm_train_eval_export(self): self._success = test_utils.test_hdfs_train_eval( '%s/configs/deepfm.config' % odps_oss_config.temp_dir, '%s/yaml_config/train.paitf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir + self._test_hdfs_dir, ) self.assertTrue(self._success) self._success = test_utils.test_hdfs_eval( '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, '%s/yaml_config/eval.tf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir + self._test_hdfs_dir, ) self.assertTrue(self._success) self._success = test_utils.test_hdfs_export( '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, '%s/yaml_config/export.tf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir + self._test_hdfs_dir, ) self.assertTrue(self._success) @@ -72,31 +71,14 @@ def test_deepfm_train_eval_export(self): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( - '--odps_config', type=str, default=None, help='odps config path' - ) - parser.add_argument( - '--oss_config', type=str, default=None, help='ossutilconfig path' - ) - parser.add_argument( - '--bucket_name', type=str, default=None, help='test oss bucket name' - ) + parser.add_argument('--odps_config', type=str, default=None, help='odps config path') + parser.add_argument('--oss_config', type=str, default=None, help='ossutilconfig path') + parser.add_argument('--bucket_name', type=str, default=None, help='test oss bucket name') parser.add_argument('--arn', type=str, default=None, help='oss rolearn') - parser.add_argument( - '--odpscmd', type=str, default='odpscmd', help='odpscmd path' - ) - parser.add_argument( - '--algo_project', type=str, default=None, help='algo project name' - ) - parser.add_argument( - '--algo_res_project', - type=str, - default=None, - help='algo resource project name' - ) - parser.add_argument( - '--algo_version', type=str, default=None, help='algo version' - ) + parser.add_argument('--odpscmd', type=str, default='odpscmd', help='odpscmd path') + parser.add_argument('--algo_project', type=str, default=None, help='algo project name') + parser.add_argument('--algo_res_project', type=str, default=None, help='algo resource project name') + parser.add_argument('--algo_version', type=str, default=None, help='algo version') args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] for unk_arg in unknown_args: @@ -124,8 +106,10 @@ def test_deepfm_train_eval_export(self): tf.test.main() # delete oss path bucket = get_oss_bucket( - odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp diff --git a/easy_rec/python/test/eval_metric_test.py b/easy_rec/python/test/eval_metric_test.py index cb310721b..c7814a3d0 100644 --- a/easy_rec/python/test/eval_metric_test.py +++ b/easy_rec/python/test/eval_metric_test.py @@ -14,13 +14,13 @@ class MetricsTest(tf.test.TestCase, parameterized.TestCase): - def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) @RunAsSubprocess def test_max_f1(self): from easy_rec.python.core.metrics import max_f1 + labels = tf.constant([1, 0, 0, 1], dtype=tf.int32) probs = tf.constant([0.9, 0.8, 0.7, 0.6], dtype=tf.float32) f1, f1_update_op = max_f1(labels, probs) @@ -34,6 +34,7 @@ def test_max_f1(self): @RunAsSubprocess def test_gauc_all_negative_label(self): from easy_rec.python.core.metrics import gauc + labels = tf.constant([0, 0, 0, 0], dtype=tf.int32) probs = tf.constant([0.9, 0.8, 0.7, 0.6], dtype=tf.float32) uids = tf.constant([1, 1, 1, 1], dtype=tf.int32) @@ -47,15 +48,16 @@ def test_gauc_all_negative_label(self): [ ['_reduction_mean', 'mean', 0.5833333], ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], - ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6] + ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6], ] ) @RunAsSubprocess def test_gauc(self, reduction, expected): from easy_rec.python.core.metrics import gauc - labels = tf.placeholder(dtype=tf.int32, shape=(None, )) - probs = tf.placeholder(dtype=tf.float32, shape=(None, )) - uids = tf.placeholder(dtype=tf.int32, shape=(None, )) + + labels = tf.placeholder(dtype=tf.int32, shape=(None,)) + probs = tf.placeholder(dtype=tf.float32, shape=(None,)) + uids = tf.placeholder(dtype=tf.int32, shape=(None,)) value_op, update_op = gauc(labels, probs, uids, reduction=reduction) with tf.Session() as sess: sess.run( @@ -63,16 +65,16 @@ def test_gauc(self, reduction, expected): feed_dict={ labels: [1, 0, 1, 1, 0], probs: [0.9, 0.8, 0.7, 0.6, 0.5], - uids: [1, 1, 1, 1, 1] - } + uids: [1, 1, 1, 1, 1], + }, ) sess.run( update_op, feed_dict={ labels: [1, 0, 0, 1], probs: [0.9, 0.8, 0.7, 0.6], - uids: [2, 2, 2, 2] - } + uids: [2, 2, 2, 2], + }, ) score = sess.run(value_op) self.assertAlmostEqual(score, expected) @@ -81,34 +83,33 @@ def test_gauc(self, reduction, expected): [ ['_reduction_mean', 'mean', 0.5833333], ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], - ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6] + ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6], ] ) @RunAsSubprocess def test_session_auc(self, reduction, expected): from easy_rec.python.core.metrics import session_auc - labels = tf.placeholder(dtype=tf.int32, shape=(None, )) - probs = tf.placeholder(dtype=tf.float32, shape=(None, )) - session_ids = tf.placeholder(dtype=tf.int32, shape=(None, )) - value_op, update_op = session_auc( - labels, probs, session_ids, reduction=reduction - ) + + labels = tf.placeholder(dtype=tf.int32, shape=(None,)) + probs = tf.placeholder(dtype=tf.float32, shape=(None,)) + session_ids = tf.placeholder(dtype=tf.int32, shape=(None,)) + value_op, update_op = session_auc(labels, probs, session_ids, reduction=reduction) with tf.Session() as sess: sess.run( update_op, feed_dict={ labels: [1, 0, 1, 1, 0], probs: [0.9, 0.8, 0.7, 0.6, 0.5], - session_ids: [1, 1, 1, 1, 1] - } + session_ids: [1, 1, 1, 1, 1], + }, ) sess.run( update_op, feed_dict={ labels: [1, 0, 0, 1], probs: [0.9, 0.8, 0.7, 0.6], - session_ids: [2, 2, 2, 2] - } + session_ids: [2, 2, 2, 2], + }, ) score = sess.run(value_op) self.assertAlmostEqual(score, expected) diff --git a/easy_rec/python/test/excel_convert_test.py b/easy_rec/python/test/excel_convert_test.py index 35f4ff84c..479202383 100644 --- a/easy_rec/python/test/excel_convert_test.py +++ b/easy_rec/python/test/excel_convert_test.py @@ -10,7 +10,6 @@ class ExcelConvertTest(tf.test.TestCase): - def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) @@ -21,48 +20,42 @@ def test_deepfm_convert(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'avazu_deepfm_excel.config') - convert_cmd = """ + convert_cmd = ( + """ python -m easy_rec.python.tools.create_config_from_excel --excel_path samples/excel_config/dwd_avazu_ctr_deepfm.xls --model_type deepfm --output_path %s --train_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv --eval_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv - """ % pipeline_config_path - proc = test_utils.run_cmd( - convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + """ + % pipeline_config_path ) + proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir - ) - ) + self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) def test_multi_tower_convert(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'avazu_deepfm_excel.config') - convert_cmd = """ + convert_cmd = ( + """ python -m easy_rec.python.tools.create_config_from_excel --excel_path samples/excel_config/dwd_avazu_ctr_deepfm.xls --model_type multi_tower --output_path %s --train_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv --eval_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv - """ % pipeline_config_path - proc = test_utils.run_cmd( - convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + """ + % pipeline_config_path ) + proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir - ) - ) + self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) diff --git a/easy_rec/python/test/export_test.py b/easy_rec/python/test/export_test.py index 3c94b5ef3..5f65c8abd 100644 --- a/easy_rec/python/test/export_test.py +++ b/easy_rec/python/test/export_test.py @@ -19,7 +19,6 @@ class ExportTest(tf.test.TestCase): - def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) @@ -34,7 +33,7 @@ def _predict_and_check( cmp_result, keys=['probs'], separator=',', - tol=1e-4 + tol=1e-4, ): predictor = Predictor(saved_model_dir) with open(data_path, 'r') as fin: @@ -52,9 +51,11 @@ def _predict_and_check( val0 = output_res[i][key] val1 = cmp_result[i][key] diff = np.max(np.abs(val0 - val1)) - assert diff < tol, \ - 'too much difference: %.6f for %s, tol=%.6f' \ - % (diff, key, tol) + assert diff < tol, 'too much difference: %.6f for %s, tol=%.6f' % ( + diff, + key, + tol, + ) def _extract_data(self, input_path, output_path, offset=1, separator=','): with open(input_path, 'r') as fin: @@ -75,27 +76,23 @@ def _extract_rtp_data(self, input_path, output_path, separator=';'): fout.write('%s\n' % line_toks[-1]) def test_multi_tower(self): - self._export_test( - 'samples/model_config/multi_tower_export.config', self._extract_data - ) + self._export_test('samples/model_config/multi_tower_export.config', self._extract_data) def test_filter_input(self): - self._export_test( - 'samples/model_config/export_filter_input.config', self._extract_data - ) + self._export_test('samples/model_config/export_filter_input.config', self._extract_data) def test_mmoe(self): self._export_test( 'samples/model_config/mmoe_on_taobao.config', functools.partial(self._extract_data, offset=2), - keys=['probs_ctr', 'probs_cvr'] + keys=['probs_ctr', 'probs_cvr'], ) def test_fg(self): self._export_test( 'samples/model_config/taobao_fg.config', self._extract_rtp_data, - separator='' + separator='', ) def test_fg_export(self): @@ -103,18 +100,14 @@ def test_fg_export(self): 'samples/model_config/taobao_fg_export.config', self._extract_rtp_data, separator='', - test_multi=False + test_multi=False, ) def test_export_with_asset(self): pipeline_config_path = 'samples/model_config/taobao_fg.config' test_dir = test_utils.get_tmp_dir() # prepare model - self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir - ) - ) + self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) test_utils.set_gpu_id(None) config_path = os.path.join(test_dir, 'pipeline.config') export_dir = os.path.join(test_dir, 'export/') @@ -128,9 +121,7 @@ def test_export_with_asset(self): config_path, export_dir, ) - proc = test_utils.run_cmd( - export_cmd, '%s/log_%s.txt' % (test_dir, 'export') - ) + proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() self.assertTrue(proc.returncode == 0) files = gfile.Glob(export_dir + '*') @@ -153,10 +144,12 @@ def _post_check_func(pipeline_config): --pipeline_config_path %s --checkpoint_path %s --export_dir %s - """ % (pipeline_config_path, ckpt_path, export_dir) - proc = test_utils.run_cmd( - export_cmd, '%s/log_%s.txt' % (test_dir, 'export') + """ % ( + pipeline_config_path, + ckpt_path, + export_dir, ) + proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() return proc.returncode == 0 @@ -165,7 +158,7 @@ def _post_check_func(pipeline_config): test_utils.test_single_train_eval( pipeline_config_path, test_dir=test_dir, - post_check_func=_post_check_func + post_check_func=_post_check_func, ) ) @@ -173,7 +166,7 @@ def test_multi_class_predict(self): self._export_test( 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', extract_data_func=self._extract_data, - keys=['probs', 'logits', 'probs_y', 'logits_y', 'y'] + keys=['probs', 'logits', 'probs_y', 'logits_y', 'y'], ) def _export_test( @@ -182,32 +175,26 @@ def _export_test( extract_data_func=None, separator=',', keys=['probs'], - test_multi=True + test_multi=True, ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) # prepare model - self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir - ) - ) + self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) test_utils.set_gpu_id(None) # prepare two version config config_path_single = os.path.join(test_dir, 'pipeline.config') config_path_multi = os.path.join(test_dir, 'pipeline_v2.config') - pipeline_config = config_util.get_configs_from_pipeline_file( - config_path_single - ) + pipeline_config = config_util.get_configs_from_pipeline_file(config_path_single) if pipeline_config.export_config.multi_placeholder: - config_path_single, config_path_multi = config_path_multi, config_path_single - pipeline_config.export_config.multi_placeholder =\ - not pipeline_config.export_config.multi_placeholder - config_util.save_pipeline_config( - pipeline_config, test_dir, 'pipeline_v2.config' - ) + config_path_single, config_path_multi = ( + config_path_multi, + config_path_single, + ) + pipeline_config.export_config.multi_placeholder = not pipeline_config.export_config.multi_placeholder + config_util.save_pipeline_config(pipeline_config, test_dir, 'pipeline_v2.config') # prepare two version export dir export_dir_single = os.path.join(test_dir, 'train/export/final') @@ -216,10 +203,11 @@ def _export_test( python -m easy_rec.python.export --pipeline_config_path %s --export_dir %s - """ % (config_path_multi, export_dir_multi) - proc = test_utils.run_cmd( - export_cmd, '%s/log_%s.txt' % (test_dir, 'export') + """ % ( + config_path_multi, + export_dir_multi, ) + proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() self.assertTrue(proc.returncode == 0) @@ -229,10 +217,11 @@ def _export_test( python -m easy_rec.python.predict --pipeline_config_path %s --output_path %s - """ % (config_path_single, result_path) - proc = test_utils.run_cmd( - predict_cmd % (), '%s/log_%s.txt' % (test_dir, 'predict') + """ % ( + config_path_single, + result_path, ) + proc = test_utils.run_cmd(predict_cmd % (), '%s/log_%s.txt' % (test_dir, 'predict')) proc.wait() self.assertTrue(proc.returncode == 0) with open(result_path, 'r') as fin: @@ -251,7 +240,7 @@ def _export_test( export_dir_single, cmp_result, keys=keys, - separator=separator + separator=separator, ) if test_multi: self._predict_and_check( @@ -259,7 +248,7 @@ def _export_test( export_dir_multi, cmp_result, keys=keys, - separator=separator + separator=separator, ) test_utils.clean_up(test_dir) @@ -268,7 +257,7 @@ def _test_big_model_export( pipeline_config_path, test_data_path, extract_data_func=None, - total_steps=50 + total_steps=50, ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) @@ -277,11 +266,7 @@ def _test_big_model_export( tf.load_op_library(lookup_op_path) # prepare model - self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir, total_steps=total_steps - ) - ) + self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir, total_steps=total_steps)) test_utils.set_gpu_id(None) # the pipeline.config is produced by the prepare model cmd @@ -298,12 +283,13 @@ def _test_big_model_export( --redis_write_kv 1 --verbose 1 """ % ( - config_path, export_dir, test_data_path, os.environ['redis_url'], - os.environ['redis_passwd'] - ) - proc = test_utils.run_cmd( - export_cmd, '%s/log_%s.txt' % (test_dir, 'export') + config_path, + export_dir, + test_data_path, + os.environ['redis_url'], + os.environ['redis_passwd'], ) + proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() self.assertTrue(proc.returncode == 0) @@ -318,10 +304,12 @@ def _test_big_model_export( --pipeline_config_path %s --input_path %s --output_path %s - """ % (config_path, test_data_path, result_path) - proc = test_utils.run_cmd( - predict_cmd % (), '%s/log_%s.txt' % (test_dir, 'predict') + """ % ( + config_path, + test_data_path, + result_path, ) + proc = test_utils.run_cmd(predict_cmd % (), '%s/log_%s.txt' % (test_dir, 'predict')) proc.wait() self.assertTrue(proc.returncode == 0) with open(result_path, 'r') as fin: @@ -338,33 +326,25 @@ def _test_big_model_export( @unittest.skipIf( 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd' + 'Only execute when redis is available: redis_url, redis_passwd', ) def test_big_model_export(self): pipeline_config_path = 'samples/model_config/multi_tower_export.config' test_data_path = 'data/test/export/data.csv' - self._test_big_model_export( - pipeline_config_path, - test_data_path, - extract_data_func=self._extract_data - ) + self._test_big_model_export(pipeline_config_path, test_data_path, extract_data_func=self._extract_data) @unittest.skipIf( 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd' + 'Only execute when redis is available: redis_url, redis_passwd', ) def test_big_model_deepfm_export(self): pipeline_config_path = 'samples/model_config/deepfm_combo_on_avazu_ctr.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' - self._test_big_model_export( - pipeline_config_path, - test_data_path, - extract_data_func=self._extract_data - ) + self._test_big_model_export(pipeline_config_path, test_data_path, extract_data_func=self._extract_data) @unittest.skipIf( 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd' + 'Only execute when redis is available: redis_url, redis_passwd', ) def test_big_model_din_export(self): pipeline_config_path = 'samples/model_config/din_on_taobao.config' @@ -372,12 +352,12 @@ def test_big_model_din_export(self): self._test_big_model_export( pipeline_config_path, test_data_path, - extract_data_func=functools.partial(self._extract_data, offset=2) + extract_data_func=functools.partial(self._extract_data, offset=2), ) @unittest.skipIf( 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd' + 'Only execute when redis is available: redis_url, redis_passwd', ) def test_big_model_wide_and_deep_export(self): pipeline_config_path = 'samples/model_config/wide_and_deep_two_opti.config' @@ -385,12 +365,12 @@ def test_big_model_wide_and_deep_export(self): self._test_big_model_export( pipeline_config_path, test_data_path, - extract_data_func=functools.partial(self._extract_data) + extract_data_func=functools.partial(self._extract_data), ) @unittest.skipIf( 'redis_url' not in os.environ or '-PAI' not in tf.__version__, - 'Only execute when pai-tf and redis is available: redis_url, redis_passwd' + 'Only execute when pai-tf and redis is available: redis_url, redis_passwd', ) def test_big_model_embedding_variable_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev.config' @@ -399,15 +379,16 @@ def test_big_model_embedding_variable_export(self): pipeline_config_path, test_data_path, self._extract_rtp_data, - total_steps=1000 + total_steps=1000, ) @unittest.skipIf( - 'oss_endpoint' not in os.environ or 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ or 'oss_path' not in os.environ + 'oss_endpoint' not in os.environ + or 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ + or 'oss_path' not in os.environ or '-PAI' not in tf.__version__, - 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' - 'and pai-tf is available.' + 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' 'and pai-tf is available.', ) def test_big_model_embedding_variable_oss_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev.config' @@ -416,15 +397,16 @@ def test_big_model_embedding_variable_oss_export(self): pipeline_config_path, test_data_path, self._extract_rtp_data, - total_steps=100 + total_steps=100, ) @unittest.skipIf( - 'oss_endpoint' not in os.environ or 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ or 'oss_path' not in os.environ + 'oss_endpoint' not in os.environ + or 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ + or 'oss_path' not in os.environ or '-PAI' not in tf.__version__, - 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' - 'and pai-tf is available.' + 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' 'and pai-tf is available.', ) def test_big_model_embedding_variable_v2_oss_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev_v2.config' @@ -433,7 +415,7 @@ def test_big_model_embedding_variable_v2_oss_export(self): pipeline_config_path, test_data_path, self._extract_rtp_data, - total_steps=100 + total_steps=100, ) def _test_big_model_export_to_oss( @@ -441,7 +423,7 @@ def _test_big_model_export_to_oss( pipeline_config_path, test_data_path, extract_data_func=None, - total_steps=50 + total_steps=50, ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) @@ -450,11 +432,7 @@ def _test_big_model_export_to_oss( tf.load_op_library(lookup_op_path) # prepare model - self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir, total_steps=total_steps - ) - ) + self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir, total_steps=total_steps)) test_utils.set_gpu_id(None) # the pipeline.config is produced by the prepare model cmd @@ -473,12 +451,15 @@ def _test_big_model_export_to_oss( --oss_write_kv 1 --verbose 1 """ % ( - config_path, export_dir, test_data_path, os.environ['oss_path'], - os.environ['oss_endpoint'], os.environ['oss_ak'], os.environ['oss_sk'] - ) - proc = test_utils.run_cmd( - export_cmd, '%s/log_%s.txt' % (test_dir, 'export') + config_path, + export_dir, + test_data_path, + os.environ['oss_path'], + os.environ['oss_endpoint'], + os.environ['oss_ak'], + os.environ['oss_sk'], ) + proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() self.assertTrue(proc.returncode == 0) @@ -493,10 +474,12 @@ def _test_big_model_export_to_oss( --pipeline_config_path %s --input_path %s --output_path %s - """ % (config_path, test_data_path, result_path) - proc = test_utils.run_cmd( - predict_cmd, '%s/log_%s.txt' % (test_dir, 'predict') + """ % ( + config_path, + test_data_path, + result_path, ) + proc = test_utils.run_cmd(predict_cmd, '%s/log_%s.txt' % (test_dir, 'predict')) proc.wait() self.assertTrue(proc.returncode == 0) with open(result_path, 'r') as fin: @@ -513,33 +496,25 @@ def _test_big_model_export_to_oss( @unittest.skipIf( 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_export_to_oss(self): pipeline_config_path = 'samples/model_config/multi_tower_export.config' test_data_path = 'data/test/export/data.csv' - self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - extract_data_func=self._extract_data - ) + self._test_big_model_export_to_oss(pipeline_config_path, test_data_path, extract_data_func=self._extract_data) @unittest.skipIf( 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_deepfm_export_to_oss(self): pipeline_config_path = 'samples/model_config/deepfm_combo_on_avazu_ctr.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' - self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - extract_data_func=self._extract_data - ) + self._test_big_model_export_to_oss(pipeline_config_path, test_data_path, extract_data_func=self._extract_data) @unittest.skipIf( 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_din_export_to_oss(self): pipeline_config_path = 'samples/model_config/din_on_taobao.config' @@ -547,12 +522,12 @@ def test_big_model_din_export_to_oss(self): self._test_big_model_export_to_oss( pipeline_config_path, test_data_path, - extract_data_func=functools.partial(self._extract_data, offset=2) + extract_data_func=functools.partial(self._extract_data, offset=2), ) @unittest.skipIf( 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_wide_and_deep_export_to_oss(self): pipeline_config_path = 'samples/model_config/wide_and_deep_two_opti.config' @@ -560,7 +535,7 @@ def test_big_model_wide_and_deep_export_to_oss(self): self._test_big_model_export_to_oss( pipeline_config_path, test_data_path, - extract_data_func=functools.partial(self._extract_data) + extract_data_func=functools.partial(self._extract_data), ) diff --git a/easy_rec/python/test/fg_test.py b/easy_rec/python/test/fg_test.py index e29e38d4e..4d3ead271 100644 --- a/easy_rec/python/test/fg_test.py +++ b/easy_rec/python/test/fg_test.py @@ -13,7 +13,6 @@ class FGTest(tf.test.TestCase): - def __init__(self, methodName='FGTest'): super(FGTest, self).__init__(methodName=methodName) @@ -33,21 +32,13 @@ def test_fg_json_to_config(self): final_pipeline_config_path = 'samples/rtp_fg/fg_test_extensions_final.config' fg_path = 'samples/rtp_fg/fg_test_extensions.json' - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path - ) + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) pipeline_config.fg_json_path = fg_path fg_util.load_fg_json_to_config(pipeline_config) - pipeline_config_str = text_format.MessageToString( - pipeline_config, as_utf8=True - ) + pipeline_config_str = text_format.MessageToString(pipeline_config, as_utf8=True) - final_pipeline_config = config_util.get_configs_from_pipeline_file( - final_pipeline_config_path - ) - final_pipeline_config_str = text_format.MessageToString( - final_pipeline_config, as_utf8=True - ) + final_pipeline_config = config_util.get_configs_from_pipeline_file(final_pipeline_config_path) + final_pipeline_config_str = text_format.MessageToString(final_pipeline_config, as_utf8=True) self.assertEqual(pipeline_config_str, final_pipeline_config_str) def test_fg_dtype(self): @@ -57,18 +48,12 @@ def test_fg_dtype(self): self.assertTrue(self._success) def test_fg_train(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/fg_train.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/fg_train.config', self._test_dir) self.assertTrue(self._success) - @unittest.skipIf( - '-PAI' not in tf.__version__, 'Only test when pai-tf is used.' - ) + @unittest.skipIf('-PAI' not in tf.__version__, 'Only test when pai-tf is used.') def test_fg_train_ev(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/fg_train_ev.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/fg_train_ev.config', self._test_dir) self.assertTrue(self._success) diff --git a/easy_rec/python/test/hive_input_test.py b/easy_rec/python/test/hive_input_test.py index 8140820fd..cb320dcb7 100644 --- a/easy_rec/python/test/hive_input_test.py +++ b/easy_rec/python/test/hive_input_test.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Define cv_input, the base class for cv tasks.""" + import logging import os import unittest @@ -28,7 +29,6 @@ class HiveInputTest(tf.test.TestCase): - def _init_config(self): hive_host = os.environ['hive_host'] hive_username = os.environ['hive_username'] @@ -59,11 +59,12 @@ def __init__(self, methodName='HiveInputTest'): super(HiveInputTest, self).__init__(methodName=methodName) @unittest.skipIf( - 'hive_host' not in os.environ or 'hive_username' not in os.environ + 'hive_host' not in os.environ + or 'hive_username' not in os.environ or 'hive_table_name' not in os.environ or 'hive_hash_fields' not in os.environ, """Only execute hive_config var are specified,hive_host、 - hive_username、hive_table_name、hive_hash_fields is available.""" + hive_username、hive_table_name、hive_hash_fields is available.""", ) def test_hive_input(self): self._init_config() @@ -231,9 +232,7 @@ def test_hive_input(self): empty_config.input_names.pop() while len(empty_config.shared_names) > 0: empty_config.shared_names.pop() - train_input_fn = HiveInput( - dataset_config, feature_configs, self.hive_train_input_config - ).create_input() + train_input_fn = HiveInput(dataset_config, feature_configs, self.hive_train_input_config).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -243,7 +242,7 @@ def test_hive_input(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=False + log_device_placement=False, ) with self.test_session(config=session_config) as sess: sess.run(init_op) @@ -256,11 +255,12 @@ def test_hive_input(self): return 0 @unittest.skipIf( - 'hive_host' not in os.environ or 'hive_username' not in os.environ + 'hive_host' not in os.environ + or 'hive_username' not in os.environ or 'hive_table_name' not in os.environ or 'hive_hash_fields' not in os.environ, """Only execute hive_config var are specified,hive_host、 - hive_username、hive_table_name、hive_hash_fields is available.""" + hive_username、hive_table_name、hive_hash_fields is available.""", ) def test_mmoe(self): pipeline_config_path = 'samples/emr_script/mmoe/mmoe_census_income.config' @@ -278,9 +278,7 @@ def test_mmoe(self): if isinstance(pipeline_config_path, EasyRecConfig): pipeline_config = pipeline_config_path else: - pipeline_config = test_utils._load_config_for_test( - pipeline_config_path, self._test_dir - ) + pipeline_config = test_utils._load_config_for_test(pipeline_config_path, self._test_dir) pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -290,11 +288,10 @@ def test_mmoe(self): test_pipeline_config_path = os.path.join(self._test_dir, 'pipeline.config') hyperparam_str = '' train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path %s %s' % ( - test_pipeline_config_path, hyperparam_str - ) - proc = test_utils.run_cmd( - train_cmd, '%s/log_%s.txt' % (self._test_dir, 'master') + test_pipeline_config_path, + hyperparam_str, ) + proc = test_utils.run_cmd(train_cmd, '%s/log_%s.txt' % (self._test_dir, 'master')) proc.wait() if proc.returncode != 0: logging.error('train %s failed' % test_pipeline_config_path) diff --git a/easy_rec/python/test/hpo_test.py b/easy_rec/python/test/hpo_test.py index 19918a691..7594a5b8e 100644 --- a/easy_rec/python/test/hpo_test.py +++ b/easy_rec/python/test/hpo_test.py @@ -20,12 +20,12 @@ GPUOptions = config_pb2.GPUOptions else: from tensorflow.python.platform import gfile + GPUOptions = tf.GPUOptions ConfigProto = tf.ConfigProto class HPOTest(tf.test.TestCase): - def __init__(self, methodName='HPOTest'): super(HPOTest, self).__init__(methodName=methodName) self._metric_data_path = 'data/test/hpo_test/eval_val/*.tfevents.*' @@ -41,9 +41,7 @@ def load_config(self, config_path): def test_save_eval_metrics(self): test_dir = test_utils.get_tmp_dir() - tmp_file = os.path.join( - test_dir, 'easy_rec_hpo_test_%d.metric' % time.time() - ) + tmp_file = os.path.join(test_dir, 'easy_rec_hpo_test_%d.metric' % time.time()) hpo_util.save_eval_metrics('data/test/hpo_test/', tmp_file, False) test_utils.clean_up(test_dir) @@ -51,18 +49,14 @@ def test_edit_config(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) assert tmp_config.feature_config.features[0].embedding_dim == 120 def test_edit_config_v2(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v2.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for tmp_fea in tmp_config.feature_configs: if tmp_fea.input_names[0] == 'site_id': assert tmp_fea.embedding_dim == 32 @@ -73,9 +67,7 @@ def test_edit_config_v3(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v3.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if i >= 10 and i < 20: assert tmp_fea.embedding_dim == 37 @@ -86,9 +78,7 @@ def test_edit_config_v4(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v4.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if i < 15: assert tmp_fea.embedding_dim == 37 @@ -99,9 +89,7 @@ def test_edit_config_v5(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v5.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if i >= 5: assert tmp_fea.embedding_dim == 37 @@ -112,9 +100,7 @@ def test_edit_config_v51(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v51.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if i == 5: assert tmp_fea.embedding_dim == 37 @@ -123,13 +109,12 @@ def test_edit_config_v6(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v6.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] >= 'site': assert tmp_fea.embedding_dim == 32, 'input_name = %s %d' % ( - tmp_fea.input_names[0], tmp_fea.embedding_dim + tmp_fea.input_names[0], + tmp_fea.embedding_dim, ) else: assert tmp_fea.embedding_dim == 16 @@ -138,46 +123,35 @@ def test_edit_config_v7(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v7.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries - ) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 + assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 def test_edit_config_v71(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v71.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries - ) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 + assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 def test_edit_config_v8(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v8.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries - ) == 4 and np.abs(tmp_fea.boundaries[0] - 4.0) < 1e-5 + assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - 4.0) < 1e-5 assert tmp_fea.embedding_dim == 32 def test_edit_config_v81(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v81.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.feature_type == tmp_fea.RawFeature: assert tmp_fea.embedding_dim == 24 @@ -186,44 +160,33 @@ def test_edit_config_v9(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v9.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) - assert tmp_config.train_config.fine_tune_checkpoint == \ - 'oss://easy-rec/test/experiment/ctr_v93/model.ckpt-1000' + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) + assert tmp_config.train_config.fine_tune_checkpoint == 'oss://easy-rec/test/experiment/ctr_v93/model.ckpt-1000' def test_edit_config_v10(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v10.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries - ) == 4 and np.abs(tmp_fea.boundaries[0] - 4.0) < 1e-5 + assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - 4.0) < 1e-5 assert tmp_fea.embedding_dim == 32 def test_edit_config_v11(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v11.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries - ) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 + assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 def test_edit_config_v12(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v12.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': assert len(tmp_fea.boundaries) == 25 @@ -233,18 +196,14 @@ def test_edit_config_v13(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v13.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) assert not tmp_config.export_config.multi_placeholder def test_edit_config_v14(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v14.json' - tmp_config = config_util.edit_config( - tmp_config, self.load_config(tmp_file) - ) + tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'hour': assert len(tmp_fea.feature_type) == FeatureConfig.RawFeature @@ -259,9 +218,7 @@ def test_save_eval_metrics_with_env(self): } """ test_dir = test_utils.get_tmp_dir() - tmp_file = os.path.join( - test_dir, 'easy_rec_hpo_test_%d.metric' % time.time() - ) + tmp_file = os.path.join(test_dir, 'easy_rec_hpo_test_%d.metric' % time.time()) hpo_util.save_eval_metrics('data/test/hpo_test/', tmp_file, False) test_utils.clean_up(test_dir) diff --git a/easy_rec/python/test/kafka_test.py b/easy_rec/python/test/kafka_test.py index 630517e01..87b2c4a9f 100644 --- a/easy_rec/python/test/kafka_test.py +++ b/easy_rec/python/test/kafka_test.py @@ -28,7 +28,6 @@ class KafkaTest(tf.test.TestCase): - def setUp(self): self._success = True self._test_dir = test_utils.get_tmp_dir() @@ -37,10 +36,7 @@ def setUp(self): self._zookeeper_proc = None return - logging.info( - 'Testing %s.%s, test_dir=%s' % - (type(self).__name__, self._testMethodName, self._test_dir) - ) + logging.info('Testing %s.%s, test_dir=%s' % (type(self).__name__, self._testMethodName, self._test_dir)) self._log_dir = os.path.join(self._test_dir, 'logs') if not gfile.IsDirectory(self._log_dir): gfile.MakeDirs(self._log_dir) @@ -61,7 +57,8 @@ def setUp(self): else: fout.write(line_str) cmd = 'bash %s/bin/zookeeper-server-start.sh %s' % ( - kafka_install_dir, zookeeper_config + kafka_install_dir, + zookeeper_config, ) log_file = os.path.join(self._log_dir, 'zookeeper.log') self._zookeeper_proc = test_utils.run_cmd(cmd, log_file) @@ -76,15 +73,15 @@ def setUp(self): else: fout.write(line_str) cmd = 'bash %s/bin/kafka-server-start.sh %s' % ( - kafka_install_dir, kafka_config + kafka_install_dir, + kafka_config, ) log_file = os.path.join(self._log_dir, 'kafka_server.log') self._kafka_server_proc = test_utils.run_cmd(cmd, log_file) started = False while not started: - if self._kafka_server_proc.poll( - ) and self._kafka_server_proc.returncode: + if self._kafka_server_proc.poll() and self._kafka_server_proc.returncode: logging.warning('start kafka server failed, will retry.') os.system('cat %s' % log_file) self._kafka_server_proc = test_utils.run_cmd(cmd, log_file) @@ -92,9 +89,7 @@ def setUp(self): else: try: admin_clt = KafkaAdminClient(bootstrap_servers=self._kafka_servers) - logging.info( - 'old topics: %s' % (','.join(admin_clt.list_topics())) - ) + logging.info('old topics: %s' % (','.join(admin_clt.list_topics()))) admin_clt.close() started = True except kafka.errors.NoBrokersAvailable: @@ -114,7 +109,7 @@ def _create_topic(self, num_partitions=2): NewTopic( name=self._test_topic, num_partitions=num_partitions, - replication_factor=1 + replication_factor=1, ) ] @@ -152,18 +147,13 @@ def tearDown(self): if self._success: test_utils.clean_up(self._test_dir) - @unittest.skipIf( - 'kafka_install_dir' not in os.environ, - 'Only execute when kafka is available' - ) + @unittest.skipIf('kafka_install_dir' not in os.environ, 'Only execute when kafka is available') def test_kafka_ops(self): try: test_utils.set_gpu_id(None) def _generate(): - producer = KafkaProducer( - bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1) - ) + producer = KafkaProducer(bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1)) i = 0 while not self._should_stop: msg = 'user_id_%d' % i @@ -181,14 +171,12 @@ def _generate(): # control the maximal read of each partition config_global=['max.partition.fetch.bytes=1048576'], message_key=True, - message_offset=True + message_offset=True, ) batch_dataset = k.batch(5) - iterator = iterator_ops.Iterator.from_structure( - batch_dataset.output_types - ) + iterator = iterator_ops.Iterator.from_structure(batch_dataset.output_types) init_batch_op = iterator.make_initializer(batch_dataset) get_next = iterator.get_next() @@ -217,10 +205,7 @@ def _generate(): self._success = False raise ex - @unittest.skipIf( - 'kafka_install_dir' not in os.environ, - 'Only execute when kafka is available' - ) + @unittest.skipIf('kafka_install_dir' not in os.environ, 'Only execute when kafka is available') def test_kafka_train(self): try: # start produce thread @@ -237,9 +222,7 @@ def test_kafka_train(self): raise ex def _generate(self): - producer = KafkaProducer( - bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1) - ) + producer = KafkaProducer(bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1)) while not self._should_stop: with open('data/test/dwd_avazu_ctr_deepmodel_10w.csv', 'r') as fin: for line_str in fin: @@ -252,10 +235,7 @@ def _generate(self): producer.close() logging.info('data generation thread done.') - @unittest.skipIf( - 'kafka_install_dir' not in os.environ, - 'Only execute when kafka is available' - ) + @unittest.skipIf('kafka_install_dir' not in os.environ, 'Only execute when kafka is available') def test_kafka_train_chief_redundant(self): try: # start produce thread @@ -266,17 +246,14 @@ def test_kafka_train_chief_redundant(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/deepfm_combo_avazu_kafka_chief_redundant.config', self._test_dir, - num_evaluator=1 + num_evaluator=1, ) self.assertTrue(self._success) except Exception as ex: self._success = False raise ex - @unittest.skipIf( - 'kafka_install_dir' not in os.environ, - 'Only execute when kafka is available' - ) + @unittest.skipIf('kafka_install_dir' not in os.environ, 'Only execute when kafka is available') def test_kafka_train_v2(self): try: # start produce thread @@ -286,7 +263,7 @@ def test_kafka_train_v2(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_avazu_kafka_time_offset.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -295,30 +272,30 @@ def test_kafka_train_v2(self): raise ex @unittest.skipIf( - 'kafka_install_dir' not in os.environ or 'oss_path' not in os.environ - or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ, 'Only execute when kafka is available' + 'kafka_install_dir' not in os.environ + or 'oss_path' not in os.environ + or 'oss_endpoint' not in os.environ + and 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ, + 'Only execute when kafka is available', ) def test_kafka_processor(self): - self._test_kafka_processor( - 'samples/model_config/taobao_fg_incr_save.config' - ) + self._test_kafka_processor('samples/model_config/taobao_fg_incr_save.config') @unittest.skipIf( - 'kafka_install_dir' not in os.environ or 'oss_path' not in os.environ - or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ, 'Only execute when kafka is available' + 'kafka_install_dir' not in os.environ + or 'oss_path' not in os.environ + or 'oss_endpoint' not in os.environ + and 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ, + 'Only execute when kafka is available', ) def test_kafka_processor_ev(self): - self._test_kafka_processor( - 'samples/model_config/taobao_fg_incr_save_ev.config' - ) + self._test_kafka_processor('samples/model_config/taobao_fg_incr_save_ev.config') def _test_kafka_processor(self, config_path): self._success = False - success = test_utils.test_distributed_train_eval( - config_path, self._test_dir, total_steps=500 - ) + success = test_utils.test_distributed_train_eval(config_path, self._test_dir, total_steps=500) self.assertTrue(success) export_cmd = """ python -m easy_rec.python.export --pipeline_config_path %s/pipeline.config @@ -326,13 +303,15 @@ def _test_kafka_processor(self, config_path): --asset_files ./samples/rtp_fg/fg.json --checkpoint_path %s/train/model.ckpt-0 """ % ( - self._test_dir, self._test_dir, os.environ['oss_path'], - os.environ['oss_ak'], os.environ['oss_sk'], os.environ['oss_endpoint'], - self._test_dir - ) - proc = test_utils.run_cmd( - export_cmd, '%s/log_export_sep.txt' % self._test_dir + self._test_dir, + self._test_dir, + os.environ['oss_path'], + os.environ['oss_ak'], + os.environ['oss_sk'], + os.environ['oss_endpoint'], + self._test_dir, ) + proc = test_utils.run_cmd(export_cmd, '%s/log_export_sep.txt' % self._test_dir) proc.wait() self.assertTrue(proc.returncode == 0) files = gfile.Glob(os.path.join(self._test_dir, 'export/sep/[1-9][0-9]*')) @@ -342,12 +321,14 @@ def _test_kafka_processor(self, config_path): python -m easy_rec.python.inference.processor.test --saved_model_dir %s --input_path data/test/rtp/taobao_test_feature.txt --output_path %s/processor.out --test_dir %s - """ % (export_sep_dir, self._test_dir, self._test_dir) + """ % ( + export_sep_dir, + self._test_dir, + self._test_dir, + ) envs = dict(os.environ) envs['PROCESSOR_TEST'] = '1' - proc = test_utils.run_cmd( - predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs - ) + proc = test_utils.run_cmd(predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs) proc.wait() self.assertTrue(proc.returncode == 0) @@ -369,9 +350,7 @@ def _test_kafka_processor(self, config_path): with open('%s/predictor.out' % self._test_dir, 'w') as fout: for i in range(len(output_res)): - fout.write( - json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n' - ) + fout.write(json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n') for i in range(len(output_res)): val0 = output_res[i]['probs'] @@ -380,10 +359,7 @@ def _test_kafka_processor(self, config_path): assert diff < 1e-4, 'too much difference[%.6f] >= 1e-4' % diff self._success = True - @unittest.skipIf( - 'kafka_install_dir' not in os.environ, - 'Only execute when kafka is available' - ) + @unittest.skipIf('kafka_install_dir' not in os.environ, 'Only execute when kafka is available') def test_kafka_train_v3(self): try: # start produce thread @@ -393,7 +369,7 @@ def test_kafka_train_v3(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_avazu_kafka_time_offset2.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) diff --git a/easy_rec/python/test/local_incr_test.py b/easy_rec/python/test/local_incr_test.py index b4ffdfe4f..26623d5d3 100644 --- a/easy_rec/python/test/local_incr_test.py +++ b/easy_rec/python/test/local_incr_test.py @@ -15,48 +15,44 @@ class LocalIncrTest(tf.test.TestCase): - def setUp(self): self._success = True self._test_dir = test_utils.get_tmp_dir() - logging.info( - 'Testing %s.%s, test_dir=%s' % - (type(self).__name__, self._testMethodName, self._test_dir) - ) + logging.info('Testing %s.%s, test_dir=%s' % (type(self).__name__, self._testMethodName, self._test_dir)) self._log_dir = os.path.join(self._test_dir, 'logs') if not gfile.IsDirectory(self._log_dir): gfile.MakeDirs(self._log_dir) @unittest.skipIf( 'oss_path' not in os.environ - or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ, 'Only execute when kafka is available' + or 'oss_endpoint' not in os.environ + and 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ, + 'Only execute when kafka is available', ) def test_incr_save(self): - self._test_incr_save( - 'samples/model_config/taobao_fg_incr_save_local.config' - ) + self._test_incr_save('samples/model_config/taobao_fg_incr_save_local.config') @unittest.skipIf( 'oss_path' not in os.environ - or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ, 'Only execute when kafka is available' + or 'oss_endpoint' not in os.environ + and 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ, + 'Only execute when kafka is available', ) def test_incr_save_ev(self): - self._test_incr_save( - 'samples/model_config/taobao_fg_incr_save_ev_local.config' - ) + self._test_incr_save('samples/model_config/taobao_fg_incr_save_ev_local.config') @unittest.skipIf( 'oss_path' not in os.environ - or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ, 'Only execute when kafka is available' + or 'oss_endpoint' not in os.environ + and 'oss_ak' not in os.environ + or 'oss_sk' not in os.environ, + 'Only execute when kafka is available', ) def test_incr_save_share_ev(self): - self._test_incr_save( - 'samples/model_config/taobao_fg_incr_save_share_ev_local.config' - ) + self._test_incr_save('samples/model_config/taobao_fg_incr_save_share_ev_local.config') def _test_incr_save(self, config_path): self._success = False @@ -65,9 +61,8 @@ def _test_incr_save(self, config_path): self._test_dir, total_steps=100, edit_config_json={ - 'train_config.incr_save_config.fs.mount_path': - os.path.join(self._test_dir, 'train/incr_save/') - } + 'train_config.incr_save_config.fs.mount_path': os.path.join(self._test_dir, 'train/incr_save/') + }, ) self.assertTrue(success) export_cmd = """ @@ -76,13 +71,15 @@ def _test_incr_save(self, config_path): --asset_files ./samples/rtp_fg/fg.json --checkpoint_path %s/train/model.ckpt-0 """ % ( - self._test_dir, self._test_dir, os.environ['oss_path'], - os.environ['oss_ak'], os.environ['oss_sk'], os.environ['oss_endpoint'], - self._test_dir - ) - proc = test_utils.run_cmd( - export_cmd, '%s/log_export_sep.txt' % self._test_dir + self._test_dir, + self._test_dir, + os.environ['oss_path'], + os.environ['oss_ak'], + os.environ['oss_sk'], + os.environ['oss_endpoint'], + self._test_dir, ) + proc = test_utils.run_cmd(export_cmd, '%s/log_export_sep.txt' % self._test_dir) proc.wait() self.assertTrue(proc.returncode == 0) files = gfile.Glob(os.path.join(self._test_dir, 'export/sep/[1-9][0-9]*')) @@ -92,12 +89,14 @@ def _test_incr_save(self, config_path): python -m easy_rec.python.inference.processor.test --saved_model_dir %s --input_path data/test/rtp/taobao_test_feature.txt --output_path %s/processor.out --test_dir %s - """ % (export_sep_dir, self._test_dir, self._test_dir) + """ % ( + export_sep_dir, + self._test_dir, + self._test_dir, + ) envs = dict(os.environ) envs['PROCESSOR_TEST'] = '1' - proc = test_utils.run_cmd( - predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs - ) + proc = test_utils.run_cmd(predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs) proc.wait() self.assertTrue(proc.returncode == 0) @@ -119,9 +118,7 @@ def _test_incr_save(self, config_path): with open('%s/predictor.out' % self._test_dir, 'w') as fout: for i in range(len(output_res)): - fout.write( - json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n' - ) + fout.write(json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n') for i in range(len(output_res)): val0 = output_res[i]['probs'] diff --git a/easy_rec/python/test/loss_test.py b/easy_rec/python/test/loss_test.py index 798538c57..aa7184df4 100644 --- a/easy_rec/python/test/loss_test.py +++ b/easy_rec/python/test/loss_test.py @@ -1,23 +1,27 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.loss.circle_loss import circle_loss, get_anchor_positive_triplet_mask # NOQA -from easy_rec.python.loss.f1_reweight_loss import f1_reweight_sigmoid_cross_entropy # NOQA -from easy_rec.python.loss.softmax_loss_with_negative_mining import softmax_loss_with_negative_mining # NOQA +from easy_rec.python.loss.circle_loss import ( # NOQA + circle_loss, + get_anchor_positive_triplet_mask, +) +from easy_rec.python.loss.f1_reweight_loss import ( # NOQA + f1_reweight_sigmoid_cross_entropy, +) +from easy_rec.python.loss.softmax_loss_with_negative_mining import ( # NOQA + softmax_loss_with_negative_mining, +) if tf.__version__ >= '2.0': tf = tf.compat.v1 class LossTest(tf.test.TestCase): - def test_f1_reweighted_loss(self): print('test_f1_reweighted_loss') logits = tf.constant([0.1, 0.5, 0.3, 0.8, -0.1, 0.3]) labels = tf.constant([1, 1, 0, 0, 1, 1]) - loss = f1_reweight_sigmoid_cross_entropy( - labels=labels, logits=logits, beta_square=4 - ) + loss = f1_reweight_sigmoid_cross_entropy(labels=labels, logits=logits, beta_square=4) with self.test_session() as sess: loss_val = sess.run(loss) self.assertAlmostEqual(loss_val, 0.47844395, delta=1e-5) @@ -26,21 +30,27 @@ def test_softmax_loss_with_negative_mining(self): print('test_softmax_loss_with_negative_mining') user_emb = tf.constant( [ - [0.1, 0.5, 0.3], [0.8, -0.1, 0.3], [0.28, 0.3, 0.9], - [0.37, 0.45, 0.93], [-0.7, 0.15, 0.03], [0.18, 0.9, -0.3] + [0.1, 0.5, 0.3], + [0.8, -0.1, 0.3], + [0.28, 0.3, 0.9], + [0.37, 0.45, 0.93], + [-0.7, 0.15, 0.03], + [0.18, 0.9, -0.3], ] ) item_emb = tf.constant( [ - [0.1, -0.5, 0.3], [0.8, -0.31, 0.3], [0.7, -0.45, 0.15], - [0.08, -0.31, -0.9], [-0.7, 0.85, 0.03], [0.18, 0.89, -0.3] + [0.1, -0.5, 0.3], + [0.8, -0.31, 0.3], + [0.7, -0.45, 0.15], + [0.08, -0.31, -0.9], + [-0.7, 0.85, 0.03], + [0.18, 0.89, -0.3], ] ) label = tf.constant([1, 1, 0, 0, 1, 1]) - loss = softmax_loss_with_negative_mining( - user_emb, item_emb, label, num_negative_samples=2, seed=1 - ) + loss = softmax_loss_with_negative_mining(user_emb, item_emb, label, num_negative_samples=2, seed=1) with self.test_session() as sess: loss_val = sess.run(loss) self.assertAlmostEqual(loss_val, 0.48577175, delta=1e-5) @@ -49,10 +59,14 @@ def test_circle_loss(self): print('test_circle_loss') emb = tf.constant( [ - [0.1, 0.2, 0.15, 0.1], [0.3, 0.6, 0.45, 0.3], [0.13, 0.6, 0.45, 0.3], - [0.3, 0.26, 0.45, 0.3], [0.3, 0.6, 0.5, 0.13], [0.08, 0.43, 0.21, 0.6] + [0.1, 0.2, 0.15, 0.1], + [0.3, 0.6, 0.45, 0.3], + [0.13, 0.6, 0.45, 0.3], + [0.3, 0.26, 0.45, 0.3], + [0.3, 0.6, 0.5, 0.13], + [0.08, 0.43, 0.21, 0.6], ], - dtype=tf.float32 + dtype=tf.float32, ) label = tf.constant([1, 1, 2, 2, 3, 3]) loss = circle_loss(emb, label, label, margin=0.25, gamma=64) @@ -65,21 +79,29 @@ def test_triplet_mask(self): label = tf.constant([1, 1, 2, 2, 3, 3, 4, 5]) positive_mask = tf.constant( [ - [0., 1., 0., 0., 0., 0., 0., 0.], [1., 0., 0., 0., 0., 0., 0., 0.], - [0., 0., 0., 1., 0., 0., 0., 0.], [0., 0., 1., 0., 0., 0., 0., 0.], - [0., 0., 0., 0., 0., 1., 0., 0.], [0., 0., 0., 0., 1., 0., 0., 0.], - [0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0.] + [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], ], - dtype=tf.float32 + dtype=tf.float32, ) negative_mask = tf.constant( [ - [0., 0., 1., 1., 1., 1., 1., 1.], [0., 0., 1., 1., 1., 1., 1., 1.], - [1., 1., 0., 0., 1., 1., 1., 1.], [1., 1., 0., 0., 1., 1., 1., 1.], - [1., 1., 1., 1., 0., 0., 1., 1.], [1., 1., 1., 1., 0., 0., 1., 1.], - [1., 1., 1., 1., 1., 1., 0., 1.], [1., 1., 1., 1., 1., 1., 1., 0.] + [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], ], - dtype=tf.float32 + dtype=tf.float32, ) with self.test_session(): pos_mask = get_anchor_positive_triplet_mask(label, label) @@ -105,18 +127,14 @@ def _get_anchor_negative_triplet_mask(labels, sessions): """ # Check if sessions[i] != sessions[k] # Uses broadcasting where the 1st argument has shape (1, batch_size) and the 2nd (batch_size, 1) - session_not_equal = tf.not_equal( - tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1) - ) + session_not_equal = tf.not_equal(tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1)) if labels is sessions: return tf.cast(session_not_equal, tf.float32) # Check if labels[i] != labels[k] # Uses broadcasting where the 1st argument has shape (1, batch_size) and the 2nd (batch_size, 1) - label_not_equal = tf.not_equal( - tf.expand_dims(labels, 0), tf.expand_dims(labels, 1) - ) + label_not_equal = tf.not_equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1)) mask = tf.logical_or(session_not_equal, label_not_equal) return tf.cast(mask, tf.float32) diff --git a/easy_rec/python/test/odps_command.py b/easy_rec/python/test/odps_command.py index 123944c0b..c6ace1bd1 100644 --- a/easy_rec/python/test/odps_command.py +++ b/easy_rec/python/test/odps_command.py @@ -9,7 +9,6 @@ class OdpsCommand: - def __init__(self, odps_oss_config): """Wrapper for running odps command. @@ -17,8 +16,10 @@ def __init__(self, odps_oss_config): odps_oss_config: instance of easy_rec.python.utils.odps_test_util.OdpsOSSConfig """ self.bucket = get_oss_bucket( - odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) self.bucket_name = odps_oss_config.bucket_name self.temp_dir = odps_oss_config.temp_dir @@ -43,22 +44,24 @@ def run_odps_cmd(self, script_file): if self.odps_config_path is None: cmd = 'nohup %s -f %s > %s.log 2>&1' % ( - self.odpscmd, exec_file_path, log_file + self.odpscmd, + exec_file_path, + log_file, ) else: cmd = 'nohup %s --config=%s -f %s > %s.log 2>&1' % ( - self.odpscmd, self.odps_config_path, exec_file_path, log_file + self.odpscmd, + self.odps_config_path, + exec_file_path, + log_file, ) logging.info('will run cmd: %s' % (cmd)) proc = subprocess.Popen(cmd, shell=True) proc.wait() - if (proc.returncode == 0): + if proc.returncode == 0: logging.info('%s run succeed' % script_file) else: - raise ValueError( - '%s run FAILED: please check log file:%s.log' % - (exec_file_path, log_file) - ) + raise ValueError('%s run FAILED: please check log file:%s.log' % (exec_file_path, log_file)) def run_list(self, files): for f in files: diff --git a/easy_rec/python/test/odps_local_run.py b/easy_rec/python/test/odps_local_run.py index d1804254d..3e83e1b27 100644 --- a/easy_rec/python/test/odps_local_run.py +++ b/easy_rec/python/test/odps_local_run.py @@ -11,13 +11,14 @@ from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare +from easy_rec.python.test.odps_test_util import ( # NOQA + OdpsOSSConfig, + delete_oss_path, + get_oss_bucket, +) from easy_rec.python.utils import test_utils -from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA - -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') odps_oss_config = OdpsOSSConfig(script_path='./samples/emr_script') @@ -37,7 +38,7 @@ def tearDown(self): def test_deepfm_local_with_common_io(self): start = [ 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql' + 'deep_fm/create_inner_deepfm_table.sql', ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) @@ -51,19 +52,11 @@ def test_deepfm_local_with_common_io(self): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( - '--odps_config', type=str, default=None, help='odps config path' - ) - parser.add_argument( - '--oss_config', type=str, default=None, help='ossutilconfig path' - ) - parser.add_argument( - '--bucket_name', type=str, default=None, help='test oss bucket name' - ) + parser.add_argument('--odps_config', type=str, default=None, help='odps config path') + parser.add_argument('--oss_config', type=str, default=None, help='ossutilconfig path') + parser.add_argument('--bucket_name', type=str, default=None, help='test oss bucket name') parser.add_argument('--arn', type=str, default=None, help='oss rolearn') - parser.add_argument( - '--odpscmd', type=str, default='odpscmd', help='odpscmd path' - ) + parser.add_argument('--odpscmd', type=str, default='odpscmd', help='odpscmd path') args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] for unk_arg in unknown_args: @@ -85,8 +78,10 @@ def test_deepfm_local_with_common_io(self): tf.test.main() # delete oss path bucket = get_oss_bucket( - odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp diff --git a/easy_rec/python/test/odps_run.py b/easy_rec/python/test/odps_run.py index 1af392943..3b4f8b99b 100644 --- a/easy_rec/python/test/odps_run.py +++ b/easy_rec/python/test/odps_run.py @@ -12,13 +12,14 @@ from easy_rec.python.test.odps_test_cls import OdpsTest from easy_rec.python.test.odps_test_prepare import prepare +from easy_rec.python.test.odps_test_util import ( # NOQA + OdpsOSSConfig, + delete_oss_path, + get_oss_bucket, +) from easy_rec.python.utils import config_util -from easy_rec.python.test.odps_test_util import OdpsOSSConfig, delete_oss_path, get_oss_bucket # NOQA - -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') odps_oss_config = OdpsOSSConfig() @@ -29,9 +30,11 @@ class TestPipelineOnOdps(tf.test.TestCase): def test_deepfm(self): start_files = ['deep_fm/create_inner_deepfm_table.sql'] test_files = [ - 'deep_fm/train_deepfm_model.sql', 'deep_fm/eval_deepfm.sql', - 'deep_fm/export_deepfm.sql', 'deep_fm/predict_deepfm.sql', - 'deep_fm/export_rtp_ckpt.sql' + 'deep_fm/train_deepfm_model.sql', + 'deep_fm/eval_deepfm.sql', + 'deep_fm/export_deepfm.sql', + 'deep_fm/predict_deepfm.sql', + 'deep_fm/export_rtp_ckpt.sql', ] end_file = ['deep_fm/drop_table.sql'] @@ -113,7 +116,7 @@ def test_best_exporter(self): tot.start_test() config_path = os.path.join( odps_oss_config.temp_dir, - 'configs/dwd_avazu_ctr_deepmodel_ext_best_export.config' + 'configs/dwd_avazu_ctr_deepmodel_ext_best_export.config', ) config = config_util.get_configs_from_pipeline_file(config_path) model_dir = config.model_dir @@ -124,24 +127,21 @@ def test_best_exporter(self): logging.info('stripped model_dir = %s' % model_dir) bucket = get_oss_bucket( - odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) best_ckpt_prefix = os.path.join(model_dir, 'best_ckpt/model.ckpt') - best_ckpts = [ - x.key for x in oss2.ObjectIterator(bucket, prefix=best_ckpt_prefix) - if x.key.endswith('.meta') - ] + best_ckpts = [x.key for x in oss2.ObjectIterator(bucket, prefix=best_ckpt_prefix) if x.key.endswith('.meta')] logging.info('best ckpts: %s' % str(best_ckpts)) assert len(best_ckpts) <= 2, 'too many best ckpts: %s' % str(best_ckpts) best_export_prefix = os.path.join(model_dir, 'export/best/') best_exports = [ - x.key for x in oss2.ObjectIterator(bucket, prefix=best_export_prefix) - if x.key.endswith('/saved_model.pb') + x.key for x in oss2.ObjectIterator(bucket, prefix=best_export_prefix) if x.key.endswith('/saved_model.pb') ] logging.info('best exports: %s' % str(best_exports)) - assert len(best_exports - ) <= 2, 'too many best exports: %s' % str(best_exports) + assert len(best_exports) <= 2, 'too many best exports: %s' % str(best_exports) return True def test_embedding_variable(self): @@ -149,8 +149,9 @@ def test_embedding_variable(self): 'embedding_variable/create_table.sql', ] test_files = [ - 'embedding_variable/train.sql', 'embedding_variable/train_work_que.sql', - 'embedding_variable/export.sql' + 'embedding_variable/train.sql', + 'embedding_variable/train_work_que.sql', + 'embedding_variable/export.sql', ] end_file = ['embedding_variable/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -172,7 +173,8 @@ def test_boundary_test(self): test_files = [ 'boundary/train_multi_tower_model.sql', 'boundary/finetune_multi_tower_model.sql', - 'boundary/finetune_multi_tower_conti.sql', 'boundary/train_compat.sql' + 'boundary/finetune_multi_tower_conti.sql', + 'boundary/train_compat.sql', ] end_file = ['boundary/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -190,43 +192,20 @@ def test_vector_retrieve(self): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( - '--odps_config', type=str, default=None, help='odps config path' - ) - parser.add_argument( - '--oss_config', type=str, default=None, help='ossutilconfig path' - ) - parser.add_argument( - '--bucket_name', type=str, default=None, help='test oss bucket name' - ) + parser.add_argument('--odps_config', type=str, default=None, help='odps config path') + parser.add_argument('--oss_config', type=str, default=None, help='ossutilconfig path') + parser.add_argument('--bucket_name', type=str, default=None, help='test oss bucket name') parser.add_argument('--arn', type=str, default=None, help='oss rolearn') - parser.add_argument( - '--odpscmd', type=str, default='odpscmd', help='odpscmd path' - ) - parser.add_argument( - '--algo_name', - type=str, - default='easy_rec_ext', - help='whether use pai-tf 1.15' - ) - parser.add_argument( - '--algo_project', type=str, default=None, help='algo project name' - ) - parser.add_argument( - '--algo_res_project', - type=str, - default=None, - help='algo resource project name' - ) - parser.add_argument( - '--algo_version', type=str, default=None, help='algo version' - ) + parser.add_argument('--odpscmd', type=str, default='odpscmd', help='odpscmd path') + parser.add_argument('--algo_name', type=str, default='easy_rec_ext', help='whether use pai-tf 1.15') + parser.add_argument('--algo_project', type=str, default=None, help='algo project name') + parser.add_argument('--algo_res_project', type=str, default=None, help='algo resource project name') + parser.add_argument('--algo_version', type=str, default=None, help='algo version') parser.add_argument( '--is_outer', type=int, default=1, - help= - 'is outer pai or inner pai, the arguments are differed slightly due to history reasons' + help='is outer pai or inner pai, the arguments are differed slightly due to history reasons', ) args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] @@ -246,9 +225,7 @@ def test_vector_retrieve(self): if args.algo_version: odps_oss_config.algo_version = args.algo_version algo_names = ['easy_rec_ext15', 'easy_rec_ext'] - assert args.algo_name in algo_names, 'algo_name must be oneof: %s' % ( - ','.join(algo_names) - ) + assert args.algo_name in algo_names, 'algo_name must be oneof: %s' % (','.join(algo_names)) odps_oss_config.algo_name = args.algo_name if args.arn: odps_oss_config.arn = args.arn @@ -259,8 +236,10 @@ def test_vector_retrieve(self): prepare(odps_oss_config) tf.test.main() bucket = get_oss_bucket( - odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) shutil.rmtree(odps_oss_config.temp_dir) diff --git a/easy_rec/python/test/odps_test_prepare.py b/easy_rec/python/test/odps_test_prepare.py index a52ee0983..c2c0f5e16 100644 --- a/easy_rec/python/test/odps_test_prepare.py +++ b/easy_rec/python/test/odps_test_prepare.py @@ -44,10 +44,7 @@ def download_data(ali_bucket, script_path): if not os.path.exists(dst_dir): os.makedirs(dst_dir) ali_bucket.get_object_to_file(obj_key, dst_path) - logging.info( - 'down file oss://%s/%s to %s completed' % - (ali_bucket.bucket_name, obj_key, dst_path) - ) + logging.info('down file oss://%s/%s to %s completed' % (ali_bucket.bucket_name, obj_key, dst_path)) def merge_files(merge_dir, merge_out): @@ -108,7 +105,9 @@ def change_files(odps_oss_config, file_path): # tmp_e = tmp_e.replace('.aliyuncs.com', '.oss-internal.aliyun-inc.com') if '-Dbuckets=' in line: line = '-Dbuckets=oss://%s/?role_arn=%s&host=%s\n' % ( - odps_oss_config.bucket_name, odps_oss_config.arn, tmp_e + odps_oss_config.bucket_name, + odps_oss_config.arn, + tmp_e, ) elif '-Darn=' in line or '-DossHost' in line: continue @@ -139,29 +138,28 @@ def put_data_to_bucket(odps_oss_config): odps_oss_config: odps oss config obj """ test_bucket = get_oss_bucket( - odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) for sub_dir in ['configs']: - for root, dirs, files in os.walk( - os.path.join(odps_oss_config.temp_dir, sub_dir) - ): + for root, dirs, files in os.walk(os.path.join(odps_oss_config.temp_dir, sub_dir)): for one_file in files: file_path = os.path.join(root, one_file) obj_path = file_path.split(sub_dir + '/')[1] dst_path = os.path.join(odps_oss_config.exp_dir, sub_dir, obj_path) test_bucket.put_object_from_file(dst_path, file_path) - logging.info( - 'put %s to oss://%s/%s' % - (file_path, odps_oss_config.bucket_name, dst_path) - ) + logging.info('put %s to oss://%s/%s' % (file_path, odps_oss_config.bucket_name, dst_path)) def prepare(odps_oss_config): logging.info('temp_dir = %s' % odps_oss_config.temp_dir) ali_bucket = get_oss_bucket( - odps_oss_config.ali_oss_key, odps_oss_config.ali_oss_secret, - odps_oss_config.ali_bucket_endpoint, odps_oss_config.ali_bucket_name + odps_oss_config.ali_oss_key, + odps_oss_config.ali_oss_secret, + odps_oss_config.ali_bucket_endpoint, + odps_oss_config.ali_bucket_name, ) shutil.copytree(odps_oss_config.script_path, odps_oss_config.temp_dir) logging.info('start down data') @@ -174,7 +172,7 @@ def prepare(odps_oss_config): file_path = os.path.join(root, file) # drop .template if file_path.endswith('.template'): - tmp_path = file_path[:-len('.template')] + tmp_path = file_path[: -len('.template')] os.rename(file_path, tmp_path) file_path = tmp_path if 'data' not in file_path: @@ -191,9 +189,7 @@ def prepare(odps_oss_config): if __name__ == '__main__': if len(sys.argv) < 5: - print( - 'usage: %s ossutilconfig bucket_name rolearn odpsconfig' % sys.argv[0] - ) + print('usage: %s ossutilconfig bucket_name rolearn odpsconfig' % sys.argv[0]) sys.exit(1) odps_oss_config = OdpsOSSConfig() diff --git a/easy_rec/python/test/odps_test_util.py b/easy_rec/python/test/odps_test_util.py index aac82167e..d61103048 100644 --- a/easy_rec/python/test/odps_test_util.py +++ b/easy_rec/python/test/odps_test_util.py @@ -10,16 +10,18 @@ try: from datahub import DataHub - from datahub.exceptions import InvalidOperationException, ResourceExistException # NOQA + from datahub.exceptions import ( # NOQA + InvalidOperationException, + ResourceExistException, + ) + # from datahub.exceptions import LimitExceededException # from datahub.exceptions import ResourceNotFoundException # from datahub.models import BlobRecord # from datahub.models import CursorType from datahub.models import FieldType, RecordSchema, RecordType, TupleRecord except Exception: - logging.error( - 'DataHub is not installed, please installed it by: pip install pydatahub' - ) + logging.error('DataHub is not installed, please installed it by: pip install pydatahub') DataHub = None try: @@ -31,7 +33,6 @@ class OdpsOSSConfig: - def __init__(self, script_path='./samples/odps_script'): self.time_stamp = int(time.time()) temp_dir = os.environ.get('TMPDIR', '/tmp') @@ -86,11 +87,11 @@ def load_oss_config(self, config_path): line_str = line_str.strip() line_str = line_str.replace(' ', '') if line_str.startswith('accessKeyID='): - self.oss_key = line_str[len('accessKeyID='):].strip() + self.oss_key = line_str[len('accessKeyID=') :].strip() elif line_str.startswith('accessKeySecret='): - self.oss_secret = line_str[len('accessKeySecret='):].strip() + self.oss_secret = line_str[len('accessKeySecret=') :].strip() elif line_str.startswith('endpoint='): - self.endpoint = line_str[len('endpoint='):].strip() + self.endpoint = line_str[len('endpoint=') :].strip() def load_odps_config(self, config_path): self.odps_config_path = config_path @@ -100,16 +101,16 @@ def load_odps_config(self, config_path): line_str = line_str.replace(' ', '') key_str = 'project_name=' if line_str.startswith(key_str): - self.project_name = line_str[len(key_str):] + self.project_name = line_str[len(key_str) :] key_str = 'end_point=' if line_str.startswith(key_str): - self.odps_endpoint = line_str[len(key_str):] + self.odps_endpoint = line_str[len(key_str) :] key_str = 'access_id=' if line_str.startswith(key_str): - self.dh_id = line_str[len(key_str):] + self.dh_id = line_str[len(key_str) :] key_str = 'access_key=' if line_str.startswith(key_str): - self.dh_key = line_str[len(key_str):] + self.dh_key = line_str[len(key_str) :] def clean_topic(self, dh_project): if not dh_project: @@ -130,9 +131,7 @@ def clean_project(self): pass def clean_subscription(self, topic_name): - subscriptions = self.dh.list_subscription( - self.dh_project, topic_name, '', 1, 100 - ).subscriptions + subscriptions = self.dh.list_subscription(self.dh_project, topic_name, '', 1, 100).subscriptions for subscription in subscriptions: self.dh.delete_subscription(self.dh_project, topic_name, subscription) @@ -143,16 +142,14 @@ def get_input_type(self, input_type): 'STRING': FieldType.STRING, 'BOOLEAN': FieldType.BOOLEAN, 'FLOAT32': FieldType.DOUBLE, - 'FLOAT64': FieldType.DOUBLE + 'FLOAT64': FieldType.DOUBLE, } return DhDict.get(input_type) def init_dh_and_odps(self): self.dh = DataHub(self.dh_id, self.dh_key, self.dh_endpoint) - self.odps = ODPS( - self.dh_id, self.dh_key, self.project_name, self.odps_endpoint - ) + self.odps = ODPS(self.dh_id, self.dh_key, self.project_name, self.odps_endpoint) self.odpsTable = 'deepfm_train_%s' % self.time_stamp self.clean_project() read_odps = DataFrame(self.odps.get_table(self.odpsTable)) @@ -174,7 +171,7 @@ def init_dh_and_odps(self): 7, 3, record_schema, - comment='EasyRecTest' + comment='EasyRecTest', ) logging.info('create tuple topic %s success!' % self.dh_topic) except ResourceExistException: @@ -184,9 +181,7 @@ def init_dh_and_odps(self): logging.error(traceback.format_exc()) try: self.dh.wait_shards_ready(self.dh_project, self.dh_topic) - logging.info( - 'datahub[%s,%s] shards all ready' % (self.dh_project, self.dh_topic) - ) + logging.info('datahub[%s,%s] shards all ready' % (self.dh_project, self.dh_topic)) topic_result = self.dh.get_topic(self.dh_project, self.dh_topic) if topic_result.record_type != RecordType.TUPLE: logging.error('invalid topic type: %s' % str(topic_result.record_type)) diff --git a/easy_rec/python/test/pre_check_test.py b/easy_rec/python/test/pre_check_test.py index 3d9e56bfb..64c894b07 100644 --- a/easy_rec/python/test/pre_check_test.py +++ b/easy_rec/python/test/pre_check_test.py @@ -13,7 +13,6 @@ class CheckTest(tf.test.TestCase): - def setUp(self): self._test_dir = test_utils.get_tmp_dir() self._success = True @@ -29,7 +28,7 @@ def test_csv_input_train_with_check(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_taobao.config', self._test_dir, - check_mode=True + check_mode=True, ) self.assertTrue(self._success) @@ -40,15 +39,11 @@ def test_rtp_input_train_with_check(self): self.assertTrue(self._success) def test_csv_input_with_pre_check(self): - self._success = test_utils.test_single_pre_check( - 'samples/model_config/dbmtl_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_pre_check('samples/model_config/dbmtl_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_rtp_input_with_pre_check(self): - self._success = test_utils.test_single_pre_check( - 'samples/model_config/dbmtl_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_pre_check('samples/model_config/dbmtl_on_taobao.config', self._test_dir) self.assertTrue(self._success) diff --git a/easy_rec/python/test/predictor_test.py b/easy_rec/python/test/predictor_test.py index 66710569a..368d222fb 100644 --- a/easy_rec/python/test/predictor_test.py +++ b/easy_rec/python/test/predictor_test.py @@ -16,7 +16,6 @@ class PredictorTest(tf.test.TestCase): - def setUp(self): self.gpus = test_utils.get_available_gpus() self.assertTrue(len(self.gpus) > 0, 'no available gpu on this machine') @@ -55,10 +54,24 @@ def test_lookup_pred(self): def test_pred_dict(self): predictor = Predictor('data/test/inference/tb_multitower_export/') field_keys = [ - 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', - 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', 'age_level', - 'pvalue_level', 'shopping_level', 'occupation', 'new_user_class_level', - 'tag_category_list', 'tag_brand_list', 'price' + 'pid', + 'adgroup_id', + 'cate_id', + 'campaign_id', + 'customer', + 'brand', + 'user_id', + 'cms_segid', + 'cms_group_id', + 'final_gender_code', + 'age_level', + 'pvalue_level', + 'shopping_level', + 'occupation', + 'new_user_class_level', + 'tag_category_list', + 'tag_brand_list', + 'price', ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -70,14 +83,26 @@ def test_pred_dict(self): @RunAsSubprocess def test_pred_placeholder_named_by_input(self): - predictor = Predictor( - 'data/test/inference/tb_multitower_placeholder_rename_export/' - ) + predictor = Predictor('data/test/inference/tb_multitower_placeholder_rename_export/') field_keys = [ - 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', - 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', 'age_level', - 'pvalue_level', 'shopping_level', 'occupation', 'new_user_class_level', - 'tag_category_list', 'tag_brand_list', 'price' + 'pid', + 'adgroup_id', + 'cate_id', + 'campaign_id', + 'customer', + 'brand', + 'user_id', + 'cms_segid', + 'cms_group_id', + 'final_gender_code', + 'age_level', + 'pvalue_level', + 'shopping_level', + 'occupation', + 'new_user_class_level', + 'tag_category_list', + 'tag_brand_list', + 'price', ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -108,10 +133,24 @@ def test_fm_pred_list(self): def test_fm_pred_dict(self): predictor = Predictor('data/test/inference/fm_export/') field_keys = [ - 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', - 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', 'age_level', - 'pvalue_level', 'shopping_level', 'occupation', 'new_user_class_level', - 'tag_category_list', 'tag_brand_list', 'price' + 'pid', + 'adgroup_id', + 'cate_id', + 'campaign_id', + 'customer', + 'brand', + 'user_id', + 'cms_segid', + 'cms_group_id', + 'final_gender_code', + 'age_level', + 'pvalue_level', + 'shopping_level', + 'occupation', + 'new_user_class_level', + 'tag_category_list', + 'tag_brand_list', + 'price', ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -123,9 +162,7 @@ def test_fm_pred_dict(self): class PredictorTestOnDS(tf.test.TestCase): - def setUp(self): - self._test_dir = test_utils.get_tmp_dir() self._test_output_path = None logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) @@ -138,21 +175,15 @@ def tearDown(self): @RunAsSubprocess def test_local_pred(self): test_input_path = 'data/test/inference/taobao_infer_data.txt' - self._test_output_path = os.path.join( - self._test_dir, 'taobao_infer_result' - ) + self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') saved_model_dir = 'data/test/inference/tb_multitower_export/' - pipeline_config_path = os.path.join( - saved_model_dir, 'assets/pipeline.config' - ) - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False - ) + pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) predictor = CSVPredictor( saved_model_dir, pipeline_config.data_config, output_sep=';', - selected_cols='' + selected_cols='', ) predictor.predict_impl( @@ -161,11 +192,13 @@ def test_local_pred(self): reserved_cols='ALL_COLUMNS', output_cols='ALL_COLUMNS', slice_id=0, - slice_num=1 + slice_num=1, + ) + header_truth = ( + 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;' + 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' + 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' ) - header_truth = 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;'\ - 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' \ - 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() @@ -175,16 +208,10 @@ def test_local_pred(self): @RunAsSubprocess def test_local_pred_with_header(self): test_input_path = 'data/test/inference/taobao_infer_data_with_header.txt' - self._test_output_path = os.path.join( - self._test_dir, 'taobao_infer_result' - ) + self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') saved_model_dir = 'data/test/inference/tb_multitower_export/' - pipeline_config_path = os.path.join( - saved_model_dir, 'assets/pipeline.config' - ) - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False - ) + pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) pipeline_config.data_config.with_header = True predictor = CSVPredictor( @@ -192,7 +219,7 @@ def test_local_pred_with_header(self): pipeline_config.data_config, with_header=True, output_sep=';', - selected_cols='' + selected_cols='', ) predictor.predict_impl( @@ -201,11 +228,13 @@ def test_local_pred_with_header(self): reserved_cols='ALL_COLUMNS', output_cols='ALL_COLUMNS', slice_id=0, - slice_num=1 + slice_num=1, + ) + header_truth = ( + 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;' + 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' + 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' ) - header_truth = 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;'\ - 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' \ - 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() @@ -215,9 +244,7 @@ def test_local_pred_with_header(self): @RunAsSubprocess def test_local_pred_without_config(self): test_input_path = 'data/test/inference/taobao_infer_data.txt' - self._test_output_path = os.path.join( - self._test_dir, 'taobao_infer_result' - ) + self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') saved_model_dir = 'data/test/inference/tb_multitower_export/' self._success = test_utils.test_single_predict( self._test_dir, test_input_path, self._test_output_path, saved_model_dir @@ -230,22 +257,16 @@ def test_local_pred_without_config(self): @RunAsSubprocess def test_local_pred_with_part_col(self): test_input_path = 'data/test/inference/taobao_infer_data.txt' - self._test_output_path = os.path.join( - self._test_dir, 'taobao_infer_result' - ) + self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') saved_model_dir = 'data/test/inference/tb_multitower_export/' - pipeline_config_path = os.path.join( - saved_model_dir, 'assets/pipeline.config' - ) - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False - ) + pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) predictor = CSVPredictor( saved_model_dir, pipeline_config.data_config, output_sep=';', - selected_cols='' + selected_cols='', ) predictor.predict_impl( @@ -254,7 +275,7 @@ def test_local_pred_with_part_col(self): reserved_cols='clk,buy,user_id,adgroup_id', output_cols='probs', slice_id=0, - slice_num=1 + slice_num=1, ) header_truth = 'probs;clk;buy;user_id;adgroup_id' @@ -266,22 +287,16 @@ def test_local_pred_with_part_col(self): @RunAsSubprocess def test_local_pred_rtp(self): test_input_path = 'data/test/inference/taobao_infer_rtp_data.txt' - self._test_output_path = os.path.join( - self._test_dir, 'taobao_test_feature_result' - ) + self._test_output_path = os.path.join(self._test_dir, 'taobao_test_feature_result') saved_model_dir = 'data/test/inference/tb_multitower_rtp_export/' - pipeline_config_path = os.path.join( - saved_model_dir, 'assets/pipeline.config' - ) - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False - ) + pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) predictor = CSVPredictor( saved_model_dir, pipeline_config.data_config, output_sep=';', - selected_cols='0,3' + selected_cols='0,3', ) predictor.predict_impl( test_input_path, @@ -289,7 +304,7 @@ def test_local_pred_rtp(self): reserved_cols='ALL_COLUMNS', output_cols='ALL_COLUMNS', slice_id=0, - slice_num=1 + slice_num=1, ) header_truth = 'logits;probs;clk;no_used_1;no_used_2;features' with open(self._test_output_path + '/part-0.csv', 'r') as f: @@ -300,22 +315,16 @@ def test_local_pred_rtp(self): @RunAsSubprocess def test_local_pred_rtp_with_part_col(self): test_input_path = 'data/test/inference/taobao_infer_rtp_data.txt' - self._test_output_path = os.path.join( - self._test_dir, 'taobao_test_feature_result' - ) + self._test_output_path = os.path.join(self._test_dir, 'taobao_test_feature_result') saved_model_dir = 'data/test/inference/tb_multitower_rtp_export/' - pipeline_config_path = os.path.join( - saved_model_dir, 'assets/pipeline.config' - ) - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False - ) + pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) predictor = CSVPredictor( saved_model_dir, pipeline_config.data_config, output_sep=';', - selected_cols='0,3' + selected_cols='0,3', ) predictor.predict_impl( test_input_path, @@ -323,7 +332,7 @@ def test_local_pred_rtp_with_part_col(self): reserved_cols='clk,features,no_used_1', output_cols='ALL_COLUMNS', slice_id=0, - slice_num=1 + slice_num=1, ) header_truth = 'logits;probs;clk;features;no_used_1' with open(self._test_output_path + '/part-0.csv', 'r') as f: @@ -334,22 +343,16 @@ def test_local_pred_rtp_with_part_col(self): @RunAsSubprocess def test_local_pred_embedding(self): test_input_path = 'data/test/inference/taobao_item_feature_data.csv' - self._test_output_path = os.path.join( - self._test_dir, 'taobao_item_feature' - ) + self._test_output_path = os.path.join(self._test_dir, 'taobao_item_feature') saved_model_dir = 'data/test/inference/dssm_item_model/' - pipeline_config_path = os.path.join( - saved_model_dir, 'assets/pipeline.config' - ) - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False - ) + pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) predictor = CSVPredictor( saved_model_dir, pipeline_config.data_config, ds_vector_recall=True, output_sep=';', - selected_cols='pid,adgroup_id,cate_id,campaign_id,customer,brand,price' + selected_cols='pid,adgroup_id,cate_id,campaign_id,customer,brand,price', ) predictor.predict_impl( @@ -358,14 +361,13 @@ def test_local_pred_embedding(self): reserved_cols='adgroup_id', output_cols='item_emb', slice_id=0, - slice_num=1 + slice_num=1, ) with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() self.assertTrue( - output_res[1] == - '-0.187066,-0.027638,-0.117294,0.115318,-0.273561,0.035698,-0.055832,' + output_res[1] == '-0.187066,-0.027638,-0.117294,0.115318,-0.273561,0.035698,-0.055832,' '0.226849,-0.105808,-0.152751,0.081528,-0.183329,0.134619,0.185392,' '0.096774,0.104428,0.161868,0.269710,-0.268538,0.138760,-0.170105,' '0.232625,-0.121130,0.198466,-0.078941,0.017774,0.268834,-0.238553,0.084058,' @@ -374,7 +376,6 @@ def test_local_pred_embedding(self): class PredictorTestV2(tf.test.TestCase): - def setUp(self): self.gpus = test_utils.get_available_gpus() self.assertTrue(len(self.gpus) > 0, 'no available gpu on this machine') @@ -403,9 +404,7 @@ def test_pred_multi(self): for line_id, line_str in enumerate(fin): line_str = line_str.strip() line_pred = json.loads(line_str) - self.assertTrue( - np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-6 - ) + self.assertTrue(np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-6) @RunAsSubprocess def test_pred_single(self): @@ -423,9 +422,7 @@ def test_pred_single(self): for line_id, line_str in enumerate(fin): line_str = line_str.strip() line_pred = json.loads(line_str) - self.assertTrue( - np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-5 - ) + self.assertTrue(np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-5) if __name__ == '__main__': diff --git a/easy_rec/python/test/rtp_convert_test.py b/easy_rec/python/test/rtp_convert_test.py index f368181ce..9c83a791f 100644 --- a/easy_rec/python/test/rtp_convert_test.py +++ b/easy_rec/python/test/rtp_convert_test.py @@ -10,7 +10,6 @@ class RTPConvertTest(tf.test.TestCase): - def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) @@ -21,7 +20,8 @@ def test_rtp_convert(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_multi_tower.config') - convert_cmd = """ + convert_cmd = ( + """ python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg.json --label clk @@ -31,24 +31,21 @@ def test_rtp_convert(self): --train_input_path data/test/rtp/taobao_train_feature.txt --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 - """ % pipeline_config_path - proc = test_utils.run_cmd( - convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + """ + % pipeline_config_path ) + proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir - ) - ) + self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) def test_rtp_convert_bucketize(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_multi_tower.config') - convert_cmd = """ + convert_cmd = ( + """ python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg_bucketize.json --label clk @@ -58,24 +55,21 @@ def test_rtp_convert_bucketize(self): --train_input_path data/test/rtp/taobao_train_bucketize_feature.txt --eval_input_path data/test/rtp/taobao_test_bucketize_feature.txt --selected_cols 0,3 --num_steps 400 - """ % pipeline_config_path - proc = test_utils.run_cmd( - convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + """ + % pipeline_config_path ) + proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir - ) - ) + self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) def test_rtp_convert_bucketize_v2(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_multi_tower.config') - convert_cmd = """ + convert_cmd = ( + """ python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg_bucketize_v2.json --label clk @@ -85,32 +79,27 @@ def test_rtp_convert_bucketize_v2(self): --train_input_path data/test/rtp/taobao_train_feature.txt --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 - """ % pipeline_config_path - proc = test_utils.run_cmd( - convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + """ + % pipeline_config_path ) + proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - tmp_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path - ) + tmp_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) for feature_config in tmp_config.feature_configs: if feature_config.input_names[0] == 'price': assert len(feature_config.boundaries) == 6 - self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir - ) - ) + self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) def test_rtp_convert_test_model_config(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_wide_and_deep.config') - convert_cmd = """ + convert_cmd = ( + """ python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg_bucketize_model_config.json --label clk @@ -119,26 +108,20 @@ def test_rtp_convert_test_model_config(self): --train_input_path data/test/rtp/taobao_train_feature.txt --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 - """ % pipeline_config_path - proc = test_utils.run_cmd( - convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert') + """ + % pipeline_config_path ) + proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - tmp_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path - ) + tmp_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) assert len(tmp_config.model_config.wide_and_deep.dnn.hidden_units) == 2 assert tmp_config.model_config.wide_and_deep.dnn.hidden_units[0] == 48 assert tmp_config.model_config.wide_and_deep.dnn.hidden_units[1] == 24 assert tmp_config.model_dir == 'experiments/rtp_fg/wide_and_deep_update_model' - self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, test_dir=test_dir - ) - ) + self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) diff --git a/easy_rec/python/test/run.py b/easy_rec/python/test/run.py index e201db9cf..c5ef94d0c 100644 --- a/easy_rec/python/test/run.py +++ b/easy_rec/python/test/run.py @@ -17,22 +17,14 @@ tf.app.flags.DEFINE_bool('list_tests', False, 'list all tests') tf.app.flags.DEFINE_string('list_test_to_file', None, 'list all tests') tf.app.flags.DEFINE_string('pattern', '*_test.py', 'test file pattern') -tf.app.flags.DEFINE_string( - 'test_dir', 'easy_rec/python/test', 'directory to be tested' -) -tf.app.flags.DEFINE_integer( - 'num_parallel', 10, 'number of parallel executed cases.' -) -tf.app.flags.DEFINE_integer( - 'timeout', 3600, 'maximal execute time in seconds for each case.' -) +tf.app.flags.DEFINE_string('test_dir', 'easy_rec/python/test', 'directory to be tested') +tf.app.flags.DEFINE_integer('num_parallel', 10, 'number of parallel executed cases.') +tf.app.flags.DEFINE_integer('timeout', 3600, 'maximal execute time in seconds for each case.') FLAGS = tf.flags.FLAGS def gather_test_cases(test_dir, pattern): - discover = unittest.defaultTestLoader.discover( - test_dir, pattern=pattern, top_level_dir=None - ) + discover = unittest.defaultTestLoader.discover(test_dir, pattern=pattern, top_level_dir=None) all_tests = [] for suite_discovered in discover: for test_case in suite_discovered: @@ -80,9 +72,7 @@ def main(argv): test_log_dir = os.path.join(test_dir, 'logs') if not os.path.exists(test_log_dir): os.makedirs(test_log_dir) - logging.info( - 'Total number of cases: %d test_dir: %s' % (len(all_tests), test_dir) - ) + logging.info('Total number of cases: %d test_dir: %s' % (len(all_tests), test_dir)) max_num_port_per_proc = 3 total_port_num = (max_num_port_per_proc + 2) * FLAGS.num_parallel * 10 @@ -105,9 +95,7 @@ def main(argv): del procs[proc] cmd = 'python -m easy_rec.python.test.%s %s' % (case_file, case_name) log_file = '%s/%s.%s.log' % (test_log_dir, case_file, case_name) - tmp_ports = ','.join( - [str(x) for x in all_available_ports[:max_num_port_per_proc]] - ) + tmp_ports = ','.join([str(x) for x in all_available_ports[:max_num_port_per_proc]]) all_available_ports = all_available_ports[max_num_port_per_proc:] logging.info('Run %s.%s Log: %s' % (case_file, case_name, log_file)) @@ -118,14 +106,10 @@ def main(argv): for proc in procs: try: - test_utils.proc_wait( - proc, timeout=int(os.environ.get('TEST_TIME_OUT', 1200)) - ) + test_utils.proc_wait(proc, timeout=int(os.environ.get('TEST_TIME_OUT', 1200))) except Exception as ex: fail_file, fail_name = procs[proc] - logging.info( - 'Case Exception: %s.%s %s' % (fail_file, fail_name, str(ex)) - ) + logging.info('Case Exception: %s.%s %s' % (fail_file, fail_name, str(ex))) proc.kill() if proc.returncode != 0: @@ -136,8 +120,7 @@ def main(argv): logging.info('Number Cases Failed: %d' % len(failed_cases)) for fail_file, fail_name, exit_code in failed_cases: logging.info( - '\t%s.%s failed, exit_code:%d log: %s.%s.log' % - (fail_file, fail_name, exit_code, fail_file, fail_name) + '\t%s.%s failed, exit_code:%d log: %s.%s.log' % (fail_file, fail_name, exit_code, fail_file, fail_name) ) return 1 else: diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 6641673d0..792f77b7c 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -7,16 +7,20 @@ import threading import time import unittest +from distutils.version import LooseVersion import numpy as np import six import tensorflow as tf -from distutils.version import LooseVersion from tensorflow.python.platform import gfile from easy_rec.python.main import predict - -from easy_rec.python.utils import config_util, constant, estimator_utils, test_utils # NOQA +from easy_rec.python.utils import ( # NOQA + config_util, + constant, + estimator_utils, + test_utils, +) try: import graphlearn as gl @@ -39,7 +43,6 @@ class TrainEvalTest(tf.test.TestCase): - def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) self._test_dir = test_utils.get_tmp_dir() @@ -52,9 +55,7 @@ def tearDown(self): test_utils.clean_up(self._test_dir) def test_deepfm_with_lookup_feature(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_lookup.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/deepfm_lookup.config', self._test_dir) self.assertTrue(self._success) def test_deepfm_with_combo_feature(self): @@ -65,15 +66,13 @@ def test_deepfm_with_combo_feature(self): def test_deepfm_with_combo_v2_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_v2_on_avazu_ctr.config', - self._test_dir + 'samples/model_config/deepfm_combo_v2_on_avazu_ctr.config', self._test_dir ) self.assertTrue(self._success) def test_deepfm_with_combo_v3_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_v3_on_avazu_ctr.config', - self._test_dir + 'samples/model_config/deepfm_combo_v3_on_avazu_ctr.config', self._test_dir ) self.assertTrue(self._success) @@ -85,22 +84,20 @@ def test_deepfm_freeze_gradient(self): def test_deepfm_with_vocab_list(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_vocab_list_on_avazu_ctr.config', - self._test_dir + 'samples/model_config/deepfm_vocab_list_on_avazu_ctr.config', self._test_dir ) self.assertTrue(self._success) def test_deepfm_with_multi_class(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - self._test_dir + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', self._test_dir ) self.assertTrue(self._success) def test_wide_and_deep_no_final(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/wide_and_deep_no_final_on_avazau_ctr.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -113,14 +110,12 @@ def test_wide_and_deep(self): def test_wide_and_deep_backbone(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/wide_and_deep_backbone_on_avazau.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_dlrm(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/dlrm_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/dlrm_on_taobao.config', self._test_dir) def test_dlrm_backbone(self): self._success = test_utils.test_single_train_eval( @@ -130,14 +125,14 @@ def test_dlrm_backbone(self): def test_adamw_optimizer(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_adamw_ctr.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_momentumw_optimizer(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_momentumw_ctr.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -146,8 +141,7 @@ def test_deepfm_with_param_edit(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', self._test_dir, - hyperparam_str='{"model_dir":"%s", ' - '"model_config.deepfm.wide_output_dim": 32}' % model_dir + hyperparam_str='{"model_dir":"%s", ' '"model_config.deepfm.wide_output_dim": 32}' % model_dir, ) self.assertTrue(self._success) config_path = os.path.join(model_dir, 'pipeline.config') @@ -163,8 +157,7 @@ def test_multi_tower(self): def test_multi_tower_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_backbone_on_taobao.config', - self._test_dir + 'samples/model_config/multi_tower_backbone_on_taobao.config', self._test_dir ) self.assertTrue(self._success) @@ -177,7 +170,7 @@ def test_multi_tower_gauc(self): def test_multi_tower_session_auc(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_on_taobao_session_auc.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -185,7 +178,7 @@ def test_multi_tower_save_checkpoint_secs(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_save_secs_on_taobao.config', self._test_dir, - total_steps=100 + total_steps=100, ) ckpts_times = [] ckpt_dir = os.path.join(self._test_dir, 'train') @@ -197,18 +190,12 @@ def test_multi_tower_save_checkpoint_secs(self): # ensure interval is 20s diffs = list(ckpts_times[1:] - ckpts_times[:-1]) logging.info('nearby ckpts_times diff = %s' % diffs) - self.assertAllClose( - ckpts_times[1:] - ckpts_times[:-1], [20] * (len(ckpts_times) - 1), - atol=20 - ) + self.assertAllClose(ckpts_times[1:] - ckpts_times[:-1], [20] * (len(ckpts_times) - 1), atol=20) self.assertTrue(self._success) def test_keep_ckpt_max(self): - def _post_check_func(pipeline_config): - ckpt_prefix = os.path.join( - pipeline_config.model_dir, 'model.ckpt-*.meta' - ) + ckpt_prefix = os.path.join(pipeline_config.model_dir, 'model.ckpt-*.meta') ckpts = gfile.Glob(ckpt_prefix) assert len(ckpts) == 3, 'invalid number of checkpoints: %d' % len(ckpts) @@ -216,11 +203,10 @@ def _post_check_func(pipeline_config): 'samples/model_config/multi_tower_ckpt_keep_3_on_taobao.config', self._test_dir, total_steps=500, - post_check_func=_post_check_func + post_check_func=_post_check_func, ) def test_multi_tower_with_best_exporter(self): - def _post_check_func(pipeline_config): model_dir = pipeline_config.model_dir best_ckpts = os.path.join(model_dir, 'best_ckpt/model.ckpt-*.meta') @@ -228,8 +214,7 @@ def _post_check_func(pipeline_config): assert len(best_ckpts) <= 2, 'too many best ckpts: %s' % str(best_ckpts) best_exports = os.path.join(model_dir, 'export/best/*') best_exports = gfile.Glob(best_exports) - assert len(best_exports - ) <= 2, 'too many best exports: %s' % str(best_exports) + assert len(best_exports) <= 2, 'too many best exports: %s' % str(best_exports) return True self._success = test_utils.test_single_train_eval( @@ -237,7 +222,7 @@ def _post_check_func(pipeline_config): self._test_dir, total_steps=800, post_check_func=_post_check_func, - timeout=3000 + timeout=3000, ) self.assertTrue(self._success) @@ -248,20 +233,17 @@ def test_latest_ckpt(self): assert tmp.endswith('model.ckpt-500') def test_latest_ckpt_v2(self): - def _post_check_func(pipeline_config): logging.info('model_dir: %s' % pipeline_config.model_dir) - logging.info( - 'latest_checkpoint: %s' % - estimator_utils.latest_checkpoint(pipeline_config.model_dir) + logging.info('latest_checkpoint: %s' % estimator_utils.latest_checkpoint(pipeline_config.model_dir)) + return tf.train.latest_checkpoint(pipeline_config.model_dir) == estimator_utils.latest_checkpoint( + pipeline_config.model_dir ) - return tf.train.latest_checkpoint(pipeline_config.model_dir) == \ - estimator_utils.latest_checkpoint(pipeline_config.model_dir) self._success = test_utils.test_single_train_eval( 'samples/model_config/taobao_fg.config', self._test_dir, - post_check_func=_post_check_func + post_check_func=_post_check_func, ) self.assertTrue(self._success) @@ -286,7 +268,7 @@ def _watch_func(): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/taobao_fg_signal_stop.config', self._test_dir, - total_steps=1000 + total_steps=1000, ) self.assertTrue(self._success) watch_th.join() @@ -301,7 +283,7 @@ def test_dead_line_stop_signal(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/dead_line_stop.config', self._test_dir, - total_steps=1000 + total_steps=1000, ) self.assertTrue(self._success) final_ckpt = estimator_utils.latest_checkpoint(train_dir) @@ -311,32 +293,25 @@ def test_dead_line_stop_signal(self): assert ckpt_version < 1000 def test_fine_tune_latest_ckpt_path(self): - def _post_check_func(pipeline_config): logging.info('model_dir: %s' % pipeline_config.model_dir) pipeline_config = config_util.get_configs_from_pipeline_file( os.path.join(pipeline_config.model_dir, 'pipeline.config'), False ) - logging.info( - 'fine_tune_checkpoint: %s' % - pipeline_config.train_config.fine_tune_checkpoint - ) - return pipeline_config.train_config.fine_tune_checkpoint == \ - 'data/test/mt_ckpt/model.ckpt-100' + logging.info('fine_tune_checkpoint: %s' % pipeline_config.train_config.fine_tune_checkpoint) + return pipeline_config.train_config.fine_tune_checkpoint == 'data/test/mt_ckpt/model.ckpt-100' self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_on_taobao.config', self._test_dir, fine_tune_checkpoint='data/test/mt_ckpt', - post_check_func=_post_check_func + post_check_func=_post_check_func, ) self.assertTrue(self._success) def test_fine_tune_ckpt(self): - def _post_check_func(pipeline_config): - pipeline_config.train_config.fine_tune_checkpoint = \ - estimator_utils.latest_checkpoint(pipeline_config.model_dir) + pipeline_config.train_config.fine_tune_checkpoint = estimator_utils.latest_checkpoint(pipeline_config.model_dir) test_dir = os.path.join(self._test_dir, 'fine_tune') pipeline_config.model_dir = os.path.join(test_dir, 'ckpt') return test_utils.test_single_train_eval(pipeline_config, test_dir) @@ -344,46 +319,36 @@ def _post_check_func(pipeline_config): self._success = test_utils.test_single_train_eval( 'samples/model_config/taobao_fg.config', self._test_dir, - post_check_func=_post_check_func + post_check_func=_post_check_func, ) self.assertTrue(self._success) def test_multi_tower_multi_value_export(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_multi_value_export_on_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_multi_tower_fg_input(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/taobao_fg.config', self._test_dir) self.assertTrue(self._success) def test_multi_tower_fg_json_config(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.json', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/taobao_fg.json', self._test_dir) self.assertTrue(self._success) def test_fm(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/fm_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/fm_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_place_embed_on_cpu(self): os.environ['place_embedding_on_cpu'] = 'True' - self._success = test_utils.test_single_train_eval( - 'samples/model_config/fm_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/fm_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_din(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/din_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/din_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_din_backbone(self): @@ -393,9 +358,7 @@ def test_din_backbone(self): self.assertTrue(self._success) def test_bst(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/bst_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/bst_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_bst_backbone(self): @@ -405,34 +368,25 @@ def test_bst_backbone(self): self.assertTrue(self._success) def test_cl4srec(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/cl4srec_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/cl4srec_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dcn(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/dcn_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/dcn_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_ziln_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mlp_on_taobao_with_ziln_loss.config', - self._test_dir + 'samples/model_config/mlp_on_taobao_with_ziln_loss.config', self._test_dir ) self.assertTrue(self._success) def test_fibinet(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/fibinet_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/fibinet_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_masknet(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/masknet_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/masknet_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dcn_backbone(self): @@ -442,21 +396,15 @@ def test_dcn_backbone(self): self.assertTrue(self._success) def test_dcn_with_f1(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/dcn_f1_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/dcn_f1_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_autoint(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/autoint_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/autoint_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_uniter(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/uniter_on_movielens.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/uniter_on_movielens.config', self._test_dir) self.assertTrue(self._success) def test_highway(self): @@ -475,35 +423,29 @@ def test_highway(self): # self.assertTrue(self._success) def test_cdn(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/cdn_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/cdn_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_ppnet(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/ppnet_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/ppnet_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_uniter_only_text_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/uniter_on_movielens_only_text_feature.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_uniter_only_image_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/uniter_on_movielens_only_image_feature.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_cmbf(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/cmbf_on_movielens.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/cmbf_on_movielens.config', self._test_dir) self.assertTrue(self._success) def test_cmbf_with_multi_loss(self): @@ -515,28 +457,26 @@ def test_cmbf_with_multi_loss(self): def test_cmbf_has_other_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/cmbf_on_movielens_has_other_feature.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_cmbf_only_text_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/cmbf_on_movielens_only_text_feature.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_cmbf_only_image_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/cmbf_on_movielens_only_image_feature.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_dssm(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/dssm_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dropoutnet(self): @@ -561,8 +501,7 @@ def test_dssm_neg_sampler(self): @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler_v2(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_neg_sampler_v2_on_taobao.config', - self._test_dir + 'samples/model_config/dssm_neg_sampler_v2_on_taobao.config', self._test_dir ) self.assertTrue(self._success) @@ -570,7 +509,7 @@ def test_dssm_neg_sampler_v2(self): def test_dssm_hard_neg_sampler(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_hard_neg_sampler_on_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -578,7 +517,7 @@ def test_dssm_hard_neg_sampler(self): def test_dssm_hard_neg_regular_sampler(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_hard_neg_sampler_regular_on_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -586,7 +525,7 @@ def test_dssm_hard_neg_regular_sampler(self): def test_dssm_hard_neg_sampler_v2(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_hard_neg_sampler_v2_on_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -612,9 +551,7 @@ def _test_kd(self, config0, config1): @test_utils.RunAsSubprocess def _gen_kd_data(train_path, eval_path): - pred_result = predict( - config_path, None, pipeline_config.train_input_path - ) + pred_result = predict(config_path, None, pipeline_config.train_input_path) with gfile.GFile(pipeline_config.train_input_path, 'r') as fin: with gfile.GFile(train_path, 'w') as fout: for line, pred in zip(fin, pred_result): @@ -637,31 +574,27 @@ def _gen_kd_data(train_path, eval_path): pipeline_config = config_util.get_configs_from_pipeline_file(config1) pipeline_config.train_input_path = train_path pipeline_config.eval_input_path = eval_path - config_util.save_pipeline_config( - pipeline_config, self._test_dir, 'kd_pipeline.config' - ) + config_util.save_pipeline_config(pipeline_config, self._test_dir, 'kd_pipeline.config') self._success = test_utils.test_single_train_eval( os.path.join(self._test_dir, 'kd_pipeline.config'), - os.path.join(self._test_dir, 'kd') + os.path.join(self._test_dir, 'kd'), ) self.assertTrue(self._success) def test_dssm_with_kd(self): self._test_kd( 'samples/model_config/multi_tower_on_taobao.config', - 'samples/model_config/dssm_kd_on_taobao.config' + 'samples/model_config/dssm_kd_on_taobao.config', ) def test_deepfm_multi_class_with_kd(self): self._test_kd( 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - 'samples/model_config/deepfm_multi_cls_small.config' + 'samples/model_config/deepfm_multi_cls_small.config', ) def test_mind(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/mind_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/mind_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_mind_with_time_id(self): @@ -679,35 +612,33 @@ def test_deepfm_with_regression(self): def test_deepfm_with_sigmoid_l2_loss(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_sigmoid_l2.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_deepfm_with_embedding_learning_rate(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_emblr_ctr.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_deepfm_with_eval_online(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_eval_online_ctr.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_deepfm_with_eval_online_gauc(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_eval_online_gauc_ctr.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_mmoe(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/mmoe_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/mmoe_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_mmoe_backbone(self): @@ -718,8 +649,7 @@ def test_mmoe_backbone(self): def test_mmoe_with_multi_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mmoe_on_taobao_with_multi_loss.config', - self._test_dir + 'samples/model_config/mmoe_on_taobao_with_multi_loss.config', self._test_dir ) self.assertTrue(self._success) @@ -738,32 +668,24 @@ def test_simple_multi_task(self): def test_simple_multi_task_backbone(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/simple_multi_task_backbone_on_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_esmm(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/esmm_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/esmm_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_tag_kv_input(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/kv_tag.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/kv_tag.config', self._test_dir) self.assertTrue(self._success) def test_aitm(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/aitm_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/aitm_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dbmtl(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/dbmtl_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dbmtl_backbone(self): @@ -787,14 +709,14 @@ def test_dbmtl_uniter(self): def test_dbmtl_with_multi_loss(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_taobao_with_multi_loss.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_early_stop(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_early_stop_on_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -807,7 +729,7 @@ def test_early_stop_custom(self): def test_early_stop_dis(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_early_stop_on_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -818,21 +740,18 @@ def test_latest_export_with_asset(self): self.assertTrue(self._success) def test_incompatible_restore(self): - def _post_check_func(config): config.feature_config.features[0].hash_bucket_size += 20000 config.feature_config.features[1].hash_bucket_size += 100 config.train_config.fine_tune_checkpoint = config.model_dir config.model_dir += '_finetune' config.train_config.force_restore_shape_compatible = True - return test_utils.test_single_train_eval( - config, os.path.join(self._test_dir, 'finetune') - ) + return test_utils.test_single_train_eval(config, os.path.join(self._test_dir, 'finetune')) self._success = test_utils.test_single_train_eval( 'samples/model_config/taobao_fg.config', self._test_dir, - post_check_func=_post_check_func + post_check_func=_post_check_func, ) self.assertTrue(self._success) @@ -840,7 +759,7 @@ def test_dbmtl_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_variational_dropout.config', self._test_dir, - post_check_func=test_utils.test_feature_selection + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) @@ -848,7 +767,7 @@ def test_dbmtl_variational_dropout_feature_num(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_variational_dropout_feature_num.config', self._test_dir, - post_check_func=test_utils.test_feature_selection + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) @@ -856,7 +775,7 @@ def test_essm_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/esmm_variational_dropout_on_taobao.config', self._test_dir, - post_check_func=test_utils.test_feature_selection + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) @@ -864,7 +783,7 @@ def test_fm_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/fm_variational_dropout_on_taobao.config', self._test_dir, - post_check_func=test_utils.test_feature_selection + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) @@ -872,7 +791,7 @@ def test_deepfm_with_combo_feature_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_variational_dropout_on_avazu_ctr.config', self._test_dir, - post_check_func=test_utils.test_feature_selection + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) @@ -880,7 +799,7 @@ def test_dbmtl_sequence_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_variational_dropout_on_sequence_feature_taobao.config', self._test_dir, - post_check_func=test_utils.test_feature_selection + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) @@ -888,27 +807,24 @@ def test_din_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/din_varitional_dropout_on_taobao.config', self._test_dir, - post_check_func=test_utils.test_feature_selection + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) def test_rocket_launching(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/rocket_launching.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/rocket_launching.config', self._test_dir) self.assertTrue(self._success) def test_rocket_launching_feature_based(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/rocket_launching_feature_based.config', - self._test_dir + 'samples/model_config/rocket_launching_feature_based.config', self._test_dir ) self.assertTrue(self._success) def test_rocket_launching_with_rtp_input(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/rocket_launching_with_rtp_input.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -931,7 +847,7 @@ def test_fit_on_eval(self): self._test_dir, total_steps=10, num_evaluator=1, - fit_on_eval=True + fit_on_eval=True, ) self.assertTrue(self._success) @@ -941,7 +857,7 @@ def test_unbalance_data(self): self._test_dir, total_steps=0, num_epoch=1, - num_evaluator=1 + num_evaluator=1, ) self.assertTrue(self._success) @@ -949,21 +865,18 @@ def test_train_with_ps_worker_with_evaluator(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_on_taobao.config', self._test_dir, - num_evaluator=1 + num_evaluator=1, ) self.assertTrue(self._success) final_export_dir = os.path.join(self._test_dir, 'train/export/final') all_saved_files = glob.glob(final_export_dir + '/*/saved_model.pb') - logging.info( - 'final_export_dir=%s all_saved_files=%s' % - (final_export_dir, ','.join(all_saved_files)) - ) + logging.info('final_export_dir=%s all_saved_files=%s' % (final_export_dir, ','.join(all_saved_files))) self.assertTrue(len(all_saved_files) == 1) def test_train_with_ps_worker_chief_redundant(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_on_taobao_chief_redundant.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -988,21 +901,19 @@ def test_tfrecord_input(self): def test_batch_tfrecord_input(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/deepfm_on_criteo_batch_tfrecord.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_autodis_embedding(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_on_criteo_with_autodis.config', - self._test_dir + 'samples/model_config/deepfm_on_criteo_with_autodis.config', self._test_dir ) self.assertTrue(self._success) def test_periodic_embedding(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_on_criteo_with_periodic.config', - self._test_dir + 'samples/model_config/deepfm_on_criteo_with_periodic.config', self._test_dir ) self.assertTrue(self._success) @@ -1022,29 +933,29 @@ def test_dssm_sample_weight(self): def test_dssm_neg_sampler_with_sample_weight(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_neg_sampler_with_sample_weight.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @unittest.skipIf( LooseVersion(tf.__version__) != LooseVersion('2.3.0'), - 'MultiWorkerMirroredStrategy need tf version == 2.3' + 'MultiWorkerMirroredStrategy need tf version == 2.3', ) def test_train_with_multi_worker_mirror(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_multi_worker_mirrored_strategy_on_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @unittest.skipIf( LooseVersion(tf.__version__) != LooseVersion('2.3.0'), - 'MultiWorkerMirroredStrategy need tf version == 2.3' + 'MultiWorkerMirroredStrategy need tf version == 2.3', ) def test_train_mmoe_with_multi_worker_mirror(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/mmoe_mirrored_strategy_on_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -1056,141 +967,137 @@ def test_fg_dtype(self): @unittest.skipIf(six.PY2, 'Only run in python3') def test_share_not_used(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/share_not_used.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/share_not_used.config', self._test_dir) self.assertTrue(self._success) def test_sequence_autoint(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/autoint_on_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_sequence_dcn(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dcn_on_sequence_feature_taobao.config', - self._test_dir + 'samples/model_config/dcn_on_sequence_feature_taobao.config', self._test_dir ) self.assertTrue(self._success) def test_sequence_dssm(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_on_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_sequence_esmm(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/esmm_on_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_sequence_mmoe(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/mmoe_on_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_sequence_ple(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/ple_on_sequence_feature_taobao.config', - self._test_dir + 'samples/model_config/ple_on_sequence_feature_taobao.config', self._test_dir ) self.assertTrue(self._success) def test_sequence_rocket_launching(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/rocket_launching_on_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_sequence_simple_multi_task(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/simple_multi_task_on_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_sequence_wide_and_deep(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/wide_and_deep_on_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_numeric_boundary_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_numeric_hash_bucket_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_numeric_hash_bucket_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_numeric_raw_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_numeric_raw_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_numeric_num_buckets_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_numeric_num_buckets_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_multi_numeric_boundary_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_boundary_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_multi_numeric_hash_bucket_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_hash_bucket_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_multi_numeric_raw_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_raw_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_multi_numeric_num_buckets_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_num_buckets_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_multi_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_sequence_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -1203,88 +1110,71 @@ def test_multi_optimizer(self): def test_embedding_separate_optimizer(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/deepfm_combo_on_avazu_embed_adagrad.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_expr_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_on_taobao_for_expr.config', - self._test_dir + 'samples/model_config/multi_tower_on_taobao_for_expr.config', self._test_dir ) self.assertTrue(self._success) def test_gzip_data(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/din_on_gzip_data.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/din_on_gzip_data.config', self._test_dir) self.assertTrue(self._success) def test_cmd_config_param(self): - def _post_check_config(pipeline_config): - train_saved_config_path = os.path.join( - self._test_dir, 'train/pipeline.config' - ) - pipeline_config = config_util.get_configs_from_pipeline_file( - train_saved_config_path + train_saved_config_path = os.path.join(self._test_dir, 'train/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file(train_saved_config_path) + assert pipeline_config.model_config.deepfm.wide_output_dim == 8, ( + 'invalid model_config.deepfm.wide_output_dim=%d' % pipeline_config.model_config.deepfm.wide_output_dim ) - assert pipeline_config.model_config.deepfm.wide_output_dim == 8,\ - 'invalid model_config.deepfm.wide_output_dim=%d' % \ - pipeline_config.model_config.deepfm.wide_output_dim self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', self._test_dir, post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim 8' + extra_cmd_args='--model_config.deepfm.wide_output_dim 8', ) def test_cmd_config_param_v2(self): - def _post_check_config(pipeline_config): - train_saved_config_path = os.path.join( - self._test_dir, 'train/pipeline.config' + train_saved_config_path = os.path.join(self._test_dir, 'train/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file(train_saved_config_path) + assert pipeline_config.model_config.deepfm.wide_output_dim == 1, ( + 'invalid model_config.deepfm.wide_output_dim=%d' % pipeline_config.model_config.deepfm.wide_output_dim ) - pipeline_config = config_util.get_configs_from_pipeline_file( - train_saved_config_path - ) - assert pipeline_config.model_config.deepfm.wide_output_dim == 1,\ - 'invalid model_config.deepfm.wide_output_dim=%d' % \ - pipeline_config.model_config.deepfm.wide_output_dim self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', self._test_dir, post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim=1' + extra_cmd_args='--model_config.deepfm.wide_output_dim=1', ) def test_cmd_config_param_v3(self): - def _post_check_config(pipeline_config): - train_saved_config_path = os.path.join( - self._test_dir, 'train/pipeline.config' + train_saved_config_path = os.path.join(self._test_dir, 'train/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file(train_saved_config_path) + assert pipeline_config.model_config.deepfm.wide_output_dim == 3, ( + 'invalid model_config.deepfm.wide_output_dim=%d' % pipeline_config.model_config.deepfm.wide_output_dim ) - pipeline_config = config_util.get_configs_from_pipeline_file( - train_saved_config_path - ) - assert pipeline_config.model_config.deepfm.wide_output_dim == 3,\ - 'invalid model_config.deepfm.wide_output_dim=%d' % \ - pipeline_config.model_config.deepfm.wide_output_dim self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', self._test_dir, post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim="3"' + extra_cmd_args='--model_config.deepfm.wide_output_dim="3"', ) def test_distribute_eval_deepfm_multi_cls(self): cur_eval_path = 'data/test/distribute_eval_test/deepfm_distribute_eval_dwd_avazu_out_multi_cls' self._success = test_utils.test_distributed_eval( 'samples/model_config/deepfm_distribute_eval_multi_cls_on_avazu_ctr.config', - cur_eval_path, self._test_dir + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) @@ -1292,7 +1182,8 @@ def test_distribute_eval_deepfm_single_cls(self): cur_eval_path = 'data/test/distribute_eval_test/dwd_distribute_eval_avazu_out_test_combo' self._success = test_utils.test_distributed_eval( 'samples/model_config/deepfm_distribute_eval_combo_on_avazu_ctr.config', - cur_eval_path, self._test_dir + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) @@ -1300,7 +1191,8 @@ def test_distribute_eval_dssm_pointwise_classification(self): cur_eval_path = 'data/test/distribute_eval_test/dssm_distribute_eval_pointwise_classification_taobao_ckpt' self._success = test_utils.test_distributed_eval( 'samples/model_config/dssm_distribute_eval_pointwise_classification_on_taobao.config', - cur_eval_path, self._test_dir + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) @@ -1308,7 +1200,8 @@ def test_distribute_eval_dssm_reg(self): cur_eval_path = 'data/test/distribute_eval_test/dssm_distribute_eval_reg_taobao_ckpt' self._success = test_utils.test_distributed_eval( 'samples/model_config/dssm_distribute_eval_reg_on_taobao.config', - cur_eval_path, self._test_dir + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) @@ -1316,7 +1209,8 @@ def test_distribute_eval_dropout(self): cur_eval_path = 'data/test/distribute_eval_test/dropoutnet_distribute_eval_taobao_ckpt' self._success = test_utils.test_distributed_eval( 'samples/model_config/dropoutnet_distribute_eval_on_taobao.config', - cur_eval_path, self._test_dir + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) @@ -1324,7 +1218,8 @@ def test_distribute_eval_esmm(self): cur_eval_path = 'data/test/distribute_eval_test/esmm_distribute_eval_taobao_ckpt' self._success = test_utils.test_distributed_eval( 'samples/model_config/esmm_distribute_eval_on_taobao.config', - cur_eval_path, self._test_dir + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) @@ -1338,7 +1233,7 @@ def test_share_no_used(self): def test_dssm_neg_sampler_sequence_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_neg_sampler_sequence_feature.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -1346,28 +1241,28 @@ def test_dssm_neg_sampler_sequence_feature(self): def test_dssm_neg_sampler_need_key_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_neg_sampler_need_key_feature.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_need_key_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_boundary_need_key_feature_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_allow_key_transform(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_boundary_allow_key_transform.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_aux_hist_seq(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_aux_hist_seq_taobao.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -1375,7 +1270,7 @@ def test_dbmtl_on_multi_numeric_boundary_aux_hist_seq(self): def test_multi_tower_recall_neg_sampler_sequence_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_recall_neg_sampler_sequence_feature.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -1383,7 +1278,7 @@ def test_multi_tower_recall_neg_sampler_sequence_feature(self): def test_multi_tower_recall_neg_sampler_only_sequence_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_recall_neg_sampler_only_sequence_feature.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -1392,25 +1287,20 @@ def test_horovod(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/deepfm_combo_on_avazu_ctr.config', self._test_dir, - use_hvd=True + use_hvd=True, ) self.assertTrue(self._success) - @unittest.skipIf( - hvd is None or sok is None, 'horovod and sok is not installed' - ) + @unittest.skipIf(hvd is None or sok is None, 'horovod and sok is not installed') def test_sok(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_on_taobao_sok.config', self._test_dir, - use_hvd=True + use_hvd=True, ) self.assertTrue(self._success) - @unittest.skipIf( - six.PY2 or tf_version.split('.')[0] != '2', - 'only run on python3 and tf 2.x' - ) + @unittest.skipIf(six.PY2 or tf_version.split('.')[0] != '2', 'only run on python3 and tf 2.x') def test_train_parquet(self): os.environ[constant.NO_ARITHMETRIC_OPTI] = '1' self._success = test_utils.test_single_train_eval( @@ -1423,7 +1313,7 @@ def test_train_parquet_embedding_parallel(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/dlrm_on_criteo_parquet_ep.config', self._test_dir, - use_hvd=True + use_hvd=True, ) self.assertTrue(self._success) @@ -1432,14 +1322,12 @@ def test_train_parquet_embedding_parallel_v2(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/dlrm_on_criteo_parquet_ep_v2.config', self._test_dir, - use_hvd=True + use_hvd=True, ) self.assertTrue(self._success) def test_pdn(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/pdn_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/pdn_on_taobao.config', self._test_dir) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') @@ -1459,8 +1347,7 @@ def test_dssm_backbone_on_taobao(self): @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_senet_backbone_on_taobao(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_senet_on_taobao_backbone.config', - self._test_dir + 'samples/model_config/dssm_senet_on_taobao_backbone.config', self._test_dir ) self.assertTrue(self._success) @@ -1468,7 +1355,7 @@ def test_dssm_senet_backbone_on_taobao(self): def test_parallel_dssm_backbone_on_taobao(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/parallel_dssm_on_taobao_backbone.config', - self._test_dir + self._test_dir, ) self.assertTrue(self._success) @@ -1480,9 +1367,7 @@ def test_xdeefm_backbone_on_taobao(self): @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dat_on_taobao(self): - self._success = test_utils.test_single_train_eval( - 'samples/model_config/dat_on_taobao.config', self._test_dir - ) + self._success = test_utils.test_single_train_eval('samples/model_config/dat_on_taobao.config', self._test_dir) self.assertTrue(self._success) diff --git a/easy_rec/python/test/util_test.py b/easy_rec/python/test/util_test.py index 90788d4e8..420069372 100644 --- a/easy_rec/python/test/util_test.py +++ b/easy_rec/python/test/util_test.py @@ -13,58 +13,43 @@ class UtilTest(tf.test.TestCase): - def test_get_ckpt_version(self): - ver = estimator_utils.get_ckpt_version( - 'oss://easyrec/ckpts/model.ckpt-6500.meta' - ) + ver = estimator_utils.get_ckpt_version('oss://easyrec/ckpts/model.ckpt-6500.meta') assert ver == 6500, 'invalid version: %s' % str(ver) - ver = estimator_utils.get_ckpt_version( - 'oss://easyrec/ckpts/model.ckpt-6500' - ) + ver = estimator_utils.get_ckpt_version('oss://easyrec/ckpts/model.ckpt-6500') assert ver == 6500, 'invalid version: %s' % str(ver) def test_get_expression_greater(self): - result = get_expression( - 'age_level>item_age_level', ['age_level', 'item_age_level'] - ) + result = get_expression('age_level>item_age_level', ['age_level', 'item_age_level']) assert result == "tf.greater(parsed_dict['age_level'], parsed_dict['item_age_level'])" def test_get_expression_greater_equal(self): - result = get_expression( - 'age_level>=item_age_level', ['age_level', 'item_age_level'] - ) + result = get_expression('age_level>=item_age_level', ['age_level', 'item_age_level']) assert result == "tf.greater_equal(parsed_dict['age_level'], parsed_dict['item_age_level'])" def test_get_expression_less(self): - result = get_expression( - 'age_level3)&(item_age_level<1)', ['age_level', 'item_age_level'] - ) + result = get_expression('(age_level>3)&(item_age_level<1)', ['age_level', 'item_age_level']) assert result == "tf.greater(parsed_dict['age_level'], 3) & tf.less(parsed_dict['item_age_level'], 1)" result = get_expression( '(age_level>item_age_level) & (age_level3)|(item_age_level<1)', ['age_level', 'item_age_level'] - ) + result = get_expression('(age_level>3)|(item_age_level<1)', ['age_level', 'item_age_level']) assert result == "tf.greater(parsed_dict['age_level'], 3) | tf.less(parsed_dict['item_age_level'], 1)" def test_dag(self): diff --git a/easy_rec/python/test/zero_inflated_lognormal_test.py b/easy_rec/python/test/zero_inflated_lognormal_test.py index 1a495231f..064209e77 100644 --- a/easy_rec/python/test/zero_inflated_lognormal_test.py +++ b/easy_rec/python/test/zero_inflated_lognormal_test.py @@ -4,18 +4,19 @@ import tensorflow as tf from scipy import stats -from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_loss # NOQA +from easy_rec.python.loss.zero_inflated_lognormal import ( # NOQA + zero_inflated_lognormal_loss, +) if tf.__version__ >= '2.0': tf = tf.compat.v1 class ZeroInflatedLognormalLossTest(tf.test.TestCase): - def setUp(self): super(ZeroInflatedLognormalLossTest, self).setUp() - self.logits = np.array([[.1, .2, .3], [.4, .5, .6]]) - self.labels = np.array([[0.], [1.5]]) + self.logits = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]) + self.labels = np.array([[0.0], [1.5]]) def zero_inflated_lognormal(self, labels, logits, max_sigma=5.0): labels = labels.astype(np.float32) @@ -44,7 +45,7 @@ def zero_inflated_lognormal(self, labels, logits, max_sigma=5.0): x=labels[mask, 0].astype(np.float64), s=sigma[mask, 0].astype(np.float64), loc=0.0, - scale=np.exp(mu[mask, 0].astype(np.float64)) + scale=np.exp(mu[mask, 0].astype(np.float64)), ) num_pos = np.sum(positive) + 1e-8 regression_loss = -(np.sum(positive * logprob) / num_pos) @@ -55,9 +56,7 @@ def zero_inflated_lognormal(self, labels, logits, max_sigma=5.0): def test_loss_value(self): expected_loss = self.zero_inflated_lognormal(self.labels, self.logits) expected_loss = np.average(expected_loss) - loss = zero_inflated_lognormal_loss( - self.labels, self.logits, mu_reg=0, sigma_reg=0 - ) + loss = zero_inflated_lognormal_loss(self.labels, self.logits, mu_reg=0, sigma_reg=0) # Absolute error tolerance in asserting array near. _ERR_TOL = 1e-6 self.assertNear(self.evaluate(loss), expected_loss, _ERR_TOL) diff --git a/easy_rec/python/tools/add_boundaries_to_config.py b/easy_rec/python/tools/add_boundaries_to_config.py index c173ace82..cae47e133 100644 --- a/easy_rec/python/tools/add_boundaries_to_config.py +++ b/easy_rec/python/tools/add_boundaries_to_config.py @@ -15,30 +15,20 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO -) -tf.app.flags.DEFINE_string( - 'template_config_path', None, 'Path to template pipeline config ' - 'file.' -) -tf.app.flags.DEFINE_string( - 'output_config_path', None, 'Path to output pipeline config ' - 'file.' + level=logging.INFO, ) +tf.app.flags.DEFINE_string('template_config_path', None, 'Path to template pipeline config ' 'file.') +tf.app.flags.DEFINE_string('output_config_path', None, 'Path to output pipeline config ' 'file.') tf.app.flags.DEFINE_string('tables', '', 'quantile binning table') FLAGS = tf.app.flags.FLAGS def main(argv): - pipeline_config = config_util.get_configs_from_pipeline_file( - FLAGS.template_config_path - ) + pipeline_config = config_util.get_configs_from_pipeline_file(FLAGS.template_config_path) feature_boundaries_info = {} - reader = common_io.table.TableReader( - FLAGS.tables, selected_cols='feature,json' - ) + reader = common_io.table.TableReader(FLAGS.tables, selected_cols='feature,json') while True: try: record = reader.read() diff --git a/easy_rec/python/tools/add_feature_info_to_config.py b/easy_rec/python/tools/add_feature_info_to_config.py index 1c8c354b2..a564b2bed 100644 --- a/easy_rec/python/tools/add_feature_info_to_config.py +++ b/easy_rec/python/tools/add_feature_info_to_config.py @@ -15,25 +15,17 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO -) -tf.app.flags.DEFINE_string( - 'template_config_path', None, 'Path to template pipeline config ' - 'file.' -) -tf.app.flags.DEFINE_string( - 'output_config_path', None, 'Path to output pipeline config ' - 'file.' + level=logging.INFO, ) +tf.app.flags.DEFINE_string('template_config_path', None, 'Path to template pipeline config ' 'file.') +tf.app.flags.DEFINE_string('output_config_path', None, 'Path to output pipeline config ' 'file.') tf.app.flags.DEFINE_string('config_table', '', 'config table') FLAGS = tf.app.flags.FLAGS def main(argv): - pipeline_config = config_util.get_configs_from_pipeline_file( - FLAGS.template_config_path - ) + pipeline_config = config_util.get_configs_from_pipeline_file(FLAGS.template_config_path) sels = 'feature,feature_info,message' feature_info_map = {} drop_feature_names = [] @@ -43,7 +35,7 @@ def main(argv): data_config=pipeline_config.data_config, hive_config=pipeline_config.hive_train_input, selected_cols=sels, - record_defaults=['', '', ''] + record_defaults=['', '', ''], ) reader = hive_util.hive_read_line(FLAGS.config_table) for record in reader: @@ -54,9 +46,8 @@ def main(argv): else: import common_io - reader = common_io.table.TableReader( - FLAGS.config_table, selected_cols=sels - ) + + reader = common_io.table.TableReader(FLAGS.config_table, selected_cols=sels) while True: try: record = reader.read() @@ -79,29 +70,18 @@ def main(argv): feature_name = feature_config.input_names[0] if feature_name in feature_info_map: logging.info('edited %s' % feature_name) - feature_config.embedding_dim = int( - feature_info_map[feature_name]['embedding_dim'] - ) + feature_config.embedding_dim = int(feature_info_map[feature_name]['embedding_dim']) logging.info('modify embedding_dim to %s' % feature_config.embedding_dim) if 'boundary' in feature_info_map[feature_name]: feature_config.ClearField('boundaries') - feature_config.boundaries.extend( - [float(i) for i in feature_info_map[feature_name]['boundary']] - ) + feature_config.boundaries.extend([float(i) for i in feature_info_map[feature_name]['boundary']]) logging.info('modify boundaries to %s' % feature_config.boundaries) elif 'hash_bucket_size' in feature_info_map[feature_name]: - feature_config.hash_bucket_size = int( - feature_info_map[feature_name]['hash_bucket_size'] - ) - logging.info( - 'modify hash_bucket_size to %s' % feature_config.hash_bucket_size - ) + feature_config.hash_bucket_size = int(feature_info_map[feature_name]['hash_bucket_size']) + logging.info('modify hash_bucket_size to %s' % feature_config.hash_bucket_size) # modify num_steps - pipeline_config.train_config.num_steps = feature_info_map['__NUM_STEPS__'][ - 'num_steps'] - logging.info( - 'modify num_steps to %s' % pipeline_config.train_config.num_steps - ) + pipeline_config.train_config.num_steps = feature_info_map['__NUM_STEPS__']['num_steps'] + logging.info('modify num_steps to %s' % pipeline_config.train_config.num_steps) # modify decay_steps optimizer_configs = pipeline_config.train_config.optimizer_config for optimizer_config in optimizer_configs: @@ -110,8 +90,7 @@ def main(argv): learning_rate = optimizer.learning_rate.WhichOneof('learning_rate') learning_rate = getattr(optimizer.learning_rate, learning_rate) if hasattr(learning_rate, 'decay_steps'): - learning_rate.decay_steps = feature_info_map['__DECAY_STEPS__'][ - 'decay_steps'] + learning_rate.decay_steps = feature_info_map['__DECAY_STEPS__']['decay_steps'] logging.info('modify decay_steps to %s' % learning_rate.decay_steps) for feature_group in pipeline_config.model_config.feature_groups: diff --git a/easy_rec/python/tools/convert_config_format.py b/easy_rec/python/tools/convert_config_format.py index a1b1906a5..49fbcc355 100644 --- a/easy_rec/python/tools/convert_config_format.py +++ b/easy_rec/python/tools/convert_config_format.py @@ -25,11 +25,7 @@ def save_config(pipeline_config, save_path): if save_path.endswith('.config'): fout.write(text_format.MessageToString(pipeline_config, as_utf8=True)) elif save_path.endswith('.json'): - fout.write( - json_format.MessageToJson( - pipeline_config, preserving_proto_field_name=True - ) - ) + fout.write(json_format.MessageToJson(pipeline_config, preserving_proto_field_name=True)) else: assert False, 'only .config/.json are supported(%s)' % save_path @@ -38,12 +34,8 @@ def save_config(pipeline_config, save_path): import argparse parser = argparse.ArgumentParser() - parser.add_argument( - '--input_config', type=str, help='input_config path', default=None - ) - parser.add_argument( - '--output_config', type=str, help='output_config path', default=None - ) + parser.add_argument('--input_config', type=str, help='input_config path', default=None) + parser.add_argument('--output_config', type=str, help='output_config path', default=None) args = parser.parse_args() assert os.path.exists(args.input_config) diff --git a/easy_rec/python/tools/convert_rtp_data.py b/easy_rec/python/tools/convert_rtp_data.py index 3edbcc94d..60ae1fcc7 100644 --- a/easy_rec/python/tools/convert_rtp_data.py +++ b/easy_rec/python/tools/convert_rtp_data.py @@ -9,6 +9,7 @@ In our new format: ...beautysmart... """ + import argparse import csv import json @@ -19,7 +20,7 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO + level=logging.INFO, ) if tf.__version__ >= '2.0': @@ -27,13 +28,9 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( - '--rtp_fg', type=str, default='', help='rtp fg path(.json)' - ) + parser.add_argument('--rtp_fg', type=str, default='', help='rtp fg path(.json)') parser.add_argument('--input_path', type=str, default='', help='input path') - parser.add_argument( - '--output_path', type=str, default='', help='output path' - ) + parser.add_argument('--output_path', type=str, default='', help='output path') parser.add_argument('--label', type=str, default='', help='label for train') args = parser.parse_args() diff --git a/easy_rec/python/tools/convert_rtp_fg.py b/easy_rec/python/tools/convert_rtp_fg.py index 2315bd43b..5c9954610 100644 --- a/easy_rec/python/tools/convert_rtp_fg.py +++ b/easy_rec/python/tools/convert_rtp_fg.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Convert rtp fg feature config to easy_rec data_config and feature_config.""" + import argparse import logging import sys @@ -12,7 +13,7 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO + level=logging.INFO, ) if tf.__version__ >= '2.0': @@ -26,69 +27,48 @@ type=str, choices=model_types, default='', - help='model type, currently support: %s' % ','.join(model_types) + help='model type, currently support: %s' % ','.join(model_types), ) parser.add_argument('--rtp_fg', type=str, help='rtp fg path') - parser.add_argument( - '--embedding_dim', type=int, default=16, help='embedding_dimension' - ) - parser.add_argument( - '--batch_size', type=int, default=1024, help='batch_size for train' - ) - parser.add_argument( - '--label', - type=str, - default='', - nargs='+', - required=True, - help='label fields' - ) + parser.add_argument('--embedding_dim', type=int, default=16, help='embedding_dimension') + parser.add_argument('--batch_size', type=int, default=1024, help='batch_size for train') + parser.add_argument('--label', type=str, default='', nargs='+', required=True, help='label fields') parser.add_argument( '--num_steps', type=int, default=1000, - help= - 'number of train steps = num_samples * num_epochs / batch_size / num_workers' + help='number of train steps = num_samples * num_epochs / batch_size / num_workers', ) parser.add_argument('--output_path', type=str, help='generated config path') parser.add_argument( '--incol_separator', type=str, default='\003', - help='separator for multi_value features' + help='separator for multi_value features', ) parser.add_argument( '--separator', type=str, default='\002', - help='separator between different features' - ) - parser.add_argument( - '--train_input_path', type=str, default=None, help='train data path' - ) - parser.add_argument( - '--eval_input_path', type=str, default=None, help='eval data path' + help='separator between different features', ) + parser.add_argument('--train_input_path', type=str, default=None, help='train data path') + parser.add_argument('--eval_input_path', type=str, default=None, help='eval data path') parser.add_argument( '--selected_cols', type=str, default=None, - help= - 'selected cols, for csv input, it is in the format of: label_col_id0,...,lable_cold_idn,feature_col_id ' - 'for odps table input, it is in the format of: label_col_name0,...,label_col_namen,feature_col_name ' - ) - parser.add_argument( - '--rtp_separator', type=str, default=';', help='separator' + help='selected cols, for csv input, it is in the format of: label_col_id0,...,lable_cold_idn,feature_col_id ' + 'for odps table input, it is in the format of: label_col_name0,...,label_col_namen,feature_col_name ', ) + parser.add_argument('--rtp_separator', type=str, default=';', help='separator') parser.add_argument( '--input_type', type=str, default='OdpsRTPInput', - help='default to OdpsRTPInput, if test local, change it to RTPInput' - ) - parser.add_argument( - '--is_async', action='store_true', help='async mode, debug to false' + help='default to OdpsRTPInput, if test local, change it to RTPInput', ) + parser.add_argument('--is_async', action='store_true', help='async mode, debug to false') args = parser.parse_args() @@ -101,10 +81,19 @@ sys.exit(1) pipeline_config = convert_rtp_fg( - args.rtp_fg, args.embedding_dim, args.batch_size, args.label, - args.num_steps, args.model_type, args.separator, args.incol_separator, - args.train_input_path, args.eval_input_path, args.selected_cols, - args.input_type, args.is_async + args.rtp_fg, + args.embedding_dim, + args.batch_size, + args.label, + args.num_steps, + args.model_type, + args.separator, + args.incol_separator, + args.train_input_path, + args.eval_input_path, + args.selected_cols, + args.input_type, + args.is_async, ) save_message(pipeline_config, args.output_path) logging.info('Conversion done.') @@ -117,7 +106,4 @@ 'if run local, please set data_config.selected_cols in the format ' 'label_col_id0,label_col_id1,...,label_col_idn,feature_col_id' ) - logging.info( - 'if run on odps, selected_cols must be set, which are label0_col,' - 'label1_col, ..., feature_col_name' - ) + logging.info('if run on odps, selected_cols must be set, which are label0_col,' 'label1_col, ..., feature_col_name') diff --git a/easy_rec/python/tools/create_config_from_excel.py b/easy_rec/python/tools/create_config_from_excel.py index 201260615..f04689408 100644 --- a/easy_rec/python/tools/create_config_from_excel.py +++ b/easy_rec/python/tools/create_config_from_excel.py @@ -9,16 +9,20 @@ from easy_rec.python.utils import config_util -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') class ModelConfigConverter: - def __init__( - self, excel_path, output_path, model_type, column_separator, - incol_separator, train_input_path, eval_input_path, model_dir + self, + excel_path, + output_path, + model_type, + column_separator, + incol_separator, + train_input_path, + eval_input_path, + model_dir, ): self._excel_path = excel_path self._output_path = output_path @@ -35,9 +39,7 @@ def __init__( self._model_dir = model_dir if not self._model_dir: self._model_dir = 'experiments/demo' - logging.warning( - 'model_dir is not specified, set to %s' % self._model_dir - ) + logging.warning('model_dir is not specified, set to %s' % self._model_dir) def _get_type_name(self, input_name): type_dict = { @@ -45,7 +47,7 @@ def _get_type_name(self, input_name): 'double': 'DOUBLE', 'float': 'FLOAT', 'string': 'STRING', - 'bool': 'BOOL' + 'bool': 'BOOL', } return type_dict[input_name] @@ -55,7 +57,7 @@ def _get_type_default(self, input_name): 'double': '0.0', 'float': '0.0', 'string': '', - 'bool': 'false' + 'bool': 'false', } return type_dict[input_name] @@ -133,19 +135,21 @@ def _is_good(v): return str(v) not in ['nan', ''] if _is_good(self._dict_global[field['global']]['default_value']): - field['default_value'] = self._dict_global[field['global'] - ]['default_value'] + field['default_value'] = self._dict_global[field['global']]['default_value'] if _is_good(self._dict_global[field['global']]['hash_bucket_size']): - field['hash_bucket_size'] = self._dict_global[field['global'] - ]['hash_bucket_size'] + field['hash_bucket_size'] = self._dict_global[field['global']]['hash_bucket_size'] if _is_good(self._dict_global[field['global']]['embedding_dim']): - field['embedding_dim'] = self._dict_global[field['global'] - ]['embedding_dim'] + field['embedding_dim'] = self._dict_global[field['global']]['embedding_dim'] field['embedding_name'] = field['global'] for t in [ - 'type', 'global', 'hash_bucket_size', 'embedding_dim', 'default_value', - 'weights', 'boundaries' + 'type', + 'global', + 'hash_bucket_size', + 'embedding_dim', + 'default_value', + 'weights', + 'boundaries', ]: if t not in row: continue @@ -184,8 +188,7 @@ def _is_good(v): # check that tag features weights are one of the fields for name, config in self._feature_details.items(): if config['type'] == 'tags': - if 'weights' in config and config['weights' - ] not in self._feature_details: + if 'weights' in config and config['weights'] not in self._feature_details: raise ValueError(config['weights'] + ' not in field names') def _write_train_eval_config(self, fout): @@ -218,7 +221,8 @@ def _write_train_eval_config(self, fout): metrics_set: { auc {} } - }""" % self._model_dir + }""" + % self._model_dir ) def _write_deepfm_config(self, fout): @@ -288,7 +292,8 @@ def _write_multi_tower_config(self, fout): dnn { hidden_units: [256, 192, 128] } - }""" % tower_name + }""" + % tower_name ) fout.write( @@ -308,15 +313,9 @@ def _write_data_config(self, fout): for name in self._feature_names: fout.write(' input_fields: {\n') fout.write(' input_name: "%s"\n' % name) - fout.write( - ' input_type: %s\n' % - self._get_type_name(self._feature_details[name]['data_type']) - ) + fout.write(' input_type: %s\n' % self._get_type_name(self._feature_details[name]['data_type'])) if 'default_value' in self._feature_details[name]: - fout.write( - ' default_val:"%s"\n' % - self._feature_details[name]['default_value'] - ) + fout.write(' default_val:"%s"\n' % self._feature_details[name]['default_value']) fout.write(' }\n') fout.write(' label_fields: "%s"\n' % self._label) @@ -346,13 +345,9 @@ def _write_feature_config(self, fout): elif feature['type'] == 'dense': fout.write(' feature_type: RawFeature\n') if self._model_type == 'deepfm': - assert feature[ - 'boundaries' - ] != '', 'raw features must be discretized by specifying boundaries' + assert feature['boundaries'] != '', 'raw features must be discretized by specifying boundaries' if 'boundaries' in feature and feature['boundaries'] != '': - fout.write( - ' boundaries: [%s]\n' % str(feature['boundaries']).strip() - ) + fout.write(' boundaries: [%s]\n' % str(feature['boundaries']).strip()) fout.write(' embedding_dim: %d\n' % int(feature['embedding_dim'])) elif feature['type'] == 'tags': if 'weights' in feature: @@ -378,10 +373,7 @@ def _write_feature_config(self, fout): def convert(self): self._parse_features() - logging.info( - 'TOWERS[%d]: %s' % - (len(self._tower_dicts), ','.join(list(self._tower_dicts.keys()))) - ) + logging.info('TOWERS[%d]: %s' % (len(self._tower_dicts), ','.join(list(self._tower_dicts.keys())))) with open(self._output_path, 'w') as fout: self._write_train_eval_config(fout) self._write_data_config(fout) @@ -392,12 +384,10 @@ def convert(self): self._write_multi_tower_config(fout) else: logging.warning( - 'the model_config could not be generated automatically, you have to write the model_config manually.' + 'the model_config could not be generated automatically, you have to write the model_config manually' ) # reformat the config - pipeline_config = config_util.get_configs_from_pipeline_file( - self._output_path - ) + pipeline_config = config_util.get_configs_from_pipeline_file(self._output_path) config_util.save_message(pipeline_config, self._output_path) @@ -411,7 +401,7 @@ def convert(self): '--model_type', type=str, choices=model_types, - help='model type, currently support: %s' % ','.join(model_types) + help='model type, currently support: %s' % ','.join(model_types), ) parser.add_argument('--excel_path', type=str, help='excel config path') parser.add_argument('--output_path', type=str, help='generated config path') @@ -419,20 +409,16 @@ def convert(self): '--column_separator', type=str, default=',', - help='column separator, separator betwen features' + help='column separator, separator between features', ) parser.add_argument( '--incol_separator', type=str, default='|', - help='separator within features, such as tag features' - ) - parser.add_argument( - '--train_input_path', type=str, default='', help='train input path' - ) - parser.add_argument( - '--eval_input_path', type=str, default='', help='eval input path' + help='separator within features, such as tag features', ) + parser.add_argument('--train_input_path', type=str, default='', help='train input path') + parser.add_argument('--eval_input_path', type=str, default='', help='eval input path') parser.add_argument('--model_dir', type=str, default='', help='model dir') args = parser.parse_args() @@ -440,15 +426,17 @@ def convert(self): parser.print_usage() sys.exit(1) - logging.info( - 'column_separator = %s in_column_separator = %s' % - (args.column_separator, args.incol_separator) - ) + logging.info('column_separator = %s in_column_separator = %s' % (args.column_separator, args.incol_separator)) converter = ModelConfigConverter( - args.excel_path, args.output_path, args.model_type, args.column_separator, - args.incol_separator, args.train_input_path, args.eval_input_path, - args.model_dir + args.excel_path, + args.output_path, + args.model_type, + args.column_separator, + args.incol_separator, + args.train_input_path, + args.eval_input_path, + args.model_dir, ) converter.convert() logging.info('Conversion done') diff --git a/easy_rec/python/tools/criteo/convert_data.py b/easy_rec/python/tools/criteo/convert_data.py index 7e1bc142e..af8fe304c 100644 --- a/easy_rec/python/tools/criteo/convert_data.py +++ b/easy_rec/python/tools/criteo/convert_data.py @@ -12,9 +12,7 @@ import six from tensorflow.python.platform import gfile -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') def save_np_bin(labels, dense_arr, cate_arr, prefix): @@ -39,10 +37,7 @@ def save_parquet(labels, dense_arr, cate_arr, prefix): def convert(input_path, prefix, part_record_num, save_format): - logging.info( - 'start to convert %s, part_record_num=%d, save_format=%s' % - (input_path, part_record_num, save_format) - ) + logging.info('start to convert %s, part_record_num=%d, save_format=%s' % (input_path, part_record_num, save_format)) save_func = save_np_bin if save_format == 'parquet': save_func = save_parquet @@ -79,8 +74,10 @@ def convert(input_path, prefix, part_record_num, save_format): sid = 0 if sid > 0: save_func( - labels[:sid], dense_arr[:sid], cate_arr[:sid], - prefix + '_' + str(part_id) + labels[:sid], + dense_arr[:sid], + cate_arr[:sid], + prefix + '_' + str(part_id), ) logging.info('\t%s write final part: %d' % (input_path, part_id)) part_id += 1 @@ -89,10 +86,7 @@ def convert(input_path, prefix, part_record_num, save_format): logging.error('convert %s failed: %s' % (input_path, str(ex))) logging.error(traceback.format_exc()) return - logging.info( - 'done convert %s, total_line=%d, part_num=%d' % - (input_path, total_line, part_id) - ) + logging.info('done convert %s, total_line=%d, part_num=%d' % (input_path, total_line, part_id)) if __name__ == '__main__': @@ -111,32 +105,25 @@ def convert(input_path, prefix, part_record_num, save_format): """ parser = argparse.ArgumentParser() - parser.add_argument( - '--input_dir', type=str, default=None, help='criteo 1t data dir' - ) - parser.add_argument( - '--save_dir', - type=str, - default=None, - help='criteo binary data output dir ' - ) + parser.add_argument('--input_dir', type=str, default=None, help='criteo 1t data dir') + parser.add_argument('--save_dir', type=str, default=None, help='criteo binary data output dir ') parser.add_argument( '--save_format', type=str, default='npy', - help='save format, choices: npy|parquet' + help='save format, choices: npy|parquet', ) parser.add_argument( '--part_record_num', type=int, default=1024 * 1024 * 8, - help='the maximal number of samples in each binary file' + help='the maximal number of samples in each binary file', ) parser.add_argument( '--dt', nargs='*', type=int, - help='select days to convert, default to select all: 0-23' + help='select days to convert, default to select all: 0-23', ) args = parser.parse_args() @@ -161,7 +148,7 @@ def convert(input_path, prefix, part_record_num, save_format): prefix = os.path.join(args.save_dir, str(d)) proc = multiprocessing.Process( target=convert, - args=(input_path, prefix, args.part_record_num, args.save_format) + args=(input_path, prefix, args.part_record_num, args.save_format), ) convert(input_path, prefix, args.part_record_num, args.save_format) proc.start() diff --git a/easy_rec/python/tools/edit_lookup_graph.py b/easy_rec/python/tools/edit_lookup_graph.py index 4ae0ad218..00d5aa8ee 100644 --- a/easy_rec/python/tools/edit_lookup_graph.py +++ b/easy_rec/python/tools/edit_lookup_graph.py @@ -15,7 +15,7 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO + level=logging.INFO, ) if __name__ == '__main__': @@ -31,22 +31,12 @@ """ parser = argparse.ArgumentParser() - parser.add_argument( - '--saved_model_dir', type=str, default=None, help='saved model dir' - ) - parser.add_argument( - '--output_dir', type=str, default=None, help='output dir' - ) - parser.add_argument( - '--redis_url', type=str, default='127.0.0.1:6379', help='redis url' - ) - parser.add_argument( - '--redis_passwd', type=str, default='', help='redis password' - ) + parser.add_argument('--saved_model_dir', type=str, default=None, help='saved model dir') + parser.add_argument('--output_dir', type=str, default=None, help='output dir') + parser.add_argument('--redis_url', type=str, default='127.0.0.1:6379', help='redis url') + parser.add_argument('--redis_passwd', type=str, default='', help='redis password') parser.add_argument('--time_out', type=int, default=1500, help='timeout') - parser.add_argument( - '--test_data_path', type=str, default='', help='test data path' - ) + parser.add_argument('--test_data_path', type=str, default='', help='test data path') parser.add_argument('--verbose', action='store_true', default=False) args = parser.parse_args() @@ -71,35 +61,26 @@ args.redis_passwd, args.time_out, meta_graph_def=None, - debug_dir=args.output_dir if args.verbose else '' + debug_dir=args.output_dir if args.verbose else '', ) meta_graph_editor.edit_graph() meta_graph_version = meta_graph_editor.meta_graph_version if meta_graph_version == '': - export_ts = [ - x for x in args.saved_model_dir.split('/') if x != '' and x is not None - ] + export_ts = [x for x in args.saved_model_dir.split('/') if x != '' and x is not None] meta_graph_version = export_ts[-1] # import edit graph tf.reset_default_graph() saver = tf.train.import_meta_graph(meta_graph_editor._meta_graph_def) - embed_name_to_id_file = os.path.join( - args.output_dir, 'embed_name_to_ids.txt' - ) + embed_name_to_id_file = os.path.join(args.output_dir, 'embed_name_to_ids.txt') with GFile(embed_name_to_id_file, 'w') as fout: for tmp_norm_name in meta_graph_editor._embed_name_to_ids: - fout.write( - '%s\t%s\n' % - (tmp_norm_name, meta_graph_editor._embed_name_to_ids[tmp_norm_name]) - ) + fout.write('%s\t%s\n' % (tmp_norm_name, meta_graph_editor._embed_name_to_ids[tmp_norm_name])) tf.add_to_collection( tf.GraphKeys.ASSET_FILEPATHS, - tf.constant( - embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt' - ) + tf.constant(embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt'), ) graph = tf.get_default_graph() @@ -117,18 +98,14 @@ with tf.Session() as sess: saver.restore(sess, args.saved_model_dir + '/variables/variables') output_dir = os.path.join(args.output_dir, meta_graph_version) - tf.saved_model.simple_save( - sess, output_dir, inputs=inputs_map, outputs=outputs_map - ) + tf.saved_model.simple_save(sess, output_dir, inputs=inputs_map, outputs=outputs_map) # the meta_graph_version could not be passed via existing interfaces # so we could only write it by the raw methods saved_model = saved_model_pb2.SavedModel() with GFile(os.path.join(output_dir, 'saved_model.pb'), 'rb') as fin: saved_model.ParseFromString(fin.read()) - saved_model.meta_graphs[ - 0 - ].meta_info_def.meta_graph_version = meta_graph_editor.meta_graph_version + saved_model.meta_graphs[0].meta_info_def.meta_graph_version = meta_graph_editor.meta_graph_version with GFile(os.path.join(output_dir, 'saved_model.pb'), 'wb') as fout: fout.write(saved_model.SerializeToString()) @@ -143,7 +120,5 @@ feature_vals.append(''.join(line_toks)) if len(feature_vals) >= 32: break - out_vals = sess.run( - outputs_map, feed_dict={inputs_map['features']: feature_vals} - ) + out_vals = sess.run(outputs_map, feed_dict={inputs_map['features']: feature_vals}) logging.info('test_data probs:' + str(out_vals)) diff --git a/easy_rec/python/tools/faiss_index_pai.py b/easy_rec/python/tools/faiss_index_pai.py index d5415be6c..13c43c8ff 100644 --- a/easy_rec/python/tools/faiss_index_pai.py +++ b/easy_rec/python/tools/faiss_index_pai.py @@ -12,9 +12,7 @@ from easy_rec.python.utils import io_util -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') tf.app.flags.DEFINE_string('tables', '', 'tables passed by pai command') tf.app.flags.DEFINE_integer('batch_size', 1024, 'batch size') @@ -30,13 +28,9 @@ def main(argv): - reader = tf.python_io.TableReader( - FLAGS.tables, slice_id=0, slice_count=1, capacity=FLAGS.batch_size * 2 - ) + reader = tf.python_io.TableReader(FLAGS.tables, slice_id=0, slice_count=1, capacity=FLAGS.batch_size * 2) i = 0 - id_map_f = tf.gfile.GFile( - os.path.join(FLAGS.index_output_dir, 'id_mapping'), 'w' - ) + id_map_f = tf.gfile.GFile(os.path.join(FLAGS.index_output_dir, 'id_mapping'), 'w') embeddings = [] while True: try: @@ -46,9 +40,7 @@ def main(argv): eid = record[0].decode('utf-8') id_map_f.write('%s\n' % eid) - embeddings.extend( - [list(map(float, record[1].split(b','))) for record in records] - ) + embeddings.extend([list(map(float, record[1].split(b','))) for record in records]) i += 1 if i % 100 == 0: logging.info('read %d embeddings.' % (i * FLAGS.batch_size)) @@ -60,14 +52,9 @@ def main(argv): logging.info('Building faiss index..') if FLAGS.index_type == 'IVFFlat': quantizer = faiss.IndexFlatIP(FLAGS.embedding_dim) - index = faiss.IndexIVFFlat( - quantizer, FLAGS.embedding_dim, FLAGS.ivf_nlist, - faiss.METRIC_INNER_PRODUCT - ) + index = faiss.IndexIVFFlat(quantizer, FLAGS.embedding_dim, FLAGS.ivf_nlist, faiss.METRIC_INNER_PRODUCT) elif FLAGS.index_type == 'HNSWFlat': - index = faiss.IndexHNSWFlat( - FLAGS.embedding_dim, FLAGS.hnsw_M, faiss.METRIC_INNER_PRODUCT - ) + index = faiss.IndexHNSWFlat(FLAGS.embedding_dim, FLAGS.hnsw_M, faiss.METRIC_INNER_PRODUCT) index.hnsw.efConstruction = FLAGS.hnsw_efConstruction else: raise NotImplementedError @@ -81,9 +68,7 @@ def main(argv): index.add(embeddings) faiss.write_index(index, 'faiss_index') - with tf.gfile.GFile( - os.path.join(FLAGS.index_output_dir, 'faiss_index'), 'wb' - ) as f_out: + with tf.gfile.GFile(os.path.join(FLAGS.index_output_dir, 'faiss_index'), 'wb') as f_out: with open('faiss_index', 'rb') as f_in: f_out.write(f_in.read()) @@ -91,16 +76,12 @@ def main(argv): # IVFFlat for ivf_nlist in [100, 500, 1000, 2000]: quantizer = faiss.IndexFlatIP(FLAGS.embedding_dim) - index = faiss.IndexIVFFlat( - quantizer, FLAGS.embedding_dim, ivf_nlist, faiss.METRIC_INNER_PRODUCT - ) + index = faiss.IndexIVFFlat(quantizer, FLAGS.embedding_dim, ivf_nlist, faiss.METRIC_INNER_PRODUCT) index.train(embeddings) index.add(embeddings) index_name = 'faiss_index_ivfflat_nlist%d' % ivf_nlist faiss.write_index(index, index_name) - with tf.gfile.GFile( - os.path.join(FLAGS.index_output_dir, index_name), 'wb' - ) as f_out: + with tf.gfile.GFile(os.path.join(FLAGS.index_output_dir, index_name), 'wb') as f_out: with open(index_name, 'rb') as f_in: f_out.write(f_in.read()) @@ -109,18 +90,12 @@ def main(argv): for hnsw_efConstruction in [64, 128, 256, 512, 1024, 2048, 4096, 8196]: if hnsw_efConstruction < hnsw_M * 2: continue - index = faiss.IndexHNSWFlat( - FLAGS.embedding_dim, hnsw_M, faiss.METRIC_INNER_PRODUCT - ) + index = faiss.IndexHNSWFlat(FLAGS.embedding_dim, hnsw_M, faiss.METRIC_INNER_PRODUCT) index.hnsw.efConstruction = hnsw_efConstruction index.add(embeddings) - index_name = 'faiss_index_hnsw_M%d_ef%d' % ( - hnsw_M, hnsw_efConstruction - ) + index_name = 'faiss_index_hnsw_M%d_ef%d' % (hnsw_M, hnsw_efConstruction) faiss.write_index(index, index_name) - with tf.gfile.GFile( - os.path.join(FLAGS.index_output_dir, index_name), 'wb' - ) as f_out: + with tf.gfile.GFile(os.path.join(FLAGS.index_output_dir, index_name), 'wb') as f_out: with open(index_name, 'rb') as f_in: f_out.write(f_in.read()) diff --git a/easy_rec/python/tools/feature_selection.py b/easy_rec/python/tools/feature_selection.py index ba7e72122..f64a6518c 100644 --- a/easy_rec/python/tools/feature_selection.py +++ b/easy_rec/python/tools/feature_selection.py @@ -20,30 +20,17 @@ matplotlib.use('Agg') # NOQA import matplotlib.pyplot as plt # NOQA -tf.app.flags.DEFINE_string( - 'model_type', 'variational_dropout', 'feature selection model type' -) -tf.app.flags.DEFINE_string( - 'config_path', '', 'feature selection model config path' -) -tf.app.flags.DEFINE_string( - 'checkpoint_path', None, 'feature selection model checkpoint path' -) -tf.app.flags.DEFINE_string( - 'output_dir', '', 'feature selection result directory' -) -tf.app.flags.DEFINE_integer( - 'topk', 100, 'select topk importance features for each feature group' -) +tf.app.flags.DEFINE_string('model_type', 'variational_dropout', 'feature selection model type') +tf.app.flags.DEFINE_string('config_path', '', 'feature selection model config path') +tf.app.flags.DEFINE_string('checkpoint_path', None, 'feature selection model checkpoint path') +tf.app.flags.DEFINE_string('output_dir', '', 'feature selection result directory') +tf.app.flags.DEFINE_integer('topk', 100, 'select topk importance features for each feature group') tf.app.flags.DEFINE_string('fg_path', '', 'fg config path') -tf.app.flags.DEFINE_bool( - 'visualize', False, 'visualization feature selection result or not' -) +tf.app.flags.DEFINE_bool('visualize', False, 'visualization feature selection result or not') FLAGS = tf.app.flags.FLAGS class VariationalDropoutFS: - def __init__( self, config_path, @@ -51,7 +38,7 @@ def __init__( topk, checkpoint_path=None, fg_path=None, - visualize=False + visualize=False, ): self._config_path = config_path self._output_dir = output_dir @@ -64,15 +51,15 @@ def __init__( def process(self): tf.logging.info('Loading logit_p of VariationalDropout layer ...') - feature_dim_dropout_p_map, embedding_wise_variational_dropout = self._feature_dim_dropout_ratio( - ) + ( + feature_dim_dropout_p_map, + embedding_wise_variational_dropout, + ) = self._feature_dim_dropout_ratio() feature_importance_map = {} for group_name, feature_dim_dropout_p in feature_dim_dropout_p_map.items(): tf.logging.info('Calculating %s feature importance ...' % group_name) - feature_importance = self._get_feature_importance( - feature_dim_dropout_p, embedding_wise_variational_dropout - ) + feature_importance = self._get_feature_importance(feature_dim_dropout_p, embedding_wise_variational_dropout) feature_importance_map[group_name] = feature_importance tf.logging.info('Dump %s feature importance to csv ...' % group_name) @@ -90,9 +77,7 @@ def process(self): def _feature_dim_dropout_ratio(self): """Get dropout ratio of embedding-wise or feature-wise.""" config = config_util.get_configs_from_pipeline_file(self._config_path) - assert config.model_config.HasField( - 'variational_dropout' - ), 'variational_dropout must be in model_config' + assert config.model_config.HasField('variational_dropout'), 'variational_dropout must be in model_config' embedding_wise_variational_dropout = config.model_config.variational_dropout.embedding_wise_variational_dropout @@ -103,8 +88,7 @@ def _feature_dim_dropout_ratio(self): meta_graph_def = read_meta_graph_file(checkpoint_path + '.meta') features_dimension_map = dict() - for col_def in meta_graph_def.collection_def['variational_dropout' - ].bytes_list.value: + for col_def in meta_graph_def.collection_def['variational_dropout'].bytes_list.value: name, features_dimension = json.loads(col_def) name = 'all' if name == '' else name features_dimension_map[name] = OrderedDict(features_dimension) @@ -129,12 +113,10 @@ def _feature_dim_dropout_ratio(self): feature_dim_dropout_p = {} if embedding_wise_variational_dropout: index_end = 0 - for feature_name, feature_dim in features_dimension_map[group_name - ].items(): + for feature_name, feature_dim in features_dimension_map[group_name].items(): index_start = index_end index_end = index_start + feature_dim - feature_dim_dropout_p[feature_name] = feature_dims_importance[ - index_start:index_end] + feature_dim_dropout_p[feature_name] = feature_dims_importance[index_start:index_end] else: index = 0 for feature_name in features_dimension_map[group_name].keys(): @@ -144,22 +126,16 @@ def _feature_dim_dropout_ratio(self): feature_dim_dropout_p_map[group_name] = feature_dim_dropout_p return feature_dim_dropout_p_map, embedding_wise_variational_dropout - def _get_feature_importance( - self, feature_dim_dropout_p, embedding_wise_variational_dropout - ): + def _get_feature_importance(self, feature_dim_dropout_p, embedding_wise_variational_dropout): """Calculate feature importance.""" if embedding_wise_variational_dropout: feature_importance = {} for item in feature_dim_dropout_p.items(): dropout_rate_mean = np.mean(item[1]) feature_importance[item[0]] = dropout_rate_mean - feature_importance = OrderedDict( - sorted(feature_importance.items(), key=lambda e: e[1]) - ) + feature_importance = OrderedDict(sorted(feature_importance.items(), key=lambda e: e[1])) else: - feature_importance = OrderedDict( - sorted(feature_dim_dropout_p.items(), key=lambda e: e[1]) - ) + feature_importance = OrderedDict(sorted(feature_dim_dropout_p.items(), key=lambda e: e[1])) return feature_importance def _process_config(self, feature_importance_map): @@ -191,8 +167,9 @@ def _process_config(self, feature_importance_map): feature_configs = [] for feature_config in config_util.get_compatible_feature_configs(config): - feature_name = feature_config.feature_name if feature_config.HasField('feature_name') \ - else feature_config.input_names[0] + feature_name = ( + feature_config.feature_name if feature_config.HasField('feature_name') else feature_config.input_names[0] + ) if feature_name not in excluded_features: feature_configs.append(feature_config) @@ -210,10 +187,7 @@ def _process_config(self, feature_importance_map): feature_names.append(feature_name) feature_group.ClearField('feature_names') feature_group.feature_names.extend(feature_names) - config_util.save_message( - config, - os.path.join(self._output_dir, os.path.basename(self._config_path)) - ) + config_util.save_message(config, os.path.join(self._output_dir, os.path.basename(self._config_path))) if self._fg_path is not None and len(self._fg_path) > 0: with tf.gfile.Open(self._fg_path) as f: @@ -226,21 +200,18 @@ def _process_config(self, feature_importance_map): else: features.append(feature) fg_json['features'] = features - with tf.gfile.Open( - os.path.join(self._output_dir, os.path.basename(self._fg_path)), 'w' - ) as f: + with tf.gfile.Open(os.path.join(self._output_dir, os.path.basename(self._fg_path)), 'w') as f: json.dump(fg_json, f, indent=4) def _dump_to_csv(self, feature_importance, group_name): """Dump feature importance data to a csv file.""" with tf.gfile.Open( - os.path.join( - self._output_dir, 'feature_dropout_ratio_%s.csv' % group_name - ), 'w' + os.path.join(self._output_dir, 'feature_dropout_ratio_%s.csv' % group_name), + 'w', ) as f: df = pd.DataFrame( columns=['feature_name', 'mean_drop_p'], - data=[list(kv) for kv in feature_importance.items()] + data=[list(kv) for kv in feature_importance.items()], ) df.to_csv(f, encoding='gbk') @@ -267,7 +238,7 @@ def _visualize_embedding_dim_importance(self, feature_dim_dropout_p): align='center', alpha=0.4, label='dropout_rate', - lw=1 + lw=1, ) for rect in b: w = rect.get_width() @@ -276,7 +247,7 @@ def _visualize_embedding_dim_importance(self, feature_dim_dropout_p): rect.get_y() + rect.get_height() / 2, '%.4f' % w, ha='left', - va='center' + va='center', ) plt.yticks(y_pos, embedding_dims) plt.xlabel(feature_name) @@ -289,7 +260,7 @@ def _visualize_feature_importance(self, feature_importance, group_name): """Draw feature importance histogram.""" df = pd.DataFrame( columns=['feature_name', 'mean_drop_p'], - data=[list(kv) for kv in feature_importance.items()] + data=[list(kv) for kv in feature_importance.items()], ) df['color'] = ['red' if x < 0.5 else 'green' for x in df['mean_drop_p']] df.sort_values('mean_drop_p', inplace=True, ascending=False) @@ -304,10 +275,7 @@ def _visualize_feature_importance(self, feature_importance, group_name): round(tex, 2), horizontalalignment='right' if x < 0 else 'left', verticalalignment='center', - fontdict={ - 'color': 'red' if x < 0 else 'green', - 'size': 14 - } + fontdict={'color': 'red' if x < 0 else 'green', 'size': 14}, ) # Decorations plt.yticks(df.index, df.feature_name, fontsize=20) @@ -315,9 +283,8 @@ def _visualize_feature_importance(self, feature_importance, group_name): plt.grid(linestyle='--', alpha=0.5) plt.xlim(0, 1) with tf.gfile.GFile( - os.path.join( - self._output_dir, 'feature_dropout_pic_%s.png' % group_name - ), 'wb' + os.path.join(self._output_dir, 'feature_dropout_pic_%s.png' % group_name), + 'wb', ) as f: plt.savefig(f, format='png') @@ -331,10 +298,8 @@ def _visualize_feature_importance(self, feature_importance, group_name): FLAGS.topk, checkpoint_path=FLAGS.checkpoint_path, fg_path=FLAGS.fg_path, - visualize=FLAGS.visualize + visualize=FLAGS.visualize, ) fs.process() else: - raise ValueError( - 'Unknown feature selection model type %s' % FLAGS.model_type - ) + raise ValueError('Unknown feature selection model type %s' % FLAGS.model_type) diff --git a/easy_rec/python/tools/hit_rate_ds.py b/easy_rec/python/tools/hit_rate_ds.py index d04fba623..8c8ef1631 100644 --- a/easy_rec/python/tools/hit_rate_ds.py +++ b/easy_rec/python/tools/hit_rate_ds.py @@ -26,28 +26,29 @@ from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils import config_util, io_util from easy_rec.python.utils.config_util import process_multi_file_input_path +from easy_rec.python.utils.hit_rate_utils import ( # NOQA + compute_hitrate_batch, + load_graph, + reduce_hitrate, +) from easy_rec.python.utils.hive_utils import HiveUtils -from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch, load_graph, reduce_hitrate # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 -from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_train_worker_num_on_ds # NOQA +from easy_rec.python.utils.distribution_utils import ( # NOQA + set_tf_config_and_get_train_worker_num_on_ds, +) logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO + level=logging.INFO, ) tf.app.flags.DEFINE_string('item_emb_table', '', 'item embedding table name') tf.app.flags.DEFINE_string('gt_table', '', 'ground truth table name') -tf.app.flags.DEFINE_string( - 'hitrate_details_result', '', 'hitrate detail file path' -) -tf.app.flags.DEFINE_string( - 'total_hitrate_result', '', 'total hitrate result file path' -) +tf.app.flags.DEFINE_string('hitrate_details_result', '', 'hitrate detail file path') +tf.app.flags.DEFINE_string('total_hitrate_result', '', 'total hitrate result file path') tf.app.flags.DEFINE_string('pipeline_config_path', '', 'pipeline config path') tf.app.flags.DEFINE_integer('batch_size', 512, 'batch size') @@ -59,9 +60,7 @@ tf.app.flags.DEFINE_integer('timeout', '60', 'timeout') tf.app.flags.DEFINE_integer('num_interests', 1, 'max number of interests') tf.app.flags.DEFINE_string('gt_table_field_sep', '\t', 'gt_table_field_sep') -tf.app.flags.DEFINE_string( - 'item_emb_table_field_sep', '\t', 'item_emb_table_field_sep' -) +tf.app.flags.DEFINE_string('item_emb_table_field_sep', '\t', 'item_emb_table_field_sep') tf.app.flags.DEFINE_bool('is_on_ds', False, help='is on ds') FLAGS = tf.app.flags.FLAGS @@ -85,26 +84,37 @@ def compute_hitrate(g, gt_all, hitrate_writer, gt_table=None): for gt_record in gt_all: gt_record = list(gt_record) - hits, gt_count, src_ids, recall_ids, recall_distances, hitrates, bad_cases, bad_dists = \ - compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, FLAGS.num_interests, FLAGS.top_k) + ( + hits, + gt_count, + src_ids, + recall_ids, + recall_distances, + hitrates, + bad_cases, + bad_dists, + ) = compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, FLAGS.num_interests, FLAGS.top_k) total_hits += hits total_gt_count += gt_count src_ids = [str(ids) for ids in src_ids] hitrates = [str(hitrate) for hitrate in hitrates] topk_recalls = [','.join(str(x) for x in ids) for ids in recall_ids] - topk_dists = [ - ','.join('|'.join(str(x) for x in dist) for dist in dists) - for dists in recall_distances - ] + topk_dists = [','.join('|'.join(str(x) for x in dist) for dist in dists) for dists in recall_distances] bad_cases = [','.join(str(x) for x in bad_case) for bad_case in bad_cases] bad_dists = [','.join(str(x) for x in dist) for dist in bad_dists] hitrate_writer.write( '\n'.join( [ - '\t'.join(line) for line in zip( - src_ids, topk_recalls, topk_dists, hitrates, bad_cases, bad_dists + '\t'.join(line) + for line in zip( + src_ids, + topk_recalls, + topk_dists, + hitrates, + bad_cases, + bad_dists, ) ] ) @@ -115,7 +125,6 @@ def compute_hitrate(g, gt_all, hitrate_writer, gt_table=None): def gt_hdfs(gt_table, batch_size, gt_file_sep): - if '*' in gt_table or ',' in gt_table: file_paths = tf.gfile.Glob(gt_table.split(',')) elif tf.gfile.IsDirectory(gt_table): @@ -150,9 +159,7 @@ def main(): i_emb_table = FLAGS.item_emb_table gt_table = FLAGS.gt_table - pipeline_config = config_util.get_configs_from_pipeline_file( - FLAGS.pipeline_config_path - ) + pipeline_config = config_util.get_configs_from_pipeline_file(FLAGS.pipeline_config_path) logging.info('i_emb_table %s', i_emb_table) input_type = pipeline_config.data_config.input_type @@ -162,32 +169,21 @@ def main(): else: hive_utils = HiveUtils( data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input + hive_config=pipeline_config.hive_train_input, ) i_emb_table = hive_utils.get_table_location(i_emb_table) - g = load_graph( - i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, - FLAGS.knn_strict - ) + g = load_graph(i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, FLAGS.knn_strict) gl.set_tracker_mode(0) gl.set_field_delimiter(FLAGS.item_emb_table_field_sep) - cluster = tf.train.ClusterSpec( - { - 'ps': tf_config['cluster']['ps'], - 'worker': tf_config['cluster']['worker'] - } - ) + cluster = tf.train.ClusterSpec({'ps': tf_config['cluster']['ps'], 'worker': tf_config['cluster']['worker']}) server = tf.train.Server(cluster, job_name=job_name, task_index=task_index) if job_name == 'ps': server.join() else: - worker_hosts = [ - str(host.split(':')[0]) + ':888' + str(i) - for i, host in enumerate(tf_config['cluster']['worker']) - ] + worker_hosts = [str(host.split(':')[0]) + ':888' + str(i) for i, host in enumerate(tf_config['cluster']['worker'])] worker_hosts = ','.join(worker_hosts) g.init(task_index=task_index, task_count=worker_count, hosts=worker_hosts) # Your model, use g to do some operation, such as sampling @@ -198,26 +194,18 @@ def main(): gt_reader = HiveUtils( data_config=pipeline_config.data_config, hive_config=pipeline_config.hive_train_input, - selected_cols='*' + selected_cols='*', ) gt_all = gt_reader.hive_read_lines(gt_table, FLAGS.batch_size) if not tf.gfile.IsDirectory(hitrate_details_result): tf.gfile.MakeDirs(hitrate_details_result) - hitrate_details_result = os.path.join( - hitrate_details_result, 'part-%s' % task_index - ) + hitrate_details_result = os.path.join(hitrate_details_result, 'part-%s' % task_index) details_writer = tf.gfile.GFile(hitrate_details_result, 'w') print('Start compute hitrate...') - total_hits, total_gt_count = compute_hitrate( - g, gt_all, details_writer, gt_table - ) - var_total_hitrate, var_worker_count = reduce_hitrate( - cluster, total_hits, total_gt_count, task_index - ) + total_hits, total_gt_count = compute_hitrate(g, gt_all, details_writer, gt_table) + var_total_hitrate, var_worker_count = reduce_hitrate(cluster, total_hits, total_gt_count, task_index) - with tf.train.MonitoredTrainingSession( - master=server.target, is_chief=(task_index == 0) - ) as sess: + with tf.train.MonitoredTrainingSession(master=server.target, is_chief=(task_index == 0)) as sess: outs = sess.run([var_total_hitrate, var_worker_count]) # write after all workers have completed the calculation of hitrate. diff --git a/easy_rec/python/tools/hit_rate_pai.py b/easy_rec/python/tools/hit_rate_pai.py index 75e7697a5..db016ec56 100644 --- a/easy_rec/python/tools/hit_rate_pai.py +++ b/easy_rec/python/tools/hit_rate_pai.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================= """Evaluation of Top k hitrate.""" + from __future__ import absolute_import, division, print_function import sys @@ -20,8 +21,11 @@ import tensorflow as tf from easy_rec.python.utils import io_util - -from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch, load_graph, reduce_hitrate # NOQA +from easy_rec.python.utils.hit_rate_utils import ( # NOQA + compute_hitrate_batch, + load_graph, + reduce_hitrate, +) flags = tf.app.flags FLAGS = flags.FLAGS @@ -59,24 +63,35 @@ def compute_hitrate(g, gt_reader, hitrate_writer): while True: try: gt_record = gt_reader.read(FLAGS.batch_size) - hits, gt_count, src_ids, recall_ids, recall_distances, hitrates, bad_cases, bad_dists = \ - compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, FLAGS.num_interests, FLAGS.top_k) + ( + hits, + gt_count, + src_ids, + recall_ids, + recall_distances, + hitrates, + bad_cases, + bad_dists, + ) = compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, FLAGS.num_interests, FLAGS.top_k) total_hits += hits total_gt_count += gt_count topk_recalls = [','.join(str(x) for x in ids) for ids in recall_ids] - topk_dists = [ - ','.join(str(x) for x in dists) for dists in recall_distances - ] + topk_dists = [','.join(str(x) for x in dists) for dists in recall_distances] bad_cases = [','.join(str(x) for x in case) for case in bad_cases] bad_dists = [','.join(str(x) for x in dist) for dist in bad_dists] hitrate_writer.write( list( zip( - src_ids, topk_recalls, topk_dists, hitrates, bad_cases, bad_dists + src_ids, + topk_recalls, + topk_dists, + hitrates, + bad_cases, + bad_dists, ) ), - indices=[0, 1, 2, 3, 4, 5] + indices=[0, 1, 2, 3, 4, 5], ) except tf.python_io.OutOfRangeException: break @@ -89,48 +104,36 @@ def main(): if FLAGS.recall_type == 'u2i': i_emb_table, gt_table = input_tables g = load_graph( - i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, - FLAGS.knn_strict + i_emb_table, + FLAGS.emb_dim, + FLAGS.knn_metric, + FLAGS.timeout, + FLAGS.knn_strict, ) else: i_emb_table, gt_table = input_tables[-2], input_tables[-1] g = load_graph( - i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, - FLAGS.knn_strict + i_emb_table, + FLAGS.emb_dim, + FLAGS.knn_metric, + FLAGS.timeout, + FLAGS.knn_strict, ) hitrate_details_table, total_hitrate_table = FLAGS.outputs.split(',') - cluster = tf.train.ClusterSpec( - { - 'ps': FLAGS.ps_hosts.split(','), - 'worker': FLAGS.worker_hosts.split(',') - } - ) - server = tf.train.Server( - cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index - ) + cluster = tf.train.ClusterSpec({'ps': FLAGS.ps_hosts.split(','), 'worker': FLAGS.worker_hosts.split(',')}) + server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index) if FLAGS.job_name == 'ps': server.join() else: g.init(task_index=FLAGS.task_index, task_count=worker_count) - gt_reader = tf.python_io.TableReader( - gt_table, - slice_id=FLAGS.task_index, - slice_count=worker_count, - capacity=2048 - ) - details_writer = tf.python_io.TableWriter( - hitrate_details_table, slice_id=FLAGS.task_index - ) + gt_reader = tf.python_io.TableReader(gt_table, slice_id=FLAGS.task_index, slice_count=worker_count, capacity=2048) + details_writer = tf.python_io.TableWriter(hitrate_details_table, slice_id=FLAGS.task_index) print('Start compute hitrate...') total_hits, total_gt_count = compute_hitrate(g, gt_reader, details_writer) - var_total_hitrate, var_worker_count = reduce_hitrate( - cluster, total_hits, total_gt_count, FLAGS.task_index - ) + var_total_hitrate, var_worker_count = reduce_hitrate(cluster, total_hits, total_gt_count, FLAGS.task_index) - with tf.train.MonitoredTrainingSession( - master=server.target, is_chief=(FLAGS.task_index == 0) - ) as sess: + with tf.train.MonitoredTrainingSession(master=server.target, is_chief=(FLAGS.task_index == 0)) as sess: outs = sess.run([var_total_hitrate, var_worker_count]) # write after all workers have completed the calculation of hitrate. diff --git a/easy_rec/python/tools/pre_check.py b/easy_rec/python/tools/pre_check.py index 769b2769e..12a009b6c 100644 --- a/easy_rec/python/tools/pre_check.py +++ b/easy_rec/python/tools/pre_check.py @@ -9,30 +9,25 @@ from easy_rec.python.input.input import Input from easy_rec.python.utils import config_util, fg_util, io_util - -from easy_rec.python.utils.check_utils import check_env_and_input_path, check_sequence # NOQA +from easy_rec.python.utils.check_utils import ( # NOQA + check_env_and_input_path, + check_sequence, +) if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO -) -tf.app.flags.DEFINE_string( - 'pipeline_config_path', None, 'Path to pipeline config ' - 'file.' -) -tf.app.flags.DEFINE_multi_string( - 'data_input_path', None, help='data input path' + level=logging.INFO, ) +tf.app.flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') +tf.app.flags.DEFINE_multi_string('data_input_path', None, help='data input path') FLAGS = tf.app.flags.FLAGS -def _get_input_fn( - data_config, feature_configs, data_path=None, export_config=None -): +def _get_input_fn(data_config, feature_configs, data_path=None, export_config=None): """Build estimator input function. Args: @@ -63,16 +58,14 @@ def _get_input_fn( data_path, task_index=task_index, task_num=worker_num, - check_mode=True + check_mode=True, ) input_fn = input_obj.create_input(export_config) return input_fn def loda_pipeline_config(pipeline_config_path): - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path, False - ) + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) if pipeline_config.fg_json_path: fg_util.load_fg_json_to_config(pipeline_config) config_util.auto_expand_share_feature_configs(pipeline_config) @@ -83,21 +76,12 @@ def run_check(pipeline_config, input_path): logging.info('data_input_path: %s' % input_path) check_env_and_input_path(pipeline_config, input_path) feature_configs = config_util.get_compatible_feature_configs(pipeline_config) - eval_input_fn = _get_input_fn( - pipeline_config.data_config, feature_configs, input_path - ) - eval_spec = tf.estimator.EvalSpec( - name='val', - input_fn=eval_input_fn, - steps=None, - throttle_secs=10, - exporters=[] - ) - input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL - ).make_one_shot_iterator() + eval_input_fn = _get_input_fn(pipeline_config.data_config, feature_configs, input_path) + eval_spec = tf.estimator.EvalSpec(name='val', input_fn=eval_input_fn, steps=None, throttle_secs=10, exporters=[]) + input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() with tf.Session() as sess: try: - while (True): + while True: input_feas, input_lbls = input_iter.get_next() features = sess.run(input_feas) check_sequence(pipeline_config, features) @@ -112,8 +96,9 @@ def main(argv): if FLAGS.data_input_path: input_path = ','.join(FLAGS.data_input_path) else: - assert pipeline_config.train_input_path or pipeline_config.eval_input_path, \ - 'input_path should not be empty when checking!' + assert ( + pipeline_config.train_input_path or pipeline_config.eval_input_path + ), 'input_path should not be empty when checking!' input_path = pipeline_config.train_input_path + ',' + pipeline_config.eval_input_path run_check(pipeline_config, input_path) diff --git a/easy_rec/python/tools/predict_and_chk.py b/easy_rec/python/tools/predict_and_chk.py index 0247eddac..fe605d17a 100644 --- a/easy_rec/python/tools/predict_and_chk.py +++ b/easy_rec/python/tools/predict_and_chk.py @@ -13,54 +13,40 @@ try: import tensorflow as tf + tf.load_op_library(os.path.join(easy_rec.ops_dir, 'libembed_op.so')) except Exception as ex: logging.warning('exception: %s' % str(ex)) -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( - '--saved_model_dir', type=str, default=None, help='saved model directory' - ) - parser.add_argument( - '--input_path', type=str, default=None, help='input feature path' - ) + parser.add_argument('--saved_model_dir', type=str, default=None, help='saved model directory') + parser.add_argument('--input_path', type=str, default=None, help='input feature path') parser.add_argument('--save_path', type=str, default=None, help='save path') - parser.add_argument( - '--cmp_res_path', type=str, default=None, help='compare result path' - ) - parser.add_argument( - '--cmp_key', type=str, default='probs', help='compare key' - ) + parser.add_argument('--cmp_res_path', type=str, default=None, help='compare result path') + parser.add_argument('--cmp_key', type=str, default='probs', help='compare key') parser.add_argument( '--rtp_fea_id', type=int, default=-1, - help='rtp feature column index, default to the last column' + help='rtp feature column index, default to the last column', ) parser.add_argument('--tol', type=float, default=1e-5, help='tolerance') parser.add_argument( '--label_id', nargs='*', type=int, - help='the label column, which is to be excluded' + help='the label column, which is to be excluded', ) parser.add_argument( '--separator', type=str, default='', - help='separator between features, default to \\u0002' - ) - parser.add_argument( - '--rtp_separator', - type=str, - default='', - help='separator, default to \\u0001' + help='separator between features, default to \\u0002', ) + parser.add_argument('--rtp_separator', type=str, default='', help='separator, default to \\u0001') args = parser.parse_args() if not args.saved_model_dir: @@ -80,9 +66,7 @@ predictor = Predictor(args.saved_model_dir) if len(predictor.input_names) == 1: - assert len( - args.label_id - ) == 0, 'label_id should not be set if rtp feature format is used.' + assert len(args.label_id) == 0, 'label_id should not be set if rtp feature format is used.' with open(args.input_path, 'r') as fin: batch_input = [] @@ -90,10 +74,7 @@ line_str = line_str.strip() line_tok = line_str.split(args.rtp_separator) feature = line_tok[args.rtp_fea_id] - feature = [ - x for fid, x in enumerate(feature.split(args.separator)) - if fid not in args.label_id - ] + feature = [x for fid, x in enumerate(feature.split(args.separator)) if fid not in args.label_id] if 'features' in predictor.input_names: feature = args.separator.join(feature) batch_input.append(feature) @@ -113,8 +94,7 @@ for line_id, line_str in enumerate(fin): line_str = line_str.strip() line_pred = json.loads(line_str) - assert np.abs(line_pred[args.cmp_key] - output[line_id][ - args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( - line_id, - np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]) - ) + assert np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( + line_id, + np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]), + ) diff --git a/easy_rec/python/tools/read_kafka.py b/easy_rec/python/tools/read_kafka.py index bc239d74b..79af83c40 100644 --- a/easy_rec/python/tools/read_kafka.py +++ b/easy_rec/python/tools/read_kafka.py @@ -8,9 +8,7 @@ from kafka import KafkaConsumer from kafka.structs import TopicPartition -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -30,7 +28,7 @@ consumer = KafkaConsumer( group_id=args.group, bootstrap_servers=servers, - consumer_timeout_ms=args.timeout * 1000 + consumer_timeout_ms=args.timeout * 1000, ) if args.partitions is not None: @@ -39,18 +37,14 @@ partitions = consumer.partitions_for_topic(args.topic) logging.info('partitions: %s' % partitions) - topics = [ - TopicPartition(topic=args.topic, partition=part_id) - for part_id in partitions - ] + topics = [TopicPartition(topic=args.topic, partition=part_id) for part_id in partitions] consumer.assign(topics) consumer.seek_to_beginning() record_id = 0 for x in consumer: logging.info( - '%d: key=%s\toffset=%d\ttimestamp=%d\tlen=%d' % - (record_id, x.key, x.offset, x.timestamp, len(x.value)) + '%d: key=%s\toffset=%d\ttimestamp=%d\tlen=%d' % (record_id, x.key, x.offset, x.timestamp, len(x.value)) ) if args.save_dir is not None: save_path = os.path.join(args.save_dir, x.key) diff --git a/easy_rec/python/tools/split_model_pai.py b/easy_rec/python/tools/split_model_pai.py index 5bfe9c0a6..b7e005b15 100644 --- a/easy_rec/python/tools/split_model_pai.py +++ b/easy_rec/python/tools/split_model_pai.py @@ -28,9 +28,7 @@ tf.app.flags.DEFINE_string('user_fg_json_path', '', '') tf.app.flags.DEFINE_string('item_fg_json_path', '', '') -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') def search_pb(directory): @@ -130,9 +128,7 @@ def load_meta_graph_def(model_dir): output_tensor_names = {} variable_protos = {} - meta_graph_def = saved_model_utils.get_meta_graph_def( - model_dir, tf.saved_model.tag_constants.SERVING - ) + meta_graph_def = saved_model_utils.get_meta_graph_def(model_dir, tf.saved_model.tag_constants.SERVING) signatures = meta_graph_def.signature_def collections = meta_graph_def.collection_def @@ -151,9 +147,7 @@ def load_meta_graph_def(model_dir): # parse signature info for SavedModel for sig_name in signatures: - if signatures[ - sig_name - ].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: + if signatures[sig_name].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: tf.logging.info('[Signature] inputs:') for input_name in signatures[sig_name].inputs: input_tensor_shape = [] @@ -161,8 +155,11 @@ def load_meta_graph_def(model_dir): for dim in input_tensor.tensor_shape.dim: input_tensor_shape.append(int(dim.size)) tf.logging.info( - '"%s": %s; %s' % ( - input_name, _TYPE_TO_STRING[input_tensor.dtype], input_tensor_shape + '"%s": %s; %s' + % ( + input_name, + _TYPE_TO_STRING[input_tensor.dtype], + input_tensor_shape, ) ) input_tensor_names[input_name] = input_tensor.name @@ -173,9 +170,11 @@ def load_meta_graph_def(model_dir): for dim in output_tensor.tensor_shape.dim: output_tensor_shape.append(int(dim.size)) tf.logging.info( - '"%s": %s; %s' % ( - output_name, _TYPE_TO_STRING[output_tensor.dtype - ], output_tensor_shape + '"%s": %s; %s' + % ( + output_name, + _TYPE_TO_STRING[output_tensor.dtype], + output_tensor_shape, ) ) output_tensor_names[output_name] = output_tensor.name @@ -184,8 +183,13 @@ def load_meta_graph_def(model_dir): def export( - model_dir, meta_graph_def, variable_protos, input_tensor_names, - output_tensor_names, part_name, part_dir + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name, + part_dir, ): """Export subpart saved model. @@ -198,17 +202,10 @@ def export( part_name: subpart model name, user or item. part_dir: subpart model export directory. """ - output_tensor_names = { - x: output_tensor_names[x] - for x in output_tensor_names.keys() if part_name in x - } - output_node_names = [ - _node_name(output_tensor_names[x]) for x in output_tensor_names.keys() - ] - - inference_graph, variables_to_keep = extract_sub_graph( - meta_graph_def.graph_def, output_node_names, variable_protos - ) + output_tensor_names = {x: output_tensor_names[x] for x in output_tensor_names.keys() if part_name in x} + output_node_names = [_node_name(output_tensor_names[x]) for x in output_tensor_names.keys()] + + inference_graph, variables_to_keep = extract_sub_graph(meta_graph_def.graph_def, output_node_names, variable_protos) tf.reset_default_graph() with tf.Session() as sess: @@ -225,34 +222,28 @@ def export( signature_inputs = {} for input_name in input_tensor_names: try: - tensor_info = tf.saved_model.utils.build_tensor_info( - graph.get_tensor_by_name(input_tensor_names[input_name]) - ) + tensor_info = tf.saved_model.utils.build_tensor_info(graph.get_tensor_by_name(input_tensor_names[input_name])) signature_inputs[input_name] = tensor_info except Exception: print('ignore input: %s' % input_name) signature_outputs = {} for output_name in output_tensor_names: - tensor_info = tf.saved_model.utils.build_tensor_info( - graph.get_tensor_by_name(output_tensor_names[output_name]) - ) + tensor_info = tf.saved_model.utils.build_tensor_info(graph.get_tensor_by_name(output_tensor_names[output_name])) signature_outputs[output_name] = tensor_info - prediction_signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=signature_inputs, - outputs=signature_outputs, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME - ) + prediction_signature = tf.saved_model.signature_def_utils.build_signature_def( + inputs=signature_inputs, + outputs=signature_outputs, + method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, ) builder.add_meta_graph_and_variables( - sess, [tf.saved_model.tag_constants.SERVING], + sess, + [tf.saved_model.tag_constants.SERVING], signature_def_map={ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: - prediction_signature, - } + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_signature, + }, ) builder.save() config_path = os.path.join(model_dir, 'assets/pipeline.config') @@ -272,9 +263,12 @@ def export( def main(argv): model_dir = search_pb(FLAGS.model_dir) tf.logging.info('Loading meta graph...') - meta_graph_def, variable_protos, input_tensor_names, output_tensor_names = load_meta_graph_def( - model_dir - ) + ( + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + ) = load_meta_graph_def(model_dir) tf.logging.info('Exporting user part model...') export( model_dir, @@ -283,7 +277,7 @@ def main(argv): input_tensor_names, output_tensor_names, part_name='user', - part_dir=FLAGS.user_model_dir + part_dir=FLAGS.user_model_dir, ) tf.logging.info('Exporting item part model...') export( @@ -293,7 +287,7 @@ def main(argv): input_tensor_names, output_tensor_names, part_name='item', - part_dir=FLAGS.item_model_dir + part_dir=FLAGS.item_model_dir, ) diff --git a/easy_rec/python/tools/split_pdn_model_pai.py b/easy_rec/python/tools/split_pdn_model_pai.py index be61f637f..287d7eace 100644 --- a/easy_rec/python/tools/split_pdn_model_pai.py +++ b/easy_rec/python/tools/split_pdn_model_pai.py @@ -20,9 +20,7 @@ tf.app.flags.DEFINE_string('trigger_model_dir', '', '') tf.app.flags.DEFINE_string('sim_model_dir', '', '') -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') def search_pb(directory): @@ -122,9 +120,7 @@ def load_meta_graph_def(model_dir): output_tensor_names = {} variable_protos = {} - meta_graph_def = saved_model_utils.get_meta_graph_def( - model_dir, tf.saved_model.tag_constants.SERVING - ) + meta_graph_def = saved_model_utils.get_meta_graph_def(model_dir, tf.saved_model.tag_constants.SERVING) signatures = meta_graph_def.signature_def collections = meta_graph_def.collection_def @@ -143,9 +139,7 @@ def load_meta_graph_def(model_dir): # parse signature info for SavedModel for sig_name in signatures: - if signatures[ - sig_name - ].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: + if signatures[sig_name].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: tf.logging.info('[Signature] inputs:') for input_name in signatures[sig_name].inputs: input_tensor_shape = [] @@ -153,8 +147,11 @@ def load_meta_graph_def(model_dir): for dim in input_tensor.tensor_shape.dim: input_tensor_shape.append(int(dim.size)) tf.logging.info( - '"%s": %s; %s' % ( - input_name, _TYPE_TO_STRING[input_tensor.dtype], input_tensor_shape + '"%s": %s; %s' + % ( + input_name, + _TYPE_TO_STRING[input_tensor.dtype], + input_tensor_shape, ) ) input_tensor_names[input_name] = input_tensor.name @@ -165,9 +162,11 @@ def load_meta_graph_def(model_dir): for dim in output_tensor.tensor_shape.dim: output_tensor_shape.append(int(dim.size)) tf.logging.info( - '"%s": %s; %s' % ( - output_name, _TYPE_TO_STRING[output_tensor.dtype - ], output_tensor_shape + '"%s": %s; %s' + % ( + output_name, + _TYPE_TO_STRING[output_tensor.dtype], + output_tensor_shape, ) ) output_tensor_names[output_name] = output_tensor.name @@ -176,8 +175,13 @@ def load_meta_graph_def(model_dir): def export( - model_dir, meta_graph_def, variable_protos, input_tensor_names, - output_tensor_names, part_name, part_dir + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name, + part_dir, ): """Export subpart saved model. @@ -190,17 +194,10 @@ def export( part_name: subpart model name, user or item. part_dir: subpart model export directory. """ - output_tensor_names = { - x: output_tensor_names[x] - for x in output_tensor_names.keys() if part_name in x - } - output_node_names = [ - _node_name(output_tensor_names[x]) for x in output_tensor_names.keys() - ] - - inference_graph, variables_to_keep = extract_sub_graph( - meta_graph_def.graph_def, output_node_names, variable_protos - ) + output_tensor_names = {x: output_tensor_names[x] for x in output_tensor_names.keys() if part_name in x} + output_node_names = [_node_name(output_tensor_names[x]) for x in output_tensor_names.keys()] + + inference_graph, variables_to_keep = extract_sub_graph(meta_graph_def.graph_def, output_node_names, variable_protos) tf.reset_default_graph() with tf.Session() as sess: @@ -217,34 +214,28 @@ def export( signature_inputs = {} for input_name in input_tensor_names: try: - tensor_info = tf.saved_model.utils.build_tensor_info( - graph.get_tensor_by_name(input_tensor_names[input_name]) - ) + tensor_info = tf.saved_model.utils.build_tensor_info(graph.get_tensor_by_name(input_tensor_names[input_name])) signature_inputs[input_name] = tensor_info except Exception: print('ignore input: %s' % input_name) signature_outputs = {} for output_name in output_tensor_names: - tensor_info = tf.saved_model.utils.build_tensor_info( - graph.get_tensor_by_name(output_tensor_names[output_name]) - ) + tensor_info = tf.saved_model.utils.build_tensor_info(graph.get_tensor_by_name(output_tensor_names[output_name])) signature_outputs[output_name] = tensor_info - prediction_signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=signature_inputs, - outputs=signature_outputs, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME - ) + prediction_signature = tf.saved_model.signature_def_utils.build_signature_def( + inputs=signature_inputs, + outputs=signature_outputs, + method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, ) builder.add_meta_graph_and_variables( - sess, [tf.saved_model.tag_constants.SERVING], + sess, + [tf.saved_model.tag_constants.SERVING], signature_def_map={ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: - prediction_signature, - } + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_signature, + }, ) builder.save() config_path = os.path.join(model_dir, 'assets/pipeline.config') @@ -258,9 +249,12 @@ def export( def main(argv): model_dir = search_pb(FLAGS.model_dir) tf.logging.info('Loading meta graph...') - meta_graph_def, variable_protos, input_tensor_names, output_tensor_names = load_meta_graph_def( - model_dir - ) + ( + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + ) = load_meta_graph_def(model_dir) tf.logging.info('Exporting trigger part model...') export( model_dir, @@ -269,7 +263,7 @@ def main(argv): input_tensor_names, output_tensor_names, part_name='trigger_out', - part_dir=FLAGS.trigger_model_dir + part_dir=FLAGS.trigger_model_dir, ) tf.logging.info('Exporting sim part model...') export( @@ -279,7 +273,7 @@ def main(argv): input_tensor_names, output_tensor_names, part_name='sim_out', - part_dir=FLAGS.sim_model_dir + part_dir=FLAGS.sim_model_dir, ) diff --git a/easy_rec/python/tools/test_saved_model.py b/easy_rec/python/tools/test_saved_model.py index 6c0fa6024..9dbfd7a4b 100644 --- a/easy_rec/python/tools/test_saved_model.py +++ b/easy_rec/python/tools/test_saved_model.py @@ -13,7 +13,7 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO + level=logging.INFO, ) lookup_op_path = os.path.join(easy_rec.ops_dir, 'libkv_lookup.so') @@ -29,26 +29,18 @@ """ parser = argparse.ArgumentParser() - parser.add_argument( - '--saved_model_dir', type=str, default=None, help='saved model dir' - ) - parser.add_argument( - '--input_path', type=str, default=None, help='output dir' - ) + parser.add_argument('--saved_model_dir', type=str, default=None, help='saved model dir') + parser.add_argument('--input_path', type=str, default=None, help='output dir') parser.add_argument('--save_path', type=str, default=None, help='save path') parser.add_argument('--separator', type=str, default=',', help='separator') - parser.add_argument( - '--cmp_res_path', type=str, default=None, help='compare result path' - ) - parser.add_argument( - '--cmp_key', type=str, default='probs', help='compare key' - ) + parser.add_argument('--cmp_res_path', type=str, default=None, help='compare result path') + parser.add_argument('--cmp_key', type=str, default='probs', help='compare key') parser.add_argument('--tol', type=float, default=1e-5, help='tolerance') parser.add_argument( '--with_lbl', action='store_true', default=False, - help='whether the test data has label field' + help='whether the test data has label field', ) args = parser.parse_args() @@ -80,8 +72,7 @@ for line_id, line_str in enumerate(fin): line_str = line_str.strip() line_pred = json.loads(line_str) - assert np.abs(line_pred[args.cmp_key] - output[line_id][ - args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( - line_id, - np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]) - ) + assert np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( + line_id, + np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]), + ) diff --git a/easy_rec/python/tools/view_saved_model.py b/easy_rec/python/tools/view_saved_model.py index 21435db66..222ea696b 100644 --- a/easy_rec/python/tools/view_saved_model.py +++ b/easy_rec/python/tools/view_saved_model.py @@ -9,17 +9,13 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO + level=logging.INFO, ) if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( - '--input', type=str, default=None, help='saved model path' - ) - parser.add_argument( - '--output', type=str, default=None, help='saved model save path' - ) + parser.add_argument('--input', type=str, default=None, help='saved model path') + parser.add_argument('--output', type=str, default=None, help='saved model save path') args = parser.parse_args() assert args.input is not None and args.output is not None diff --git a/easy_rec/python/tools/write_kafka.py b/easy_rec/python/tools/write_kafka.py index be94f2690..71f6d2931 100644 --- a/easy_rec/python/tools/write_kafka.py +++ b/easy_rec/python/tools/write_kafka.py @@ -10,9 +10,7 @@ # from kafka.structs import TopicPartition -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -43,10 +41,10 @@ name=args.topic, num_partitions=1, replication_factor=1, - topic_configs={'max.message.bytes': 1024 * 1024 * 1024} + topic_configs={'max.message.bytes': 1024 * 1024 * 1024}, ) ], - validate_only=False + validate_only=False, ) logging.info('create increment save topic: %s' % args.topic) admin_clt.close() @@ -54,7 +52,7 @@ producer = KafkaProducer( bootstrap_servers=servers, request_timeout_ms=args.timeout * 1000, - api_version=(0, 10, 1) + api_version=(0, 10, 1), ) i = 1 diff --git a/easy_rec/python/train_eval.py b/easy_rec/python/train_eval.py index 57f3c4ee9..ab116de5c 100644 --- a/easy_rec/python/train_eval.py +++ b/easy_rec/python/train_eval.py @@ -9,23 +9,34 @@ from easy_rec.python.main import _train_and_evaluate_impl from easy_rec.python.protos.train_pb2 import DistributionStrategy - -from easy_rec.python.utils import config_util, ds_util, estimator_utils, fg_util, hpo_util # NOQA -from easy_rec.python.utils.config_util import process_neg_sampler_data_path, set_eval_input_path, set_train_input_path # NOQA +from easy_rec.python.utils import ( # NOQA + config_util, + ds_util, + estimator_utils, + fg_util, + hpo_util, +) +from easy_rec.python.utils.config_util import ( # NOQA + process_neg_sampler_data_path, + set_eval_input_path, + set_train_input_path, +) if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: import tensorflow.io.gfile as gfile -from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_train_worker_num_on_ds # NOQA +from easy_rec.python.utils.distribution_utils import ( # NOQA + set_tf_config_and_get_train_worker_num_on_ds, +) if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO + level=logging.INFO, ) if __name__ == '__main__': @@ -34,93 +45,71 @@ '--pipeline_config_path', type=str, default=None, - help='Path to pipeline config file.' + help='Path to pipeline config file.', ) parser.add_argument( '--continue_train', action='store_true', default=False, - help='continue train using existing model_dir' - ) - parser.add_argument( - '--hpo_param_path', - type=str, - default=None, - help='hyperparam tuning param path' + help='continue train using existing model_dir', ) + parser.add_argument('--hpo_param_path', type=str, default=None, help='hyperparam tuning param path') parser.add_argument( '--hpo_metric_save_path', type=str, default=None, - help='hyperparameter save metric path' + help='hyperparameter save metric path', ) parser.add_argument( '--model_dir', type=str, default=None, - help='will update the model_dir in pipeline_config' + help='will update the model_dir in pipeline_config', ) parser.add_argument( '--train_input_path', type=str, nargs='*', default=None, - help='train data input path' + help='train data input path', ) parser.add_argument( '--eval_input_path', type=str, nargs='*', default=None, - help='eval data input path' + help='eval data input path', ) parser.add_argument( '--fit_on_eval', action='store_true', default=False, - help='Fit evaluation data after fitting and evaluating train data' - ) - parser.add_argument( - '--fit_on_eval_steps', - type=int, - default=None, - help='Fit evaluation data steps' + help='Fit evaluation data after fitting and evaluating train data', ) + parser.add_argument('--fit_on_eval_steps', type=int, default=None, help='Fit evaluation data steps') parser.add_argument( '--fine_tune_checkpoint', type=str, default=None, - help='will update the train_config.fine_tune_checkpoint in pipeline_config' + help='will update the train_config.fine_tune_checkpoint in pipeline_config', ) parser.add_argument( '--edit_config_json', type=str, default=None, help='edit pipeline config str, example: {"model_dir":"experiments/",' - '"feature_config.feature[0].boundaries":[4,5,6,7]}' + '"feature_config.feature[0].boundaries":[4,5,6,7]}', ) parser.add_argument( '--ignore_finetune_ckpt_error', action='store_true', default=False, - help= - 'During incremental training, ignore the problem of missing fine_tune_checkpoint files' - ) - parser.add_argument( - '--odps_config', type=str, default=None, help='odps config path' - ) - parser.add_argument( - '--is_on_ds', action='store_true', default=False, help='is on ds' - ) - parser.add_argument( - '--check_mode', - action='store_true', - default=False, - help='is use check mode' - ) - parser.add_argument( - '--selected_cols', type=str, default=None, help='select input columns' + help='During incremental training, ignore the problem of missing fine_tune_checkpoint files', ) + parser.add_argument('--odps_config', type=str, default=None, help='odps config path') + parser.add_argument('--is_on_ds', action='store_true', default=False, help='is on ds') + parser.add_argument('--check_mode', action='store_true', default=False, help='is use check mode') + parser.add_argument('--selected_cols', type=str, default=None, help='select input columns') parser.add_argument('--gpu', type=str, default=None, help='gpu id') args, extra_args = parser.parse_known_args() @@ -135,9 +124,7 @@ config_util.parse_extra_config_param(extra_args, edit_config_json) if args.pipeline_config_path is not None: - pipeline_config = config_util.get_configs_from_pipeline_file( - args.pipeline_config_path, False - ) + pipeline_config = config_util.get_configs_from_pipeline_file(args.pipeline_config_path, False) if args.selected_cols: pipeline_config.data_config.selected_cols = args.selected_cols if args.model_dir: @@ -163,9 +150,7 @@ os.environ['ODPS_CONFIG_FILE_PATH'] = args.odps_config if len(edit_config_json) > 0: - fine_tune_checkpoint = edit_config_json.get('train_config', {}).get( - 'fine_tune_checkpoint', None - ) + fine_tune_checkpoint = edit_config_json.get('train_config', {}).get('fine_tune_checkpoint', None) if fine_tune_checkpoint: ckpt_path = estimator_utils.get_latest_checkpoint_from_checkpoint_path( args.fine_tune_checkpoint, args.ignore_finetune_ckpt_error @@ -187,7 +172,7 @@ estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.SokStrategy, ]: estimator_utils.init_hvd() estimator_utils.init_sok() @@ -198,13 +183,11 @@ hpo_params = hpo_config['param'] config_util.edit_config(pipeline_config, hpo_params) config_util.auto_expand_share_feature_configs(pipeline_config) - _train_and_evaluate_impl( - pipeline_config, args.continue_train, args.check_mode - ) + _train_and_evaluate_impl(pipeline_config, args.continue_train, args.check_mode) hpo_util.save_eval_metrics( pipeline_config.model_dir, metric_save_path=args.hpo_metric_save_path, - has_evaluator=False + has_evaluator=False, ) else: config_util.auto_expand_share_feature_configs(pipeline_config) @@ -213,7 +196,7 @@ args.continue_train, args.check_mode, fit_on_eval=args.fit_on_eval, - fit_on_eval_steps=args.fit_on_eval_steps + fit_on_eval_steps=args.fit_on_eval_steps, ) else: raise ValueError('pipeline_config_path should not be empty when training!') diff --git a/easy_rec/python/utils/activation.py b/easy_rec/python/utils/activation.py index 7f4c296db..0abdd32dc 100644 --- a/easy_rec/python/utils/activation.py +++ b/easy_rec/python/utils/activation.py @@ -30,7 +30,7 @@ def dice(_x, axis=-1, epsilon=1e-9, name='dice', training=True): 'alpha_' + name, _x.get_shape()[-1], initializer=tf.constant_initializer(0.0), - dtype=tf.float32 + dtype=tf.float32, ) inputs_normed = tf.layers.batch_normalization( inputs=_x, @@ -38,7 +38,7 @@ def dice(_x, axis=-1, epsilon=1e-9, name='dice', training=True): epsilon=epsilon, center=False, scale=False, - training=training + training=training, ) x_p = tf.sigmoid(inputs_normed) return alphas * (1.0 - x_p) * _x + x_p * _x @@ -58,9 +58,7 @@ def gelu(x, name='gelu'): `x` with the GELU activation applied. """ with tf.name_scope(name): - cdf = 0.5 * ( - 1.0 + tf.tanh((np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3)))) - ) + cdf = 0.5 * (1.0 + tf.tanh((np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) return x * cdf diff --git a/easy_rec/python/utils/check_utils.py b/easy_rec/python/utils/check_utils.py index 2c6282643..c2901054c 100644 --- a/easy_rec/python/utils/check_utils.py +++ b/easy_rec/python/utils/check_utils.py @@ -10,20 +10,20 @@ def check_split(line, sep, requried_field_num, field_name=''): - assert sep, 'must have separator.' + ( - ' field: %s.' % field_name - ) if field_name else '' + assert sep, 'must have separator.' + (' field: %s.' % field_name) if field_name else '' for one_line in line: field_num = len(one_line.split(sep)) if field_name: - assert_info = 'sep[%s] maybe invalid. field_num=%d, required_num=%d, field: %s, value: %s, ' \ - 'please check separator and data.' % \ - (sep, field_num, requried_field_num, field_name, one_line) + assert_info = ( + 'sep[%s] maybe invalid. field_num=%d, required_num=%d, field: %s, value: %s, ' + 'please check separator and data.' % (sep, field_num, requried_field_num, field_name, one_line) + ) else: - assert_info = 'sep[%s] maybe invalid. field_num=%d, required_num=%d, current line is: %s, ' \ - 'please check separator and data.' % \ - (sep, field_num, requried_field_num, one_line) + assert_info = ( + 'sep[%s] maybe invalid. field_num=%d, required_num=%d, current line is: %s, ' + 'please check separator and data.' % (sep, field_num, requried_field_num, one_line) + ) assert field_num == requried_field_num, assert_info return True @@ -33,8 +33,14 @@ def check_string_to_number(field_vals, field_name): try: float(val) except: # noqa: E722 - assert False, 'StringToNumber ERROR: cannot convert string_to_number, field: %s, value: %s. ' \ - 'please check data.' % (field_name, val) + assert False, ( + 'StringToNumber ERROR: cannot convert string_to_number, field: %s, value: %s. ' + 'please check data.' + % ( + field_name, + val, + ) + ) return True @@ -47,16 +53,21 @@ def check_sequence(pipeline_config_path, features): if not seq_att_maps: return for seq_att_map in seq_att_maps: - assert len(seq_att_map.key) == len(seq_att_map.hist_seq), \ - 'The size of hist_seq must equal to the size of key in one seq_att_map.' + assert len(seq_att_map.key) == len( + seq_att_map.hist_seq + ), 'The size of hist_seq must equal to the size of key in one seq_att_map.' size_list = [] for hist_seq in seq_att_map.hist_seq: cur_seq_size = len(features[hist_seq].values) size_list.append(cur_seq_size) hist_seqs = ' '.join(seq_att_map.hist_seq) - assert len(set(size_list)) == 1, \ - 'SequenceFeature Error: The size in [%s] should be consistent. Please check input: [%s].' % \ - (hist_seqs, hist_seqs) + assert len(set(size_list)) == 1, ( + 'SequenceFeature Error: The size in [%s] should be consistent. Please check input: [%s].' + % ( + hist_seqs, + hist_seqs, + ) + ) def check_env_and_input_path(pipeline_config, input_path): @@ -72,8 +83,10 @@ def check_env_and_input_path(pipeline_config, input_path): ] if input_type in ignore_input_list: return True - assert_info = 'Current InputType is %s, InputPath is %s. Please check InputType and InputPath.' % \ - (input_type_name, input_path) + assert_info = 'Current InputType is %s, InputPath is %s. Please check InputType and InputPath.' % ( + input_type_name, + input_path, + ) if input_type_name.startswith('Odps'): # is on pai for path in input_path.split(','): diff --git a/easy_rec/python/utils/config_util.py b/easy_rec/python/utils/config_util.py index 2e4536fa1..fbdccc3f4 100644 --- a/easy_rec/python/utils/config_util.py +++ b/easy_rec/python/utils/config_util.py @@ -57,9 +57,7 @@ def get_configs_from_pipeline_file(pipeline_config_path, auto_expand=True): if isinstance(pipeline_config_path, pipeline_pb2.EasyRecConfig): return pipeline_config_path - assert tf.gfile.Exists( - pipeline_config_path - ), 'pipeline_config_path [%s] not exists' % pipeline_config_path + assert tf.gfile.Exists(pipeline_config_path), 'pipeline_config_path [%s] not exists' % pipeline_config_path pipeline_config = pipeline_pb2.EasyRecConfig() with tf.gfile.GFile(pipeline_config_path, 'r') as f: @@ -157,9 +155,7 @@ def create_pipeline_proto_from_configs(configs): return pipeline_config -def save_pipeline_config( - pipeline_config, directory, filename='pipeline.config' -): +def save_pipeline_config(pipeline_config, directory, filename='pipeline.config'): """Saves a pipeline config text file to disk. Args: @@ -177,10 +173,26 @@ def save_pipeline_config( def _get_basic_types(): dtypes = [ - bool, int, str, float, - type(u''), np.float16, np.float32, np.float64, np.char, np.byte, np.uint8, - np.int8, np.int16, np.uint16, np.uint32, np.int32, np.uint64, np.int64, - bool, str + bool, + int, + str, + float, + type(''), + np.float16, + np.float32, + np.float64, + np.char, + np.byte, + np.uint8, + np.int8, + np.int16, + np.uint16, + np.uint32, + np.int32, + np.uint64, + np.int64, + bool, + str, ] if six.PY2: dtypes.append(long) # noqa: F821 @@ -224,7 +236,7 @@ def _get_attr(obj, attr, only_last=False): for obj in objs: if '[' in key: pos = key.find('[') - name, cond = key[:pos], key[pos + 1:] + name, cond = key[:pos], key[pos + 1 :] cond = cond[:-1] update_objs = getattr(obj, name) # select all update_objs @@ -242,7 +254,7 @@ def _get_attr(obj, attr, only_last=False): sid = 0 else: sid = int(sid) - eid = cond[(colon_pos + 1):] + eid = cond[(colon_pos + 1) :] if len(eid) == 0: eid = len(update_objs) else: @@ -268,7 +280,7 @@ def _get_attr(obj, attr, only_last=False): '<=': lambda x, y: x <= y, '<': lambda x, y: x < y, '>': lambda x, y: x > y, - '=': lambda x, y: x == y + '=': lambda x, y: x == y, } cond_key = None cond_val = None @@ -277,7 +289,7 @@ def _get_attr(obj, attr, only_last=False): tmp_pos = cond.rfind(op) if tmp_pos != -1: cond_key = cond[:tmp_pos] - cond_val = cond[(tmp_pos + len(op)):] + cond_val = cond[(tmp_pos + len(op)) :] op_func = op_func_map[op] break @@ -285,9 +297,7 @@ def _get_attr(obj, attr, only_last=False): assert cond_val is not None, 'invalid cond: %s' % cond for tid, update_obj in enumerate(update_objs): - tmp, tmp_parent, _, _ = _get_attr( - update_obj, cond_key, only_last=True - ) + tmp, tmp_parent, _, _ = _get_attr(update_obj, cond_key, only_last=True) cond_val = _type_convert(tmp, cond_val, tmp_parent) @@ -381,9 +391,7 @@ def add_boundaries_to_config(pipeline_config, tables): if feature_name in feature_boundaries_info: if feature_config.feature_type != feature_config.SequenceFeature: logging.info( - 'feature = {0}, type = {1}, will turn to RawFeature.'.format( - feature_name, feature_config.feature_type - ) + 'feature = {0}, type = {1}, will turn to RawFeature.'.format(feature_name, feature_config.feature_type) ) feature_config.feature_type = feature_config.RawFeature feature_config.hash_bucket_size = 0 @@ -408,12 +416,9 @@ def parse_time(time_data): Return: timestamp: int """ - if isinstance(time_data, str) or isinstance(time_data, type(u'')): + if isinstance(time_data, str) or isinstance(time_data, type('')): if len(time_data) == 17: - return int( - datetime.datetime.strptime(time_data, - '%Y%m%d %H:%M:%S').strftime('%s') - ) + return int(datetime.datetime.strptime(time_data, '%Y%m%d %H:%M:%S').strftime('%s')) elif len(time_data) == 10: return int(time_data) else: @@ -475,19 +480,12 @@ def get_model_dir_path(pipeline_config): def set_train_input_path(pipeline_config, train_input_path): if pipeline_config.WhichOneof('train_path') == 'hive_train_input': if isinstance(train_input_path, list): - assert len( - train_input_path - ) <= 1, 'only support one hive_train_input.table_name when hive input' + assert len(train_input_path) <= 1, 'only support one hive_train_input.table_name when hive input' pipeline_config.hive_train_input.table_name = train_input_path[0] else: - assert len( - train_input_path.split(',') - ) <= 1, 'only support one hive_train_input.table_name when hive input' + assert len(train_input_path.split(',')) <= 1, 'only support one hive_train_input.table_name when hive input' pipeline_config.hive_train_input.table_name = train_input_path - logging.info( - 'update hive_train_input.table_name to %s' % - pipeline_config.hive_train_input.table_name - ) + logging.info('update hive_train_input.table_name to %s' % pipeline_config.hive_train_input.table_name) elif pipeline_config.WhichOneof('train_path') == 'kafka_train_input': if isinstance(train_input_path, list): @@ -504,28 +502,19 @@ def set_train_input_path(pipeline_config, train_input_path): pipeline_config.train_input_path = ','.join(train_input_path) else: pipeline_config.train_input_path = train_input_path - logging.info( - 'update train_input_path to %s' % pipeline_config.train_input_path - ) + logging.info('update train_input_path to %s' % pipeline_config.train_input_path) return pipeline_config def set_eval_input_path(pipeline_config, eval_input_path): if pipeline_config.WhichOneof('eval_path') == 'hive_eval_input': if isinstance(eval_input_path, list): - assert len( - eval_input_path - ) <= 1, 'only support one hive_eval_input.table_name when hive input' + assert len(eval_input_path) <= 1, 'only support one hive_eval_input.table_name when hive input' pipeline_config.hive_eval_input.table_name = eval_input_path[0] else: - assert len( - eval_input_path.split(',') - ) <= 1, 'only support one hive_eval_input.table_name when hive input' + assert len(eval_input_path.split(',')) <= 1, 'only support one hive_eval_input.table_name when hive input' pipeline_config.hive_eval_input.table_name = eval_input_path - logging.info( - 'update hive_eval_input.table_name to %s' % - pipeline_config.hive_eval_input.table_name - ) + logging.info('update hive_eval_input.table_name to %s' % pipeline_config.hive_eval_input.table_name) elif pipeline_config.WhichOneof('eval_path') == 'parquet_eval_input': if isinstance(eval_input_path, list): pipeline_config.parquet_eval_input = ','.join(eval_input_path) @@ -541,9 +530,7 @@ def set_eval_input_path(pipeline_config, eval_input_path): pipeline_config.eval_input_path = ','.join(eval_input_path) else: pipeline_config.eval_input_path = eval_input_path - logging.info( - 'update eval_input_path to %s' % pipeline_config.eval_input_path - ) + logging.info('update eval_input_path to %s' % pipeline_config.eval_input_path) return pipeline_config @@ -569,47 +556,39 @@ def process_neg_sampler_data_path(pipeline_config): return hive_util = HiveUtils( data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input + hive_config=pipeline_config.hive_train_input, ) sampler_type = pipeline_config.data_config.WhichOneof('sampler') sampler_config = getattr(pipeline_config.data_config, sampler_type) if hasattr(sampler_config, 'input_path'): - sampler_config.input_path = process_data_path( - sampler_config.input_path, hive_util - ) + sampler_config.input_path = process_data_path(sampler_config.input_path, hive_util) if hasattr(sampler_config, 'user_input_path'): - sampler_config.user_input_path = process_data_path( - sampler_config.user_input_path, hive_util - ) + sampler_config.user_input_path = process_data_path(sampler_config.user_input_path, hive_util) if hasattr(sampler_config, 'item_input_path'): - sampler_config.item_input_path = process_data_path( - sampler_config.item_input_path, hive_util - ) + sampler_config.item_input_path = process_data_path(sampler_config.item_input_path, hive_util) if hasattr(sampler_config, 'pos_edge_input_path'): - sampler_config.pos_edge_input_path = process_data_path( - sampler_config.pos_edge_input_path, hive_util - ) + sampler_config.pos_edge_input_path = process_data_path(sampler_config.pos_edge_input_path, hive_util) if hasattr(sampler_config, 'hard_neg_edge_input_path'): - sampler_config.hard_neg_edge_input_path = process_data_path( - sampler_config.hard_neg_edge_input_path, hive_util - ) + sampler_config.hard_neg_edge_input_path = process_data_path(sampler_config.hard_neg_edge_input_path, hive_util) def parse_extra_config_param(extra_args, edit_config_json): arg_num = len(extra_args) arg_id = 0 while arg_id < arg_num: - if extra_args[arg_id].startswith('--data_config.') or \ - extra_args[arg_id].startswith('--train_config.') or \ - extra_args[arg_id].startswith('--feature_config.') or \ - extra_args[arg_id].startswith('--model_config.') or \ - extra_args[arg_id].startswith('--export_config.') or \ - extra_args[arg_id].startswith('--eval_config.'): + if ( + extra_args[arg_id].startswith('--data_config.') + or extra_args[arg_id].startswith('--train_config.') + or extra_args[arg_id].startswith('--feature_config.') + or extra_args[arg_id].startswith('--model_config.') + or extra_args[arg_id].startswith('--export_config.') + or extra_args[arg_id].startswith('--eval_config.') + ): tmp_arg = extra_args[arg_id][2:] if '=' in tmp_arg: sep_pos = tmp_arg.find('=') k = tmp_arg[:sep_pos] - v = tmp_arg[(sep_pos + 1):] + v = tmp_arg[(sep_pos + 1) :] v = v.strip(' "\'') edit_config_json[k] = v arg_id += 1 @@ -625,12 +604,8 @@ def parse_extra_config_param(extra_args, edit_config_json): def process_multi_file_input_path(sampler_config_input_path): - if '*' in sampler_config_input_path: - input_path = ','.join( - file_path - for file_path in tf.gfile.Glob(sampler_config_input_path.split(',')) - ) + input_path = ','.join(file_path for file_path in tf.gfile.Glob(sampler_config_input_path.split(','))) else: input_path = sampler_config_input_path diff --git a/easy_rec/python/utils/constant.py b/easy_rec/python/utils/constant.py index 7b886d3d3..b2d6e3044 100644 --- a/easy_rec/python/utils/constant.py +++ b/easy_rec/python/utils/constant.py @@ -35,8 +35,7 @@ def enable_avx_str_split(): def has_avx_str_split(): - return ENABLE_AVX_STR_SPLIT in os.environ and os.environ[ENABLE_AVX_STR_SPLIT - ] == '1' + return ENABLE_AVX_STR_SPLIT in os.environ and os.environ[ENABLE_AVX_STR_SPLIT] == '1' def disable_avx_str_split(): diff --git a/easy_rec/python/utils/convert_rtp_fg.py b/easy_rec/python/utils/convert_rtp_fg.py index 5792b1bbe..9bcb150cc 100644 --- a/easy_rec/python/utils/convert_rtp_fg.py +++ b/easy_rec/python/utils/convert_rtp_fg.py @@ -9,20 +9,21 @@ from google.protobuf import text_format from easy_rec.python.protos.dataset_pb2 import DatasetConfig +from easy_rec.python.protos.feature_config_pb2 import ( # NOQA + FeatureConfig, + FeatureGroupConfig, + WideOrDeep, +) from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig from easy_rec.python.utils import config_util -from easy_rec.python.protos.feature_config_pb2 import FeatureConfig, FeatureGroupConfig, WideOrDeep # NOQA - if tf.__version__ >= '2.0': tf = tf.compat.v1 MAX_HASH_BUCKET_SIZE = 9223372036854775807 -def _gen_raw_config( - feature, input_field, feature_config, is_multi, curr_embed_dim -): +def _gen_raw_config(feature, input_field, feature_config, is_multi, curr_embed_dim): if 'bucketize_boundaries' in feature: if is_multi: input_field.input_type = DatasetConfig.STRING @@ -30,9 +31,7 @@ def _gen_raw_config( else: input_field.input_type = DatasetConfig.INT32 feature_config.feature_type = feature_config.IdFeature - feature_config.num_buckets = len( - feature['bucketize_boundaries'].split(',') - ) + 1 + feature_config.num_buckets = len(feature['bucketize_boundaries'].split(',')) + 1 feature_config.embedding_dim = curr_embed_dim else: feature_config.feature_type = feature_config.RawFeature @@ -57,10 +56,7 @@ def _set_hash_bucket(feature, feature_config, input_field): feature_config.hash_bucket_size = feature['hash_bucket_size'] if feature_config.hash_bucket_size > 10000000: if 'max_partitions' not in feature: - logging.error( - 'it is suggested to set max_partitions > 1 for large hash buckets[%s]' - % feature['feature_name'] - ) + logging.error('it is suggested to set max_partitions > 1 for large hash buckets[%s]' % feature['feature_name']) sys.exit(1) if feature.get('filter_freq', -1) >= 0: feature_config.ev_params.filter_freq = feature['filter_freq'] @@ -86,16 +82,14 @@ def process_features( pipeline_config, embedding_dim, incol_separator, - is_sequence=False + is_sequence=False, ): feature_config = FeatureConfig() feature_config.input_names.append(feature_name) feature_config.separator = incol_separator input_field = DatasetConfig.Field() input_field.input_name = feature_name - curr_embed_dim = feature.get( - 'embedding_dimension', feature.get('embedding_dim', embedding_dim) - ) + curr_embed_dim = feature.get('embedding_dimension', feature.get('embedding_dim', embedding_dim)) curr_combiner = feature.get('combiner', 'sum') if feature.get('is_cache', False): logging.info('will cache %s' % feature_name) @@ -133,9 +127,7 @@ def process_features( elif feature_type == 'lookup_feature': need_discrete = feature.get('needDiscrete', True) if not need_discrete: - _gen_raw_config( - feature, input_field, feature_config, is_multi, curr_embed_dim - ) + _gen_raw_config(feature, input_field, feature_config, is_multi, curr_embed_dim) else: feature_config.feature_type = feature_config.TagFeature if feature.get('needWeighting', False): @@ -144,9 +136,7 @@ def process_features( _set_hash_bucket(feature, feature_config, input_field) feature_config.combiner = curr_combiner elif feature_type == 'raw_feature': - _gen_raw_config( - feature, input_field, feature_config, is_multi, curr_embed_dim - ) + _gen_raw_config(feature, input_field, feature_config, is_multi, curr_embed_dim) elif feature_type == 'match_feature': need_discrete = feature.get('needDiscrete', True) if feature.get('matchType', '') == 'multihit': @@ -160,9 +150,7 @@ def process_features( feature_config.combiner = curr_combiner else: assert 'bucketize_boundaries' not in feature - _gen_raw_config( - feature, input_field, feature_config, is_multi, curr_embed_dim - ) + _gen_raw_config(feature, input_field, feature_config, is_multi, curr_embed_dim) elif feature_type == 'combo_feature': feature_config.feature_type = feature_config.TagFeature _set_hash_bucket(feature, feature_config, input_field) @@ -190,9 +178,7 @@ def process_features( if 'extra_combo_info' in feature: extra_combo_info = feature['extra_combo_info'] feature_names = extra_combo_info.get('feature_names', []) - assert len( - feature_names - ) >= 1, 'The feature number for ComboFeature must be greater than 2.' + assert len(feature_names) >= 1, 'The feature number for ComboFeature must be greater than 2.' combo_feature_config = FeatureConfig() combo_feature_config.input_names.append(feature_name) @@ -200,15 +186,10 @@ def process_features( combo_feature_config.input_names.append(fea_name) final_feature_name = 'combo__' + '_'.join(combo_feature_config.input_names) - final_feature_name = extra_combo_info.get( - 'final_feature_name', final_feature_name - ) + final_feature_name = extra_combo_info.get('final_feature_name', final_feature_name) combo_feature_config.feature_name = final_feature_name combo_feature_config.feature_type = combo_feature_config.ComboFeature - curr_embed_dim = extra_combo_info.get( - 'embedding_dimension', - extra_combo_info.get('embedding_dim', embedding_dim) - ) + curr_embed_dim = extra_combo_info.get('embedding_dimension', extra_combo_info.get('embedding_dim', embedding_dim)) curr_combiner = extra_combo_info.get('combiner', 'mean') combo_feature_config.embedding_dim = curr_embed_dim combo_feature_config.combiner = curr_combiner @@ -222,9 +203,7 @@ def process_features( return pipeline_config -def load_input_field_and_feature_config( - rtp_fg, label_fields, embedding_dim=16, incol_separator='\003' -): +def load_input_field_and_feature_config(rtp_fg, label_fields, embedding_dim=16, incol_separator='\003'): embedding_dim = rtp_fg.get('embedding_dim', embedding_dim) logging.info('embedding_dim = %s' % embedding_dim) logging.info('label_fields = %s' % ','.join(label_fields)) @@ -247,8 +226,12 @@ def load_input_field_and_feature_config( feature_type = feature['feature_type'] feature_name = feature['feature_name'] pipeline_config = process_features( - feature_type, feature_name, feature, pipeline_config, embedding_dim, - incol_separator + feature_type, + feature_name, + feature, + pipeline_config, + embedding_dim, + incol_separator, ) elif 'sequence_name' in feature: logging.info('Set sequence_features group later.') @@ -264,13 +247,10 @@ def load_input_field_and_feature_config( pipeline_config, embedding_dim, incol_separator, - is_sequence=True + is_sequence=True, ) except Exception: - logging.info( - 'convert feature[%s] exception[%s]' % - (str(feature), traceback.format_exc()) - ) + logging.info('convert feature[%s] exception[%s]' % (str(feature), traceback.format_exc())) sys.exit(1) return pipeline_config @@ -288,7 +268,7 @@ def convert_rtp_fg( eval_input_path=None, selected_cols='', input_type='OdpsRTPInput', - is_async=False + is_async=False, ): with tf.gfile.GFile(rtp_fg, 'r') as fin: rtp_fg = json.load(fin) @@ -307,9 +287,7 @@ def convert_rtp_fg( logging.info('model_path = %s' % model_path) logging.info('edit_config_json = %s' % edit_config_json) - pipeline_config = load_input_field_and_feature_config( - rtp_fg, label_fields, embedding_dim, incol_separator - ) + pipeline_config = load_input_field_and_feature_config(rtp_fg, label_fields, embedding_dim, incol_separator) pipeline_config.model_dir = model_dir pipeline_config.data_config.separator = separator if selected_cols: @@ -353,7 +331,7 @@ def convert_rtp_fg( } """ % ( 'adam_optimizer' if not is_async else 'adam_async_optimizer', - 'true' if not is_async else 'false' + 'true' if not is_async else 'false', ) text_format.Merge(train_config_str, pipeline_config) @@ -431,7 +409,7 @@ def convert_rtp_fg( 'i': 'item', 'ctx': 'combo', 'q': 'combo', - 'comb': 'combo' + 'comb': 'combo', } for feature in rtp_features: feature_name = feature['feature_name'].strip() @@ -464,22 +442,28 @@ def convert_rtp_fg( multi_tower_config_str = ' multi_tower {\n' for group_name in feature_groups: - multi_tower_config_str += """ + multi_tower_config_str += ( + """ towers { input: "%s" dnn { hidden_units: [256, 192, 128] } } - """ % group_name + """ + % group_name + ) - multi_tower_config_str = multi_tower_config_str + """ + multi_tower_config_str = ( + multi_tower_config_str + + """ final_dnn { hidden_units: [192, 128, 64] } l2_regularization: 1e-4 } """ + ) text_format.Merge(multi_tower_config_str, pipeline_config.model_config) pipeline_config.model_config.embedding_regularization = 1e-5 @@ -506,13 +490,16 @@ def convert_rtp_fg( esmm_config_str = ' esmm {\n' for group_name in feature_groups: - esmm_config_str += """ + esmm_config_str += ( + """ groups { input: "%s" dnn { hidden_units: [256, 128, 96, 64] } - }""" % group_name + }""" + % group_name + ) esmm_config_str += """ ctr_tower { @@ -542,7 +529,10 @@ def convert_rtp_fg( } } l2_regularization: 1e-6 - }""" % (label_fields[0], label_fields[1]) + }""" % ( + label_fields[0], + label_fields[1], + ) text_format.Merge(esmm_config_str, pipeline_config.model_config) pipeline_config.model_config.embedding_regularization = 5e-5 elif model_type == 'dbmtl': @@ -608,7 +598,10 @@ def convert_rtp_fg( } l2_regularization: 1e-6 } - """ % (label_fields[0], label_fields[1]) + """ % ( + label_fields[0], + label_fields[1], + ) text_format.Merge(dbmtl_config_str, pipeline_config.model_config) pipeline_config.model_config.embedding_regularization = 5e-6 @@ -618,14 +611,16 @@ def convert_rtp_fg( metrics_set { auc {} } - """, pipeline_config.eval_config + """, + pipeline_config.eval_config, ) text_format.Merge( """ export_config { multi_placeholder: false } - """, pipeline_config + """, + pipeline_config, ) if edit_config_json: diff --git a/easy_rec/python/utils/dag.py b/easy_rec/python/utils/dag.py index f16191997..acfabe103 100644 --- a/easy_rec/python/utils/dag.py +++ b/easy_rec/python/utils/dag.py @@ -69,7 +69,6 @@ def rename_edges(self, old_task_name, new_task_name, graph=None): if not graph: graph = self.graph for node, edges in graph.items(): - if node == old_task_name: graph[new_task_name] = copy(edges) del graph[old_task_name] @@ -110,11 +109,7 @@ def all_downstreams(self, node, graph=None): nodes_seen.add(downstream_node) nodes.append(downstream_node) i += 1 - return list( - filter( - lambda node: node in nodes_seen, self.topological_sort(graph=graph) - ) - ) + return list(filter(lambda node: node in nodes_seen, self.topological_sort(graph=graph))) def all_leaves(self, graph=None): """Return a list of all leaves (nodes with no downstreams).""" @@ -145,9 +140,7 @@ def independent_nodes(self, graph=None): if graph is None: graph = self.graph - dependent_nodes = set( - node for dependents in graph.values() for node in dependents - ) + dependent_nodes = set(node for dependents in graph.values() for node in dependents) return [node for node in graph.keys() if node not in dependent_nodes] def validate(self, graph=None): diff --git a/easy_rec/python/utils/distribution_utils.py b/easy_rec/python/utils/distribution_utils.py index f06e265a4..43015e2fd 100644 --- a/easy_rec/python/utils/distribution_utils.py +++ b/easy_rec/python/utils/distribution_utils.py @@ -10,25 +10,26 @@ from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import estimator_utils - -from easy_rec.python.utils.estimator_utils import chief_to_master, master_to_chief # NOQA +from easy_rec.python.utils.estimator_utils import ( # NOQA + chief_to_master, + master_to_chief, +) DistributionStrategyMap = { '': DistributionStrategy.NoStrategy, 'ps': DistributionStrategy.PSStrategy, 'ess': DistributionStrategy.ExascaleStrategy, 'mirrored': DistributionStrategy.MirroredStrategy, - 'collective': DistributionStrategy.CollectiveAllReduceStrategy + 'collective': DistributionStrategy.CollectiveAllReduceStrategy, } -def set_distribution_config( - pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy -): +def set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy): if distribute_strategy in [ - DistributionStrategy.PSStrategy, DistributionStrategy.MirroredStrategy, + DistributionStrategy.PSStrategy, + DistributionStrategy.MirroredStrategy, DistributionStrategy.CollectiveAllReduceStrategy, - DistributionStrategy.ExascaleStrategy + DistributionStrategy.ExascaleStrategy, ]: pipeline_config.train_config.sync_replicas = False pipeline_config.train_config.train_distribute = distribute_strategy @@ -43,12 +44,9 @@ def set_tf_config_and_get_train_worker_num( task_index, job_name, distribute_strategy=DistributionStrategy.NoStrategy, - eval_method='none' + eval_method='none', ): - logging.info( - 'set_tf_config_and_get_train_worker_num: distribute_strategy = %d' % - distribute_strategy - ) + logging.info('set_tf_config_and_get_train_worker_num: distribute_strategy = %d' % distribute_strategy) worker_hosts = worker_hosts.split(',') ps_hosts = ps_hosts.split(',') if ps_hosts else [] @@ -57,17 +55,18 @@ def set_tf_config_and_get_train_worker_num( print('Original TF_CONFIG=%s' % os.environ.get('TF_CONFIG', '')) print( - 'worker_hosts=%s ps_hosts=%s task_index=%d job_name=%s' % - (','.join(worker_hosts), ','.join(ps_hosts), task_index, job_name) + 'worker_hosts=%s ps_hosts=%s task_index=%d job_name=%s' + % (','.join(worker_hosts), ','.join(ps_hosts), task_index, job_name) ) print('eval_method=%s' % eval_method) if distribute_strategy == DistributionStrategy.MirroredStrategy: assert total_worker_num == 1, 'mirrored distribute strategy only need 1 worker' elif distribute_strategy in [ - DistributionStrategy.NoStrategy, DistributionStrategy.PSStrategy, + DistributionStrategy.NoStrategy, + DistributionStrategy.PSStrategy, DistributionStrategy.CollectiveAllReduceStrategy, - DistributionStrategy.ExascaleStrategy + DistributionStrategy.ExascaleStrategy, ]: cluster, task_type, task_index_ = estimator_utils.parse_tf_config() train_worker_num = 0 @@ -85,18 +84,16 @@ def set_tf_config_and_get_train_worker_num( del cluster['evaluator'] tf_config = { 'cluster': cluster, - 'task': { - 'type': task_type, - 'index': task_index_ - } + 'task': {'type': task_type, 'index': task_index_}, } os.environ['TF_CONFIG'] = json.dumps(tf_config) else: # backward compatibility, if user does not assign one evaluator in # -Dcluster, we use first worker for chief, second for evaluation train_worker_num = total_worker_num - 1 - assert train_worker_num > 0, 'in distribution mode worker num must be greater than 1, ' \ - 'the second worker will be used as evaluator' + assert train_worker_num > 0, ( + 'in distribution mode worker num must be greater than 1, ' 'the second worker will be used as evaluator' + ) if len(worker_hosts) > 1: cluster = {'chief': [worker_hosts[0]], 'worker': worker_hosts[2:]} if distribute_strategy != DistributionStrategy.NoStrategy: @@ -107,10 +104,7 @@ def set_tf_config_and_get_train_worker_num( os.environ['TF_CONFIG'] = json.dumps( { 'cluster': cluster, - 'task': { - 'type': job_name, - 'index': task_index - } + 'task': {'type': job_name, 'index': task_index}, } ) elif job_name == 'worker': @@ -118,30 +112,21 @@ def set_tf_config_and_get_train_worker_num( os.environ['TF_CONFIG'] = json.dumps( { 'cluster': cluster, - 'task': { - 'type': 'chief', - 'index': 0 - } + 'task': {'type': 'chief', 'index': 0}, } ) elif task_index == 1: os.environ['TF_CONFIG'] = json.dumps( { 'cluster': cluster, - 'task': { - 'type': 'evaluator', - 'index': 0 - } + 'task': {'type': 'evaluator', 'index': 0}, } ) else: os.environ['TF_CONFIG'] = json.dumps( { 'cluster': cluster, - 'task': { - 'type': job_name, - 'index': task_index - 2 - } + 'task': {'type': job_name, 'index': task_index - 2}, } ) else: @@ -162,18 +147,12 @@ def set_tf_config_and_get_train_worker_num( if task_type == 'evaluator': tf_config = { 'cluster': cluster, - 'task': { - 'type': 'worker', - 'index': train_worker_num - 2 - } + 'task': {'type': 'worker', 'index': train_worker_num - 2}, } else: tf_config = { 'cluster': cluster, - 'task': { - 'type': task_type, - 'index': task_index_ - } + 'task': {'type': task_type, 'index': task_index_}, } os.environ['TF_CONFIG'] = json.dumps(tf_config) else: @@ -185,31 +164,17 @@ def set_tf_config_and_get_train_worker_num( os.environ['TF_CONFIG'] = json.dumps( { 'cluster': cluster, - 'task': { - 'type': job_name, - 'index': task_index - } + 'task': {'type': job_name, 'index': task_index}, } ) else: if task_index == 0: - os.environ['TF_CONFIG'] = json.dumps( - { - 'cluster': cluster, - 'task': { - 'type': 'chief', - 'index': 0 - } - } - ) + os.environ['TF_CONFIG'] = json.dumps({'cluster': cluster, 'task': {'type': 'chief', 'index': 0}}) else: os.environ['TF_CONFIG'] = json.dumps( { 'cluster': cluster, - 'task': { - 'type': 'worker', - 'index': task_index - 1 - } + 'task': {'type': 'worker', 'index': task_index - 1}, } ) if eval_method == 'none': @@ -219,15 +184,11 @@ def set_tf_config_and_get_train_worker_num( # change chief to master, will evaluate on master chief_to_master() else: - assert distribute_strategy == '', 'invalid distribute_strategy %s'\ - % distribute_strategy + assert distribute_strategy == '', 'invalid distribute_strategy %s' % distribute_strategy cluster, task_type, task_index = estimator_utils.parse_tf_config() print('Final TF_CONFIG = %s' % os.environ.get('TF_CONFIG', '')) tf.logging.info('TF_CONFIG %s' % os.environ.get('TF_CONFIG', '')) - tf.logging.info( - 'distribute_stategy %s, train_worker_num: %d' % - (distribute_strategy, train_worker_num) - ) + tf.logging.info('distribute_stategy %s, train_worker_num: %d' % (distribute_strategy, train_worker_num)) # remove pai chief-worker waiting strategy # which is conflicted with worker waiting strategy in easyrec @@ -240,9 +201,7 @@ def set_tf_config_and_get_train_worker_num_on_ds(): if 'TF_CONFIG' not in os.environ: return tf_config = json.loads(os.environ['TF_CONFIG']) - if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and ( - 'evaluator' not in tf_config['cluster'] - ): + if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and ('evaluator' not in tf_config['cluster']): easyrec_tf_config = dict() easyrec_tf_config['cluster'] = {} easyrec_tf_config['task'] = {} @@ -250,12 +209,10 @@ def set_tf_config_and_get_train_worker_num_on_ds(): easyrec_tf_config['cluster']['chief'] = [tf_config['cluster']['worker'][0]] easyrec_tf_config['cluster']['worker'] = tf_config['cluster']['worker'][2:] - if tf_config['task']['type'] == 'worker' and tf_config['task']['index' - ] == 0: + if tf_config['task']['type'] == 'worker' and tf_config['task']['index'] == 0: easyrec_tf_config['task']['type'] = 'chief' easyrec_tf_config['task']['index'] = 0 - elif tf_config['task']['type'] == 'worker' and tf_config['task']['index' - ] == 1: + elif tf_config['task']['type'] == 'worker' and tf_config['task']['index'] == 1: easyrec_tf_config['task']['type'] = 'evaluator' easyrec_tf_config['task']['index'] = 0 elif tf_config['task']['type'] == 'worker': @@ -270,9 +227,7 @@ def set_tf_config_and_get_train_worker_num_on_ds(): def set_tf_config_and_get_distribute_eval_worker_num_on_ds(): assert 'TF_CONFIG' in os.environ, "'TF_CONFIG' must in os.environ" tf_config = json.loads(os.environ['TF_CONFIG']) - if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and ( - 'evaluator' not in tf_config['cluster'] - ): + if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and ('evaluator' not in tf_config['cluster']): easyrec_tf_config = dict() easyrec_tf_config['cluster'] = {} easyrec_tf_config['task'] = {} @@ -280,8 +235,7 @@ def set_tf_config_and_get_distribute_eval_worker_num_on_ds(): easyrec_tf_config['cluster']['chief'] = [tf_config['cluster']['worker'][0]] easyrec_tf_config['cluster']['worker'] = tf_config['cluster']['worker'][1:] - if tf_config['task']['type'] == 'worker' and tf_config['task']['index' - ] == 0: + if tf_config['task']['type'] == 'worker' and tf_config['task']['index'] == 0: easyrec_tf_config['task']['type'] = 'chief' easyrec_tf_config['task']['index'] = 0 elif tf_config['task']['type'] == 'worker': diff --git a/easy_rec/python/utils/ds_util.py b/easy_rec/python/utils/ds_util.py index f2d051b21..9b2117c22 100644 --- a/easy_rec/python/utils/ds_util.py +++ b/easy_rec/python/utils/ds_util.py @@ -27,8 +27,7 @@ def cache_ckpt(pipeline_config): # there is no need to cache if remote directories are mounted return - if estimator_utils.is_ps() or estimator_utils.is_chief( - ) or estimator_utils.is_master(): + if estimator_utils.is_ps() or estimator_utils.is_chief() or estimator_utils.is_master(): tmpdir = os.path.dirname(fine_tune_ckpt_path.replace('hdfs://', '')) tmpdir = os.path.join('/tmp/experiments', tmpdir) logging.info('will cache fine_tune_ckpt to local dir: %s' % tmpdir) @@ -41,7 +40,7 @@ def cache_ckpt(pipeline_config): meta_files = [x for x in src_files if '.data-' not in x] if estimator_utils.is_ps(): _, _, ps_id = estimator_utils.parse_tf_config() - ps_id = (ps_id % len(data_files)) + ps_id = ps_id % len(data_files) data_files = data_files[ps_id:] + data_files[:ps_id] src_files = meta_files + data_files else: @@ -51,9 +50,7 @@ def cache_ckpt(pipeline_config): dst_path = os.path.join(tmpdir, os.path.basename(src_path)) logging.info('will copy %s to local path %s' % (src_path, dst_path)) try: - output = subprocess.check_output( - 'hadoop fs -get %s %s' % (src_path, dst_path), shell=True - ) + output = subprocess.check_output('hadoop fs -get %s %s' % (src_path, dst_path), shell=True) logging.info('copy succeed: %s' % output) except Exception: logging.warning('exception: %s' % traceback.format_exc()) diff --git a/easy_rec/python/utils/embedding_utils.py b/easy_rec/python/utils/embedding_utils.py index cb61dd177..064ce5043 100644 --- a/easy_rec/python/utils/embedding_utils.py +++ b/easy_rec/python/utils/embedding_utils.py @@ -58,7 +58,6 @@ def set_embedding_parallel(): def is_embedding_parallel(): - global embedding_parallel return embedding_parallel diff --git a/easy_rec/python/utils/estimator_utils.py b/easy_rec/python/utils/estimator_utils.py index a89b9ad90..7801f4000 100644 --- a/easy_rec/python/utils/estimator_utils.py +++ b/easy_rec/python/utils/estimator_utils.py @@ -17,14 +17,15 @@ from tensorflow.python.framework import errors_impl, meta_graph, ops from tensorflow.python.ops import array_ops from tensorflow.python.platform import gfile -from tensorflow.python.training.summary_io import SummaryWriterCache - -from easy_rec.python.utils import constant, embedding_utils, shape_utils - from tensorflow.python.training import basic_session_run_hooks, session_run_hook # NOQA from tensorflow.python.training.basic_session_run_hooks import SecondOrStepTimer # NOQA +from tensorflow.python.training.summary_io import SummaryWriterCache -from easy_rec.python.ops.incr_record import get_sparse_indices, kv_resource_incr_gather # NOQA +from easy_rec.python.ops.incr_record import ( # NOQA + get_sparse_indices, + kv_resource_incr_gather, +) +from easy_rec.python.utils import constant, embedding_utils, shape_utils try: import horovod.tensorflow as hvd @@ -84,30 +85,26 @@ def __init__(self, num_worker, is_chief, model_dir): def begin(self): """Count the number of workers and masters, and setup barrier queue.""" tf.logging.info('number workers(including master) = %d' % self._num_worker) - with tf.device( - tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0) - ): + with tf.device(tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0)): self._queue = tf.FIFOQueue( capacity=self._num_worker, dtypes=[tf.float32], shapes=[()], name='exit_counter', - shared_name='exit_counter' + shared_name='exit_counter', ) self._signal_que = tf.FIFOQueue( capacity=self._num_worker, dtypes=[tf.string], shapes=[()], name='exit_counter_signal', - shared_name='exit_counter_signal' + shared_name='exit_counter_signal', ) self._enque = self._queue.enqueue(1.0) self._que_size = self._queue.size() self._deque = self._queue.dequeue() if self._is_chief: - self._flag_file = os.path.join( - self._model_dir, 'atexit_sync_' + str(int(time.time())) - ) + self._flag_file = os.path.join(self._model_dir, 'atexit_sync_' + str(int(time.time()))) self._send = self._signal_que.enqueue([self._flag_file]) else: self._recv = self._signal_que.dequeue() @@ -133,10 +130,7 @@ def end(self, session): while que_size < self._num_worker: que_size = session.run(self._que_size) time.sleep(5) - tf.logging.info( - 'waiting for other worker to exit, finished %d, total %d' % - (que_size, self._num_worker) - ) + tf.logging.info('waiting for other worker to exit, finished %d, total %d' % (que_size, self._num_worker)) # prepare on_exit synchronize base on self._flag_file if self._is_chief: for i in range(self._num_worker - 1): @@ -145,9 +139,7 @@ def end(self, session): self._flag_file = session.run(self._recv) def _check_flag_file(is_chief, flag_file): - logging.info( - '_check_flag_file: is_chief = %d flag_file=%s' % (is_chief, flag_file) - ) + logging.info('_check_flag_file: is_chief = %d flag_file=%s' % (is_chief, flag_file)) if is_chief: with gfile.GFile(flag_file, 'w') as fout: fout.write('atexit time: %d' % int(time.time())) @@ -156,9 +148,8 @@ def _check_flag_file(is_chief, flag_file): time.sleep(1) from atexit import register - register( - _check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file - ) + + register(_check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file) logging.info('ExitBarrier passed') @@ -187,30 +178,26 @@ def __init__(self, num_worker, is_chief, model_dir, metric_ops=None): def begin(self): """Count the number of workers and masters, and setup barrier queue.""" tf.logging.info('number workers(including master) = %d' % self._num_worker) - with tf.device( - tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0) - ): + with tf.device(tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0)): self._queue = tf.FIFOQueue( capacity=self._num_worker, dtypes=[tf.float32], shapes=[()], name='exit_counter', - shared_name='exit_counter' + shared_name='exit_counter', ) self._signal_que = tf.FIFOQueue( capacity=self._num_worker, dtypes=[tf.string], shapes=[()], name='exit_counter_signal', - shared_name='exit_counter_signal' + shared_name='exit_counter_signal', ) self._enque = self._queue.enqueue(1.0) self._que_size = self._queue.size() self._deque = self._queue.dequeue() if self._is_chief: - self._flag_file = os.path.join( - self._model_dir, 'atexit_sync_' + str(int(time.time())) - ) + self._flag_file = os.path.join(self._model_dir, 'atexit_sync_' + str(int(time.time()))) self._send = self._signal_que.enqueue([self._flag_file]) else: self._recv = self._signal_que.dequeue() @@ -236,10 +223,7 @@ def end(self, session): while que_size < self._num_worker: que_size = session.run(self._que_size) time.sleep(5) - tf.logging.info( - 'waiting for other worker to exit, finished %d, total %d' % - (que_size, self._num_worker) - ) + tf.logging.info('waiting for other worker to exit, finished %d, total %d' % (que_size, self._num_worker)) # prepare on_exit synchronize base on self._flag_file if self._is_chief: self.eval_result = session.run(self.metric_ops) @@ -249,9 +233,7 @@ def end(self, session): self._flag_file = session.run(self._recv) def _check_flag_file(is_chief, flag_file): - logging.info( - '_check_flag_file: is_chief = %d flag_file=%s' % (is_chief, flag_file) - ) + logging.info('_check_flag_file: is_chief = %d flag_file=%s' % (is_chief, flag_file)) if is_chief: with gfile.GFile(flag_file, 'w') as fout: fout.write('atexit time: %d' % int(time.time())) @@ -260,16 +242,14 @@ def _check_flag_file(is_chief, flag_file): time.sleep(1) from atexit import register - register( - _check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file - ) + + register(_check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file) session.run(self.metric_ops) logging.info('ExitBarrier passed') class ProgressHook(SessionRunHook): - def __init__(self, num_steps, filename, is_chief): """Initializes a `ProgressHook`. @@ -290,11 +270,7 @@ def before_run(self, run_context): if self._is_chief: return tf.train.SessionRunArgs([tf.train.get_global_step()]) - def after_run( - self, - run_context, # pylint: disable=unused-argument - run_values - ): + def after_run(self, run_context, run_values): # pylint: disable=unused-argument if self._is_chief: global_step = run_values.results[0] curr_progress = global_step / self._num_steps @@ -326,7 +302,7 @@ def __init__( listeners=None, write_graph=True, data_offset_var=None, - increment_save_config=None + increment_save_config=None, ): """Initializes a `CheckpointSaverHook`. @@ -355,7 +331,7 @@ def __init__( saver=saver, checkpoint_basename=checkpoint_basename, scaffold=scaffold, - listeners=listeners + listeners=listeners, ) self._cuda_profile_start = 0 self._cuda_profile_stop = 0 @@ -368,34 +344,28 @@ def __init__( if increment_save_config is not None: self._kafka_timeout_ms = os.environ.get('KAFKA_TIMEOUT', 600) * 1000 logging.info('KAFKA_TIMEOUT: %dms' % self._kafka_timeout_ms) - self._kafka_max_req_size = os.environ.get( - 'KAFKA_MAX_REQ_SIZE', 1024 * 1024 * 64 - ) + self._kafka_max_req_size = os.environ.get('KAFKA_MAX_REQ_SIZE', 1024 * 1024 * 64) logging.info('KAFKA_MAX_REQ_SIZE: %d' % self._kafka_max_req_size) - self._kafka_max_msg_size = os.environ.get( - 'KAFKA_MAX_MSG_SIZE', 1024 * 1024 * 1024 - ) + self._kafka_max_msg_size = os.environ.get('KAFKA_MAX_MSG_SIZE', 1024 * 1024 * 1024) logging.info('KAFKA_MAX_MSG_SIZE: %d' % self._kafka_max_msg_size) self._dense_name_to_ids = embedding_utils.get_dense_name_to_ids() self._sparse_name_to_ids = embedding_utils.get_sparse_name_to_ids() - with gfile.GFile( - os.path.join(checkpoint_dir, constant.DENSE_UPDATE_VARIABLES), 'w' - ) as fout: + with gfile.GFile(os.path.join(checkpoint_dir, constant.DENSE_UPDATE_VARIABLES), 'w') as fout: json.dump(self._dense_name_to_ids, fout, indent=2) save_secs = increment_save_config.dense_save_secs save_steps = increment_save_config.dense_save_steps self._dense_timer = SecondOrStepTimer( every_secs=save_secs if save_secs > 0 else None, - every_steps=save_steps if save_steps > 0 else None + every_steps=save_steps if save_steps > 0 else None, ) save_secs = increment_save_config.sparse_save_secs save_steps = increment_save_config.sparse_save_steps self._sparse_timer = SecondOrStepTimer( every_secs=save_secs if save_secs > 0 else None, - every_steps=save_steps if save_steps > 0 else None + every_steps=save_steps if save_steps > 0 else None, ) self._dense_timer.update_last_triggered_step(0) @@ -407,23 +377,20 @@ def __init__( for sparse_var, indice_dtype in sparse_train_vars: with ops.control_dependencies([tf.train.get_global_step()]): with ops.colocate_with(sparse_var): - sparse_indice = get_sparse_indices( - var_name=sparse_var.op.name, ktype=indice_dtype - ) + sparse_indice = get_sparse_indices(var_name=sparse_var.op.name, ktype=indice_dtype) # sparse_indice = sparse_indice.global_indices self._sparse_indices.append(sparse_indice) if 'EmbeddingVariable' in str(type(sparse_var)): self._sparse_values.append( kv_resource_incr_gather( - sparse_var._handle, sparse_indice, - np.zeros(sparse_var.shape.as_list(), dtype=np.float32) + sparse_var._handle, + sparse_indice, + np.zeros(sparse_var.shape.as_list(), dtype=np.float32), ) ) # sparse_var.sparse_read(sparse_indice)) else: - self._sparse_values.append( - array_ops.gather(sparse_var, sparse_indice) - ) + self._sparse_values.append(array_ops.gather(sparse_var, sparse_indice)) self._kafka_producer = None self._incr_save_dir = None @@ -434,7 +401,7 @@ def __init__( admin_clt = KafkaAdminClient( bootstrap_servers=increment_save_config.kafka.server, request_timeout_ms=self._kafka_timeout_ms, - api_version_auto_timeout_ms=self._kafka_timeout_ms + api_version_auto_timeout_ms=self._kafka_timeout_ms, ) if self._topic not in admin_clt.list_topics(): admin_clt.create_topics( @@ -443,10 +410,10 @@ def __init__( name=self._topic, num_partitions=1, replication_factor=1, - topic_configs={'max.message.bytes': self._kafka_max_msg_size} + topic_configs={'max.message.bytes': self._kafka_max_msg_size}, ) ], - validate_only=False + validate_only=False, ) logging.info('create increment save topic: %s' % self._topic) admin_clt.close() @@ -456,7 +423,7 @@ def __init__( bootstrap_servers=servers, max_request_size=self._kafka_max_req_size, api_version_auto_timeout_ms=self._kafka_timeout_ms, - request_timeout_ms=self._kafka_timeout_ms + request_timeout_ms=self._kafka_timeout_ms, ) elif increment_save_config.HasField('fs'): fs = increment_save_config.fs @@ -469,13 +436,9 @@ def __init__( if not gfile.IsDirectory(self._incr_save_dir): gfile.MakeDirs(self._incr_save_dir) elif increment_save_config.HasField('datahub'): - raise NotImplementedError( - 'datahub increment saving is in development.' - ) + raise NotImplementedError('datahub increment saving is in development.') else: - raise ValueError( - 'incr_update not specified correctly, must be oneof: kafka,fs' - ) + raise ValueError('incr_update not specified correctly, must be oneof: kafka,fs') self._debug_save_update = increment_save_config.debug_save_update else: @@ -490,7 +453,8 @@ def after_create_session(self, session, coord): # add variables at begin. Graph is finalized after all begin calls. tf.train.write_graph( tf.get_default_graph().as_graph_def(add_shapes=True), - self._checkpoint_dir, 'graph.pbtxt' + self._checkpoint_dir, + 'graph.pbtxt', ) saver_def = self._get_saver().saver_def if self._get_saver() else None graph = tf.get_default_graph() @@ -511,9 +475,7 @@ def before_run(self, run_context): # pylint: disable=unused-argument def _send_dense(self, global_step, session): dense_train_vars = ops.get_collection(constant.DENSE_UPDATE_VARIABLES) dense_train_vals = session.run(dense_train_vars) - logging.info( - 'global_step=%d, increment save dense variables' % global_step - ) + logging.info('global_step=%d, increment save dense variables' % global_step) # build msg header msg_num = len(dense_train_vals) @@ -531,18 +493,11 @@ def _send_dense(self, global_step, session): if self._kafka_producer is not None: msg_key = 'dense_update_%d' % global_step - send_res = self._kafka_producer.send( - self._topic, bytes_buf, key=msg_key.encode('utf-8') - ) - logging.info( - 'kafka send dense: %d exception: %s' % - (global_step, send_res.exception) - ) + send_res = self._kafka_producer.send(self._topic, bytes_buf, key=msg_key.encode('utf-8')) + logging.info('kafka send dense: %d exception: %s' % (global_step, send_res.exception)) if self._incr_save_dir is not None: - save_path = os.path.join( - self._incr_save_dir, 'dense_update_%d' % global_step - ) + save_path = os.path.join(self._incr_save_dir, 'dense_update_%d' % global_step) with gfile.GFile(save_path, 'wb') as fout: fout.write(bytes_buf) save_flag = save_path + '.done' @@ -558,10 +513,7 @@ def _send_dense(self, global_step, session): with gfile.GFile(save_path, 'wb') as fout: fout.write(bytes_buf) - logging.info( - 'global_step=%d, increment update dense variables, msg_num=%d' % - (global_step, msg_num) - ) + logging.info('global_step=%d, increment update dense variables, msg_num=%d' % (global_step, msg_num)) def _send_sparse(self, global_step, session): sparse_train_vars = ops.get_collection(constant.SPARSE_UPDATE_VARIABLES) @@ -573,16 +525,12 @@ def _send_sparse(self, global_step, session): sparse_val_res = [sparse_res[i + msg_num] for i in sel_ids] sparse_train_vars = [sparse_train_vars[i][0] for i in sel_ids] - sel_embed_ids = [ - self._sparse_name_to_ids[x.name] for x in sparse_train_vars - ] + sel_embed_ids = [self._sparse_name_to_ids[x.name] for x in sparse_train_vars] msg_num = len(sel_ids) if msg_num == 0: - logging.warning( - 'there are no sparse updates, will skip this send: %d' % global_step - ) + logging.warning('there are no sparse updates, will skip this send: %d' % global_step) return # build msg header @@ -594,9 +542,7 @@ def _send_sparse(self, global_step, session): bytes_buf = np.array(msg_header, dtype=np.int32).tobytes() # build msg body - for tmp_id, tmp_key, tmp_val, tmp_var in zip( - sel_embed_ids, sparse_key_res, sparse_val_res, sparse_train_vars - ): + for tmp_id, tmp_key, tmp_val, tmp_var in zip(sel_embed_ids, sparse_key_res, sparse_val_res, sparse_train_vars): # for non kv embedding variables, add partition offset to tmp_key if 'EmbeddingVariable' not in str(type(tmp_var)): if tmp_var._save_slice_info is not None: @@ -605,17 +551,11 @@ def _send_sparse(self, global_step, session): bytes_buf += tmp_val.tobytes() if self._kafka_producer is not None: msg_key = 'sparse_update_%d' % global_step - send_res = self._kafka_producer.send( - self._topic, bytes_buf, key=msg_key.encode('utf-8') - ) - logging.info( - 'kafka send sparse: %d %s' % (global_step, send_res.exception) - ) + send_res = self._kafka_producer.send(self._topic, bytes_buf, key=msg_key.encode('utf-8')) + logging.info('kafka send sparse: %d %s' % (global_step, send_res.exception)) if self._incr_save_dir is not None: - save_path = os.path.join( - self._incr_save_dir, 'sparse_update_%d' % global_step - ) + save_path = os.path.join(self._incr_save_dir, 'sparse_update_%d' % global_step) with gfile.GFile(save_path, 'wb') as fout: fout.write(bytes_buf) save_flag = save_path + '.done' @@ -669,9 +609,7 @@ def _save(self, session, step): for x in save_data_offset: if x: data_offset_json.update(json.loads(x)) - save_offset_path = os.path.join( - self._checkpoint_dir, 'model.ckpt-%d.offset' % step - ) + save_offset_path = os.path.join(self._checkpoint_dir, 'model.ckpt-%d.offset' % step) with gfile.GFile(save_offset_path, 'w') as fout: json.dump(data_offset_json, fout) @@ -679,34 +617,28 @@ def _save(self, session, step): session, self._save_path, global_step=step, - write_meta_graph=self._write_graph + write_meta_graph=self._write_graph, ) self._summary_writer.add_session_log( - tf.SessionLog( - status=tf.SessionLog.CHECKPOINT, checkpoint_path=self._save_path - ), step + tf.SessionLog(status=tf.SessionLog.CHECKPOINT, checkpoint_path=self._save_path), + step, ) should_stop = False for l in self._listeners: # noqa: E741 if l.after_save(session, step): - logging.info( - 'A CheckpointSaverListener requested that training be stopped. ' - 'listener: {}'.format(l) - ) + logging.info('A CheckpointSaverListener requested that training be stopped. ' 'listener: {}'.format(l)) should_stop = True return should_stop def end(self, session): global_step = session.run(self._global_step_tensor) super(CheckpointSaverHook, self).end(session) - if self._dense_timer is not None and \ - global_step != self._dense_timer.last_triggered_step(): + if self._dense_timer is not None and global_step != self._dense_timer.last_triggered_step(): self._dense_timer.update_last_triggered_step(global_step) self._send_dense(global_step, session) - if self._sparse_timer is not None and \ - global_step != self._sparse_timer.last_triggered_step(): + if self._sparse_timer is not None and global_step != self._sparse_timer.last_triggered_step(): self._sparse_timer.update_last_triggered_step(global_step) self._send_sparse(global_step, session) @@ -740,8 +672,7 @@ def begin(self): assign_ops.append(var.assign(var_data)) else: logging.error( - 'variable [%s] shape not match %r vs %r' % - (var.name.split(':')[0], var_shape, list(var_data.shape)) + 'variable [%s] shape not match %r vs %r' % (var.name.split(':')[0], var_shape, list(var_data.shape)) ) has_shape_unmatch = True elif 'Momentum' not in var_name and 'global_step' not in var_name: @@ -753,8 +684,7 @@ def begin(self): for var_name in sorted(vars_not_inited.keys()): f.write('%s:%s\n' % (var_name, vars_not_inited[var_name])) assert not has_shape_unmatch, 'exist variable shape not match, restore failed' - assert len(vars_not_inited.keys()) == 0, \ - 'exist variable shape not inited, restore failed' + assert len(vars_not_inited.keys()) == 0, 'exist variable shape not inited, restore failed' def after_create_session(self, session, coord): assert self._restore_op is not None @@ -779,15 +709,10 @@ def __init__(self, incompatible_shape_var_map): def begin(self): assign_ops = [] for var, var_tmp in six.iteritems(self._incompatible_shape_var_map): - assign_ops.append( - var.assign( - shape_utils.pad_or_clip_nd(var_tmp, - var.get_shape().as_list()) - ) - ) + assign_ops.append(var.assign(shape_utils.pad_or_clip_nd(var_tmp, var.get_shape().as_list()))) logging.info( - 'Assign variable[%s] from shape%s to shape%s' % - (var.name, var_tmp.get_shape().as_list(), var.get_shape().as_list()) + 'Assign variable[%s] from shape%s to shape%s' + % (var.name, var_tmp.get_shape().as_list(), var.get_shape().as_list()) ) self._restore_op = tf.group(assign_ops) @@ -799,6 +724,7 @@ def after_create_session(self, session, coord): class MultipleCheckpointsRestoreHook(SessionRunHook): """Restore variable from numpy checkpoint.""" + SEP = ';' def __init__(self, ckpt_paths): @@ -825,10 +751,7 @@ def begin(self): var_name = re.sub(':[0-9]$', '', var.name) if var_name in ckpt_var2shape_map: if restore_status[var_name]: - logging.warning( - 'variable %s find in more than one checkpoint, skipped %s' % - (var_name, ckpt_path) - ) + logging.warning('variable %s find in more than one checkpoint, skipped %s' % (var_name, ckpt_path)) continue name2var[var_name] = var restore_status[var_name] = True @@ -851,7 +774,6 @@ def after_create_session(self, session, coord): class OnlineEvaluationHook(SessionRunHook): - def __init__(self, metric_dict, output_dir): self._metric_dict = metric_dict self._output_dir = output_dir @@ -871,9 +793,7 @@ def end(self, session): self._summary_writer.add_summary(summary, global_step=global_step) self._summary_writer.flush() - eval_result_file = os.path.join( - self._output_dir, 'online_eval_result.txt-%s' % global_step - ) + eval_result_file = os.path.join(self._output_dir, 'online_eval_result.txt-%s' % global_step) logging.info('Saving online eval result to file %s' % eval_result_file) with gfile.GFile(eval_result_file, 'w') as ofile: result_to_write = {} @@ -932,9 +852,7 @@ def get_ckpt_version(ckpt_path): return int(toks[-1]) -def get_latest_checkpoint_from_checkpoint_path( - checkpoint_path, ignore_ckpt_error -): +def get_latest_checkpoint_from_checkpoint_path(checkpoint_path, ignore_ckpt_error): ckpt_path = None if checkpoint_path.endswith('/') or gfile.IsDirectory(checkpoint_path + '/'): checkpoint_dir = checkpoint_path @@ -943,10 +861,7 @@ def get_latest_checkpoint_from_checkpoint_path( if gfile.Exists(checkpoint_dir): ckpt_path = latest_checkpoint(checkpoint_dir) if ckpt_path: - logging.info( - 'fine_tune_checkpoint is directory, will use the latest checkpoint: %s' - % ckpt_path - ) + logging.info('fine_tune_checkpoint is directory, will use the latest checkpoint: %s' % ckpt_path) else: assert ignore_ckpt_error, 'fine_tune_checkpoint(%s) is not exists.' % checkpoint_path else: @@ -1065,9 +980,7 @@ def has_sok(): def init_hvd(): if hvd is None: - logging.error( - 'horovod is not installed: HOROVOD_WITH_TENSORFLOW=1 pip install horovod' - ) + logging.error('horovod is not installed: HOROVOD_WITH_TENSORFLOW=1 pip install horovod') sys.exit(1) hvd.init() diff --git a/easy_rec/python/utils/export_big_model.py b/easy_rec/python/utils/export_big_model.py index 9575ee40f..1270a0920 100644 --- a/easy_rec/python/utils/export_big_model.py +++ b/easy_rec/python/utils/export_big_model.py @@ -12,20 +12,30 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.framework import ops from tensorflow.python.ops.variables import global_variables +from tensorflow.python.platform.gfile import ( # NOQA + DeleteRecursively, + Exists, + GFile, + Remove, +) from tensorflow.python.saved_model import signature_constants from tensorflow.python.training.device_setter import replica_device_setter +from tensorflow.python.training.monitored_session import ( # NOQA + ChiefSessionCreator, + Scaffold, +) from tensorflow.python.training.saver import export_meta_graph import easy_rec - -from tensorflow.python.platform.gfile import DeleteRecursively, Exists, GFile, Remove # NOQA -from tensorflow.python.training.monitored_session import ChiefSessionCreator, Scaffold # NOQA - from easy_rec.python.utils import constant, estimator_utils, io_util, proto_util # NOQA -from easy_rec.python.utils.meta_graph_editor import EMBEDDING_INITIALIZERS, MetaGraphEditor # NOQA +from easy_rec.python.utils.meta_graph_editor import ( # NOQA + EMBEDDING_INITIALIZERS, + MetaGraphEditor, +) if tf.__version__ >= '2.0': from tensorflow.python.framework.ops import disable_eager_execution + disable_eager_execution() ConfigProto = config_pb2.ConfigProto @@ -35,8 +45,13 @@ def export_big_model( - export_dir, pipeline_config, redis_params, serving_input_fn, estimator, - checkpoint_path, verbose + export_dir, + pipeline_config, + redis_params, + serving_input_fn, + estimator, + checkpoint_path, + verbose, ): for key in redis_params: logging.info('%s: %s' % (key, redis_params[key])) @@ -54,17 +69,13 @@ def export_big_model( kv_module = tf.load_op_library(write_kv_lib_path) try: - sparse_kv_lib_path = os.path.join( - easy_rec.ops_dir, 'libwrite_sparse_kv.so' - ) + sparse_kv_lib_path = os.path.join(easy_rec.ops_dir, 'libwrite_sparse_kv.so') sparse_kv_module = tf.load_op_library(sparse_kv_lib_path) except Exception as ex: logging.warning('load libwrite_sparse_kv.so failed: %s' % str(ex)) sparse_kv_module = None if not checkpoint_path: - checkpoint_path = estimator_utils.latest_checkpoint( - pipeline_config.model_dir - ) + checkpoint_path = estimator_utils.latest_checkpoint(pipeline_config.model_dir) logging.info('checkpoint_path = %s' % checkpoint_path) server = None @@ -74,9 +85,7 @@ def export_big_model( tf_config = estimator_utils.chief_to_master() if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = tf.train.Server( - cluster, job_name='ps', task_index=tf_config['task']['index'] - ) + server = tf.train.Server(cluster, job_name='ps', task_index=tf_config['task']['index']) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: @@ -91,24 +100,17 @@ def export_big_model( if cluster: logging.info('cluster = ' + str(cluster)) - with tf.device( - replica_device_setter(worker_device='/job:master/task:0', cluster=cluster) - ): - outputs = estimator._export_model_fn( - features, None, None, estimator.params - ).predictions + with tf.device(replica_device_setter(worker_device='/job:master/task:0', cluster=cluster)): + outputs = estimator._export_model_fn(features, None, None, estimator.params).predictions meta_graph_def = export_meta_graph() redis_embedding_version = redis_params.get('redis_embedding_version', '') if not redis_embedding_version: - meta_graph_def.meta_info_def.meta_graph_version =\ - str(int(time.time())) + meta_graph_def.meta_info_def.meta_graph_version = str(int(time.time())) else: meta_graph_def.meta_info_def.meta_graph_version = redis_embedding_version - logging.info( - 'meta_graph_version = %s' % meta_graph_def.meta_info_def.meta_graph_version - ) + logging.info('meta_graph_version = %s' % meta_graph_def.meta_info_def.meta_graph_version) embed_var_parts = {} embed_norm_name = {} @@ -122,13 +124,9 @@ def export_big_model( norm_name_to_ids[norm_name] = 1 tmp_export = x.export() if x.device not in embedding_vars: - embedding_vars[x.device] = [ - (norm_name, tmp_export.keys, tmp_export.values) - ] + embedding_vars[x.device] = [(norm_name, tmp_export.keys, tmp_export.values)] else: - embedding_vars[x.device].append( - (norm_name, tmp_export.keys, tmp_export.values) - ) + embedding_vars[x.device].append((norm_name, tmp_export.keys, tmp_export.values)) elif '/embedding_weights:' in x.name or '/embedding_weights/part_' in x.name: norm_name, part_id = proto_util.get_norm_embed_name(x.name) norm_name_to_ids[norm_name] = 1 @@ -141,10 +139,7 @@ def export_big_model( for tid, t in enumerate(norm_name_to_ids.keys()): norm_name_to_ids[t] = str(tid) - is_cache_from_redis = [ # noqa: F841 - proto_util.is_cache_from_redis(x, redis_cache_names) - for x in norm_name_to_ids - ] + is_cache_from_redis = [proto_util.is_cache_from_redis(x, redis_cache_names) for x in norm_name_to_ids] # noqa: F841 for x in embed_norm_name: embed_norm_name[x] = norm_name_to_ids[embed_norm_name[x]] @@ -190,7 +185,7 @@ def export_big_model( threads=redis_params.get('redis_threads', 5), batch_size=redis_params.get('redis_batch_size', 32), expire=redis_params.get('redis_expire', 24), - verbose=verbose + verbose=verbose, ) all_write_res.append(write_kv_res) @@ -211,37 +206,27 @@ def export_big_model( threads=redis_params.get('redis_threads', 5), batch_size=redis_params.get('redis_batch_size', 32), expire=redis_params.get('redis_expire', 24), - verbose=verbose + verbose=verbose, ) all_write_res.append(write_sparse_kv_res) - session_config = ConfigProto( - allow_soft_placement=True, log_device_placement=False - ) + session_config = ConfigProto(allow_soft_placement=True, log_device_placement=False) chief_sess_creator = ChiefSessionCreator( master=server.target if server else '', checkpoint_filename_with_path=checkpoint_path, - config=session_config + config=session_config, ) - with tf.train.MonitoredSession( - session_creator=chief_sess_creator, - hooks=None, - stop_grace_period_secs=120 - ) as sess: + with tf.train.MonitoredSession(session_creator=chief_sess_creator, hooks=None, stop_grace_period_secs=120) as sess: dump_flags = sess.run(all_write_res) logging.info('write embedding to redis succeed: %s' % str(dump_flags)) else: - logging.info( - 'will skip write embedding to redis because ' - 'redis_write_kv is set to 0.' - ) + logging.info('will skip write embedding to redis because ' 'redis_write_kv is set to 0.') # delete embedding_weights collections so that it could be re imported tmp_drop = [] for k in meta_graph_def.collection_def: v = meta_graph_def.collection_def[k] - if len(v.node_list.value - ) > 0 and 'embedding_weights' in v.node_list.value[0]: + if len(v.node_list.value) > 0 and 'embedding_weights' in v.node_list.value[0]: tmp_drop.append(k) for k in tmp_drop: meta_graph_def.collection_def.pop(k) @@ -255,7 +240,7 @@ def export_big_model( redis_cache_names=redis_cache_names, meta_graph_def=meta_graph_def, norm_name_to_ids=norm_name_to_ids, - debug_dir=export_dir if verbose else '' + debug_dir=export_dir if verbose else '', ) meta_graph_editor.edit_graph() tf.reset_default_graph() @@ -269,14 +254,10 @@ def export_big_model( fout.write('%s\t%s\n' % (tmp_norm_name, norm_name_to_ids[tmp_norm_name])) ops.add_to_collection( tf.GraphKeys.ASSET_FILEPATHS, - tf.constant( - embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt' - ) + tf.constant(embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt'), ) - export_dir = os.path.join( - export_dir, meta_graph_def.meta_info_def.meta_graph_version - ) + export_dir = os.path.join(export_dir, meta_graph_def.meta_info_def.meta_graph_version) export_dir = io_util.fix_oss_dir(export_dir) logging.info('export_dir=%s' % export_dir) if Exists(export_dir): @@ -287,38 +268,33 @@ def export_big_model( tensor_info_inputs = {} for tmp_key in inputs: tmp = graph.get_tensor_by_name(inputs[tmp_key].name) - tensor_info_inputs[tmp_key] = \ - tf.saved_model.utils.build_tensor_info(tmp) + tensor_info_inputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) tensor_info_outputs = {} for tmp_key in outputs: tmp = graph.get_tensor_by_name(outputs[tmp_key].name) - tensor_info_outputs[tmp_key] = \ - tf.saved_model.utils.build_tensor_info(tmp) - signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=tensor_info_inputs, - outputs=tensor_info_outputs, - method_name=signature_constants.PREDICT_METHOD_NAME - ) + tensor_info_outputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) + signature = tf.saved_model.signature_def_utils.build_signature_def( + inputs=tensor_info_inputs, + outputs=tensor_info_outputs, + method_name=signature_constants.PREDICT_METHOD_NAME, ) - session_config = ConfigProto( - allow_soft_placement=True, log_device_placement=True - ) + session_config = ConfigProto(allow_soft_placement=True, log_device_placement=True) saver = tf.train.Saver() with tf.Session(target=server.target if server else '') as sess: saver.restore(sess, checkpoint_path) builder.add_meta_graph_and_variables( - sess, [tf.saved_model.tag_constants.SERVING], + sess, + [tf.saved_model.tag_constants.SERVING], signature_def_map={ signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature, }, assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), saver=saver, strip_default_attrs=True, - clear_devices=True + clear_devices=True, ) builder.save() @@ -328,8 +304,13 @@ def export_big_model( def export_big_model_to_oss( - export_dir, pipeline_config, oss_params, serving_input_fn, estimator, - checkpoint_path, verbose + export_dir, + pipeline_config, + oss_params, + serving_input_fn, + estimator, + checkpoint_path, + verbose, ): for key in oss_params: logging.info('%s: %s' % (key, oss_params[key])) @@ -338,9 +319,7 @@ def export_big_model_to_oss( kv_module = tf.load_op_library(write_kv_lib_path) if not checkpoint_path: - checkpoint_path = estimator_utils.latest_checkpoint( - pipeline_config.model_dir - ) + checkpoint_path = estimator_utils.latest_checkpoint(pipeline_config.model_dir) logging.info('checkpoint_path = %s' % checkpoint_path) server = None @@ -350,9 +329,7 @@ def export_big_model_to_oss( tf_config = estimator_utils.chief_to_master() if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = tf.train.Server( - cluster, job_name='ps', task_index=tf_config['task']['index'] - ) + server = tf.train.Server(cluster, job_name='ps', task_index=tf_config['task']['index']) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: @@ -367,25 +344,18 @@ def export_big_model_to_oss( if cluster: logging.info('cluster = ' + str(cluster)) - with tf.device( - replica_device_setter(worker_device='/job:master/task:0', cluster=cluster) - ): - outputs = estimator._export_model_fn( - features, None, None, estimator.params - ).predictions + with tf.device(replica_device_setter(worker_device='/job:master/task:0', cluster=cluster)): + outputs = estimator._export_model_fn(features, None, None, estimator.params).predictions meta_graph_def = export_meta_graph() meta_graph_def.meta_info_def.meta_graph_version = str(int(time.time())) oss_embedding_version = oss_params.get('oss_embedding_version', '') if not oss_embedding_version: - meta_graph_def.meta_info_def.meta_graph_version =\ - str(int(time.time())) + meta_graph_def.meta_info_def.meta_graph_version = str(int(time.time())) else: meta_graph_def.meta_info_def.meta_graph_version = oss_embedding_version - logging.info( - 'meta_graph_version = %s' % meta_graph_def.meta_info_def.meta_graph_version - ) + logging.info('meta_graph_version = %s' % meta_graph_def.meta_info_def.meta_graph_version) embed_var_parts = {} embed_norm_name = {} @@ -400,13 +370,9 @@ def export_big_model_to_oss( norm_name_to_ids[norm_name] = 1 tmp_export = x.export() if x.device not in embedding_vars: - embedding_vars[x.device] = [ - (norm_name, tmp_export.keys, tmp_export.values, part_id) - ] + embedding_vars[x.device] = [(norm_name, tmp_export.keys, tmp_export.values, part_id)] else: - embedding_vars[x.device].append( - (norm_name, tmp_export.keys, tmp_export.values, part_id) - ) + embedding_vars[x.device].append((norm_name, tmp_export.keys, tmp_export.values, part_id)) elif '/embedding_weights:' in x.name or '/embedding_weights/part_' in x.name: norm_name, part_id = proto_util.get_norm_embed_name(x.name) norm_name_to_ids[norm_name] = 1 @@ -437,9 +403,7 @@ def export_big_model_to_oss( oss_endpoint = oss_params.get('oss_endpoint', '') oss_ak = oss_params.get('oss_ak', '') oss_sk = oss_params.get('oss_sk', '') - logging.info( - 'will export to oss: %s %s %s %s', oss_path, oss_endpoint, oss_ak, oss_sk - ) + logging.info('will export to oss: %s %s %s %s', oss_path, oss_endpoint, oss_ak, oss_sk) if oss_params.get('oss_write_kv', ''): # group embed by devices @@ -467,7 +431,7 @@ def export_big_model_to_oss( threads=oss_params.get('oss_threads', 5), timeout=5, expire=5, - verbose=verbose + verbose=verbose, ) all_write_res.append(write_kv_res) @@ -489,37 +453,27 @@ def export_big_model_to_oss( sk=oss_sk, version=meta_graph_def.meta_info_def.meta_graph_version, threads=oss_params.get('oss_threads', 5), - verbose=verbose + verbose=verbose, ) all_write_res.append(write_sparse_kv_res) - session_config = ConfigProto( - allow_soft_placement=True, log_device_placement=False - ) + session_config = ConfigProto(allow_soft_placement=True, log_device_placement=False) chief_sess_creator = ChiefSessionCreator( master=server.target if server else '', checkpoint_filename_with_path=checkpoint_path, - config=session_config + config=session_config, ) - with tf.train.MonitoredSession( - session_creator=chief_sess_creator, - hooks=None, - stop_grace_period_secs=120 - ) as sess: + with tf.train.MonitoredSession(session_creator=chief_sess_creator, hooks=None, stop_grace_period_secs=120) as sess: dump_flags = sess.run(all_write_res) logging.info('write embedding to oss succeed: %s' % str(dump_flags)) else: - logging.info( - 'will skip write embedding to oss because ' - 'oss_write_kv is set to 0.' - ) + logging.info('will skip write embedding to oss because ' 'oss_write_kv is set to 0.') # delete embedding_weights collections so that it could be re imported tmp_drop = [] for k in meta_graph_def.collection_def: v = meta_graph_def.collection_def[k] - if len(v.node_list.value - ) > 0 and 'embedding_weights' in v.node_list.value[0]: + if len(v.node_list.value) > 0 and 'embedding_weights' in v.node_list.value[0]: tmp_drop.append(k) for k in tmp_drop: meta_graph_def.collection_def.pop(k) @@ -535,7 +489,7 @@ def export_big_model_to_oss( meta_graph_def=meta_graph_def, norm_name_to_ids=norm_name_to_ids, incr_update_params=oss_params.get('incr_update', None), - debug_dir=export_dir if verbose else '' + debug_dir=export_dir if verbose else '', ) meta_graph_editor.edit_graph_for_oss() tf.reset_default_graph() @@ -549,22 +503,18 @@ def export_big_model_to_oss( fout.write('%s\t%s\n' % (tmp_norm_name, norm_name_to_ids[tmp_norm_name])) ops.add_to_collection( ops.GraphKeys.ASSET_FILEPATHS, - tf.constant( - embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt' - ) + tf.constant(embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt'), ) if 'incr_update' in oss_params: - dense_train_vars_path = os.path.join( - os.path.dirname(checkpoint_path), constant.DENSE_UPDATE_VARIABLES - ) + dense_train_vars_path = os.path.join(os.path.dirname(checkpoint_path), constant.DENSE_UPDATE_VARIABLES) ops.add_to_collection( ops.GraphKeys.ASSET_FILEPATHS, tf.constant( dense_train_vars_path, dtype=tf.string, - name=constant.DENSE_UPDATE_VARIABLES - ) + name=constant.DENSE_UPDATE_VARIABLES, + ), ) asset_file = 'incr_update.txt' @@ -575,32 +525,24 @@ def export_big_model_to_oss( if 'kafka' in incr_update: incr_update_json['storage'] = 'kafka' incr_update_json['kafka'] = json.loads( - json_format.MessageToJson( - incr_update['kafka'], preserving_proto_field_name=True - ) + json_format.MessageToJson(incr_update['kafka'], preserving_proto_field_name=True) ) elif 'datahub' in incr_update: incr_update_json['storage'] = 'datahub' incr_update_json['datahub'] = json.loads( - json_format.MessageToJson( - incr_update['datahub'], preserving_proto_field_name=True - ) + json_format.MessageToJson(incr_update['datahub'], preserving_proto_field_name=True) ) elif 'fs' in incr_update: incr_update_json['storage'] = 'fs' - incr_update_json['fs'] = { - 'incr_save_dir': incr_update['fs'].mount_path - } + incr_update_json['fs'] = {'incr_save_dir': incr_update['fs'].mount_path} json.dump(incr_update_json, fout, indent=2) ops.add_to_collection( ops.GraphKeys.ASSET_FILEPATHS, - tf.constant(asset_file_path, dtype=tf.string, name=asset_file) + tf.constant(asset_file_path, dtype=tf.string, name=asset_file), ) - export_dir = os.path.join( - export_dir, meta_graph_def.meta_info_def.meta_graph_version - ) + export_dir = os.path.join(export_dir, meta_graph_def.meta_info_def.meta_graph_version) export_dir = io_util.fix_oss_dir(export_dir) logging.info('export_dir=%s' % export_dir) if Exists(export_dir): @@ -611,20 +553,16 @@ def export_big_model_to_oss( tensor_info_inputs = {} for tmp_key in inputs: tmp = graph.get_tensor_by_name(inputs[tmp_key].name) - tensor_info_inputs[tmp_key] = \ - tf.saved_model.utils.build_tensor_info(tmp) + tensor_info_inputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) tensor_info_outputs = {} for tmp_key in outputs: tmp = graph.get_tensor_by_name(outputs[tmp_key].name) - tensor_info_outputs[tmp_key] = \ - tf.saved_model.utils.build_tensor_info(tmp) - signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=tensor_info_inputs, - outputs=tensor_info_outputs, - method_name=signature_constants.PREDICT_METHOD_NAME - ) + tensor_info_outputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) + signature = tf.saved_model.signature_def_utils.build_signature_def( + inputs=tensor_info_inputs, + outputs=tensor_info_outputs, + method_name=signature_constants.PREDICT_METHOD_NAME, ) if 'incr_update' in oss_params: @@ -636,25 +574,19 @@ def export_big_model_to_oss( tensor_info_incr_update_outputs = {} for tmp_key in incr_update_inputs: tmp = graph.get_tensor_by_name(incr_update_inputs[tmp_key].name) - tensor_info_incr_update_inputs[tmp_key] = \ - tf.saved_model.utils.build_tensor_info(tmp) + tensor_info_incr_update_inputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) for tmp_key in incr_update_outputs: tmp = graph.get_tensor_by_name(incr_update_outputs[tmp_key].name) - tensor_info_incr_update_outputs[tmp_key] = \ - tf.saved_model.utils.build_tensor_info(tmp) - incr_update_signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=tensor_info_incr_update_inputs, - outputs=tensor_info_incr_update_outputs, - method_name=signature_constants.PREDICT_METHOD_NAME - ) + tensor_info_incr_update_outputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) + incr_update_signature = tf.saved_model.signature_def_utils.build_signature_def( + inputs=tensor_info_incr_update_inputs, + outputs=tensor_info_incr_update_outputs, + method_name=signature_constants.PREDICT_METHOD_NAME, ) else: incr_update_signature = None - session_config = ConfigProto( - allow_soft_placement=True, log_device_placement=True - ) + session_config = ConfigProto(allow_soft_placement=True, log_device_placement=True) saver = tf.train.Saver() with tf.Session(target=server.target if server else '') as sess: @@ -662,22 +594,21 @@ def export_big_model_to_oss( main_op = tf.group( [ Scaffold.default_local_init_op(), - ops.get_collection(EMBEDDING_INITIALIZERS) + ops.get_collection(EMBEDDING_INITIALIZERS), ] ) - incr_update_sig_map = { - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature - } + incr_update_sig_map = {signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature} if incr_update_signature is not None: incr_update_sig_map[INCR_UPDATE_SIGNATURE_KEY] = incr_update_signature builder.add_meta_graph_and_variables( - sess, [tf.saved_model.tag_constants.SERVING], + sess, + [tf.saved_model.tag_constants.SERVING], signature_def_map=incr_update_sig_map, assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), saver=saver, main_op=main_op, strip_default_attrs=True, - clear_devices=True + clear_devices=True, ) builder.save() diff --git a/easy_rec/python/utils/expr_util.py b/easy_rec/python/utils/expr_util.py index b8fe5277e..ffd970ab7 100644 --- a/easy_rec/python/utils/expr_util.py +++ b/easy_rec/python/utils/expr_util.py @@ -7,9 +7,9 @@ def _process_multi_expr(expr): idx = 0 two_expr = ['>=', '<=', '=='] expr_list = [] - while (idx < size): - if idx + 2 <= size and expr[idx:idx + 2] in two_expr: - expr_list.append(expr[idx:idx + 2]) + while idx < size: + if idx + 2 <= size and expr[idx : idx + 2] in two_expr: + expr_list.append(expr[idx : idx + 2]) idx += 2 else: expr_list.append(expr[idx]) @@ -25,9 +25,7 @@ def _process_enum(enum, input_names, prefix=''): def _get_expression_list(expression, input_names, prefix=''): - ops = [ - '+', '-', '*', '/', '(', ')', '>', '>=', '<', '<=', '==', '=', '&', '|' - ] + ops = ['+', '-', '*', '/', '(', ')', '>', '>=', '<', '<=', '==', '=', '&', '|'] expression_list = [] eunm = '' pre_expr = '' @@ -113,8 +111,6 @@ def _expression_eval(expr_list): def get_expression(expression, input_names, prefix=''): - expression_list = _get_expression_list( - expression, input_names, prefix=prefix - ) + expression_list = _get_expression_list(expression, input_names, prefix=prefix) expression = _expression_eval(expression_list) return expression diff --git a/easy_rec/python/utils/fg_util.py b/easy_rec/python/utils/fg_util.py index dadfb4124..35036d01a 100644 --- a/easy_rec/python/utils/fg_util.py +++ b/easy_rec/python/utils/fg_util.py @@ -6,8 +6,9 @@ from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.protos.feature_config_pb2 import FeatureConfig from easy_rec.python.utils.config_util import get_compatible_feature_configs - -from easy_rec.python.utils.convert_rtp_fg import load_input_field_and_feature_config # NOQA +from easy_rec.python.utils.convert_rtp_fg import ( # NOQA + load_input_field_and_feature_config, +) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -26,9 +27,7 @@ def load_fg_json_to_config(pipeline_config): with tf.gfile.GFile(fg_json_path, 'r') as fin: rtp_fg = json.load(fin) - fg_config = load_input_field_and_feature_config( - rtp_fg, label_fields=label_fields - ) + fg_config = load_input_field_and_feature_config(rtp_fg, label_fields=label_fields) pipeline_config.data_config.ClearField('input_fields') pipeline_config.ClearField('feature_configs') diff --git a/easy_rec/python/utils/hit_rate_utils.py b/easy_rec/python/utils/hit_rate_utils.py index 32347e40b..258f7a5b3 100644 --- a/easy_rec/python/utils/hit_rate_utils.py +++ b/easy_rec/python/utils/hit_rate_utils.py @@ -30,14 +30,12 @@ def load_graph(i_emb_table, emb_dim, knn_metric, timeout, knn_strict): i_emb_table, node_type='i', decoder=gl.Decoder(attr_types=['float'] * emb_dim, attr_delimiter=','), - option=option + option=option, ) return g -def batch_hitrate( - src_ids, recall_ids, recall_distances, gt_items, num_interests, mask=None -): +def batch_hitrate(src_ids, recall_ids, recall_distances, gt_items, num_interests, mask=None): """Compute hitrate of a batch of src ids. Args: @@ -106,43 +104,32 @@ def reduce_hitrate(cluster, hits, count, task_index): var_worker_count: variable used to mark the number of worker that have completed the calculation of hitrate. """ - with tf.device( - tf.train.replica_device_setter( - worker_device='/job:worker/task:%d' % task_index, cluster=cluster - ) - ): + with tf.device(tf.train.replica_device_setter(worker_device='/job:worker/task:%d' % task_index, cluster=cluster)): with tf.variable_scope('hitrate_var', reuse=tf.AUTO_REUSE): var_worker_count = tf.get_variable( 'worker_count', shape=(), dtype=tf.int32, - initializer=tf.zeros_initializer() - ) - var_hits = tf.get_variable( - 'hits', shape=(), dtype=tf.float32, initializer=tf.zeros_initializer() + initializer=tf.zeros_initializer(), ) + var_hits = tf.get_variable('hits', shape=(), dtype=tf.float32, initializer=tf.zeros_initializer()) var_gt_count = tf.get_variable( 'gt_count', shape=(), dtype=tf.float32, - initializer=tf.zeros_initializer() + initializer=tf.zeros_initializer(), ) var_total_hitrate = tf.get_variable( 'total_hitate', shape=(), dtype=tf.float32, - initializer=tf.zeros_initializer() + initializer=tf.zeros_initializer(), ) var_hits = tf.assign_add(var_hits, hits, use_locking=True) var_gt_count = tf.assign_add(var_gt_count, count, use_locking=True) - var_gt_count = tf.Print( - var_gt_count, [var_gt_count, var_hits], - message='var_gt_count/var_hits' - ) - var_total_hitrate = tf.assign( - var_total_hitrate, var_hits / var_gt_count, use_locking=True - ) + var_gt_count = tf.Print(var_gt_count, [var_gt_count, var_hits], message='var_gt_count/var_hits') + var_total_hitrate = tf.assign(var_total_hitrate, var_hits / var_gt_count, use_locking=True) with tf.control_dependencies([var_total_hitrate]): var_worker_count = tf.assign_add(var_worker_count, 1, use_locking=True) return var_total_hitrate, var_worker_count @@ -174,8 +161,7 @@ def _to_float_attrs(x): if x == '': return np.zeros([emb_dim], dtype=np.float32) embed = np.array(x.split(','), dtype=np.float32) - assert len(embed - ) == emb_dim, 'invalid embed len=%d, x=%s' % (len(embed), x) + assert len(embed) == emb_dim, 'invalid embed len=%d, x=%s' % (len(embed), x) return embed def _to_multi_float_attrs(x, userid): @@ -184,49 +170,46 @@ def _to_multi_float_attrs(x, userid): else: arr = [_to_float_attrs(sub_x) for sub_x in x.split('|')] assert len(arr) == num_interests, 'invalid arr len=%d, x=%s, userid=%s' % ( - len(arr), x, userid + len(arr), + x, + userid, ) return arr src_ids = np.array([src_items[0] for src_items in gt_record]) - user_embedding = np.array( - [ - _to_multi_float_attrs(src_items[2], src_items[0]) - for src_items in gt_record - ] - ) + user_embedding = np.array([_to_multi_float_attrs(src_items[2], src_items[0]) for src_items in gt_record]) user_emb_num = [src_items[3] for src_items in gt_record] - print( - 'max(user_emb_num) = %d len(src_ids) = %d' % - (np.max(user_emb_num), len(src_ids)) - ) + print('max(user_emb_num) = %d len(src_ids) = %d' % (np.max(user_emb_num), len(src_ids))) # a list of list. - gt_items = [ - list(map(int, src_items[1].split(','))) for src_items in gt_record - ] + gt_items = [list(map(int, src_items[1].split(','))) for src_items in gt_record] logging.info('src_nodes.float_attrs.shape=%s' % str(user_embedding.shape)) user_embedding = user_embedding.reshape([-1, user_embedding.shape[-1]]) # numpy array - recall_ids, recall_distances = g.search( - 'i', user_embedding, gl.KnnOption(k=top_k) - ) + recall_ids, recall_distances = g.search('i', user_embedding, gl.KnnOption(k=top_k)) logging.info('recall_ids.shape=%s' % str(recall_ids.shape)) def _make_mask(lens): mask = np.ones([len(lens), num_interests], dtype=np.float32) for tmp_id, tmp_len in enumerate(lens): - mask[tmp_id, int(tmp_len):] = 0 + mask[tmp_id, int(tmp_len) :] = 0 return mask mask = _make_mask(user_emb_num) recall_ids = recall_ids.reshape([-1, num_interests, recall_ids.shape[-1]]) - recall_distances = recall_distances.reshape( - [-1, num_interests, recall_distances.shape[-1]] - ) + recall_distances = recall_distances.reshape([-1, num_interests, recall_distances.shape[-1]]) hitrates, bad_cases, bad_dists, hits, gt_count = batch_hitrate( src_ids, recall_ids, recall_distances, gt_items, num_interests, mask ) - return hits, gt_count, src_ids, recall_ids, recall_distances, hitrates, bad_cases, bad_dists + return ( + hits, + gt_count, + src_ids, + recall_ids, + recall_distances, + hitrates, + bad_cases, + bad_dists, + ) diff --git a/easy_rec/python/utils/hive_utils.py b/easy_rec/python/utils/hive_utils.py index 2cf3a5b8a..700ff3d03 100644 --- a/easy_rec/python/utils/hive_utils.py +++ b/easy_rec/python/utils/hive_utils.py @@ -9,7 +9,6 @@ class TableInfo(object): - def __init__(self, tablename, selected_cols, partition_kv, limit_num): self.tablename = tablename self.selected_cols = selected_cols @@ -45,9 +44,8 @@ def __init__( selected_cols='', record_defaults=[], task_index=0, - task_num=1 + task_num=1, ): - self._data_config = data_config self._hive_config = hive_config @@ -67,9 +65,7 @@ def _construct_table_info(self, table_name, limit_num): else: partition_kv = None - table_info = TableInfo( - table_name, self._selected_cols, partition_kv, limit_num - ) + table_info = TableInfo(table_name, self._selected_cols, partition_kv, limit_num) return table_info def _construct_hive_connect(self): @@ -77,7 +73,7 @@ def _construct_hive_connect(self): host=self._hive_config.host, port=self._hive_config.port, username=self._hive_config.username, - database=self._hive_config.database + database=self._hive_config.database, ) return conn @@ -123,12 +119,12 @@ def run_sql(self, sql): data = [] return data - def is_table_or_partition_exist( - self, table_name, partition_name=None, partition_val=None - ): + def is_table_or_partition_exist(self, table_name, partition_name=None, partition_val=None): if partition_name and partition_val: sql = 'show partitions %s partition(%s=%s)' % ( - table_name, partition_name, partition_val + table_name, + partition_name, + partition_val, ) try: res = self.run_sql(sql) @@ -178,8 +174,7 @@ def get_all_cols(self, input_path): for col in data: col_name = col[0].strip() - if col_name and (not col_name.startswith('#') - ) and (col_name not in col_names): + if col_name and (not col_name.startswith('#')) and (col_name not in col_names): if col_name != pt_name: col_names.append(col_name) cols_types.append(col[1].strip()) diff --git a/easy_rec/python/utils/hpo_util.py b/easy_rec/python/utils/hpo_util.py index 1592ccc0e..af684d910 100644 --- a/easy_rec/python/utils/hpo_util.py +++ b/easy_rec/python/utils/hpo_util.py @@ -122,10 +122,7 @@ def kill_old_proc(tmp_dir, platform='pai'): if platform == 'emr': # clear easy_rec_hpo yarn jobs yarn_job_file = os.path.join(tmp_dir, 'yarn_job.txt') - os.system( - "yarn application -list | awk '{ if ($2 == \"easy_rec_hpo\") print $1 }' > %s" - % yarn_job_file - ) + os.system('yarn application -list | awk \'{ if ($2 == "easy_rec_hpo") print $1 }\' > %s' % yarn_job_file) yarn_job_arr = [] with open(yarn_job_file, 'r') as fin: for line_str in fin: @@ -133,7 +130,5 @@ def kill_old_proc(tmp_dir, platform='pai'): yarn_job_arr.append(line_str) yarn_job_arr = list(set(yarn_job_arr)) if len(yarn_job_arr) > 0: - logging.info( - 'will kill the easy_rec_hpo yarn jobs: %s' % ','.join(yarn_job_arr) - ) + logging.info('will kill the easy_rec_hpo yarn jobs: %s' % ','.join(yarn_job_arr)) os.system('yarn application -kill %s' % ' '.join(yarn_job_arr)) diff --git a/easy_rec/python/utils/hvd_utils.py b/easy_rec/python/utils/hvd_utils.py index 14929bd73..e0efd8c4c 100644 --- a/easy_rec/python/utils/hvd_utils.py +++ b/easy_rec/python/utils/hvd_utils.py @@ -46,10 +46,7 @@ def begin(self): # if '/embedding' not in x.name and 'DynamicVariable' not in str(type(x)): if x.name not in embed_para_vars: bcast_vars.append(x) - logging.info( - 'will broadcast variable: name=%s shape=%s' % - (x.name, x.get_shape()) - ) + logging.info('will broadcast variable: name=%s shape=%s' % (x.name, x.get_shape())) if not self.bcast_op or self.bcast_op.graph != tf.get_default_graph(): with tf.device(self.device): self.bcast_op = broadcast_variables(bcast_vars, self.root_rank) diff --git a/easy_rec/python/utils/input_utils.py b/easy_rec/python/utils/input_utils.py index 03f41c1ed..b986d9d2b 100644 --- a/easy_rec/python/utils/input_utils.py +++ b/easy_rec/python/utils/input_utils.py @@ -17,7 +17,7 @@ def get_type_defaults(field_type, default_val=''): DatasetConfig.STRING: '', DatasetConfig.BOOL: False, DatasetConfig.FLOAT: 0.0, - DatasetConfig.DOUBLE: 0.0 + DatasetConfig.DOUBLE: 0.0, } assert field_type in type_defaults, 'invalid type: %s' % field_type if default_val == '': @@ -49,28 +49,20 @@ def string_to_number(field, ftype, default_value, name=''): Returns: A name for the operation (optional). """ default_vals = tf.tile(tf.constant([str(default_value)]), tf.shape(field)) - field = tf.where( - tf.greater(tf.strings.length(field), 0), field, default_vals - ) + field = tf.where(tf.greater(tf.strings.length(field), 0), field, default_vals) if ftype in [DatasetConfig.INT32, DatasetConfig.INT64]: # Int type is not supported in fg. # If you specify INT32, INT64 in DatasetConfig, you need to perform a cast at here. - tmp_field = tf.string_to_number( - field, tf.double, name='field_as_flt_%s' % name - ) + tmp_field = tf.string_to_number(field, tf.double, name='field_as_flt_%s' % name) if ftype in [DatasetConfig.INT64]: tmp_field = tf.cast(tmp_field, tf.int64) else: tmp_field = tf.cast(tmp_field, tf.int32) elif ftype in [DatasetConfig.FLOAT]: - tmp_field = tf.string_to_number( - field, tf.float32, name='field_as_flt_%s' % name - ) + tmp_field = tf.string_to_number(field, tf.float32, name='field_as_flt_%s' % name) elif ftype in [DatasetConfig.DOUBLE]: - tmp_field = tf.string_to_number( - field, tf.float64, name='field_as_flt_%s' % name - ) + tmp_field = tf.string_to_number(field, tf.float64, name='field_as_flt_%s' % name) elif ftype in [DatasetConfig.BOOL]: tmp_field = tf.logical_or(tf.equal(field, 'True'), tf.equal(field, 'true')) elif ftype in [DatasetConfig.STRING]: @@ -89,7 +81,7 @@ def np_to_tf_type(np_type): np.float: tf.float32, np.float32: tf.float32, float: tf.float32, - np.double: tf.float64 + np.double: tf.float64, } if np_type in _types_map: return _types_map[np_type] diff --git a/easy_rec/python/utils/io_util.py b/easy_rec/python/utils/io_util.py index 687282997..45eee96b5 100644 --- a/easy_rec/python/utils/io_util.py +++ b/easy_rec/python/utils/io_util.py @@ -4,6 +4,7 @@ isort:skip_file """ + import logging from future import standard_library @@ -18,6 +19,7 @@ from six.moves import http_client from six.moves import urllib import json + if six.PY2: from urllib import quote else: @@ -79,14 +81,9 @@ def download(oss_or_url, dst_dir=''): response = urllib.request.urlopen(oss_or_url, timeout=HTTP_MAX_TIMEOUT) file_content = response.read() except Exception as e: - raise RuntimeError( - 'Download %s failed: %s\n %s' % - (oss_or_url, str(e), traceback.format_exc()) - ) + raise RuntimeError('Download %s failed: %s\n %s' % (oss_or_url, str(e), traceback.format_exc())) else: - tf.logging.warning( - 'skip downloading %s, seems to be a local file' % oss_or_url - ) + tf.logging.warning('skip downloading %s, seems to be a local file' % oss_or_url) return oss_or_url if dst_dir != '' and not os.path.exists(dst_dir): @@ -132,26 +129,19 @@ def download_and_uncompress_resource(resource_path, dst_dir=EASY_REC_RES_DIR): create_module_dir(dst_dir) _, basename = os.path.split(resource_path) - if not basename.endswith('.tar.gz') and not basename.endswith('.zip') and \ - not basename.endswith('.py'): - raise ValueError( - 'resource %s should be tar.gz or zip or py' % resource_path - ) + if not basename.endswith('.tar.gz') and not basename.endswith('.zip') and not basename.endswith('.py'): + raise ValueError('resource %s should be tar.gz or zip or py' % resource_path) download(resource_path, dst_dir) stat = 0 if basename.endswith('tar.gz'): - stat, output = getstatusoutput( - 'cd %s && tar -zxf %s' % (dst_dir, basename) - ) + stat, output = getstatusoutput('cd %s && tar -zxf %s' % (dst_dir, basename)) elif basename.endswith('zip'): stat, output = getstatusoutput('cd %s && unzip %s' % (dst_dir, basename)) if stat != 0: - raise ValueError( - 'uncompress resoruce %s failed: %s' % resource_path, output - ) + raise ValueError('uncompress resoruce %s failed: %s' % resource_path, output) return dst_dir @@ -165,7 +155,6 @@ def oss_has_t_mode(target_file): try: with tf.gfile.GFile(test_file, 't') as ofile: ofile.write('a') - pass tf.gfile.Remove(test_file) return True except: # noqa: E722 @@ -205,6 +194,7 @@ def convert_tf_flags_to_argparse(flags): """ import argparse import ast + parser = argparse.ArgumentParser() args = {} @@ -217,8 +207,11 @@ def convert_tf_flags_to_argparse(flags): flag_type = type(default) help_str = flag.help or '' args[flag_name] = [ - False, flag_type, default, help_str, - flag.choices if hasattr(flag, 'choices') else None + False, + flag_type, + default, + help_str, + flag.choices if hasattr(flag, 'choices') else None, ] def str2bool(v): @@ -231,8 +224,7 @@ def str2bool(v): else: raise argparse.ArgumentTypeError('Boolean value expected.') - for flag_name, (multi, flag_type, default, help_str, - choices) in args.items(): + for flag_name, (multi, flag_type, default, help_str, choices) in args.items(): if flag_type == bool: parser.add_argument( '--' + flag_name, @@ -240,7 +232,7 @@ def str2bool(v): nargs='?', const=True, default=False, - help=help_str + help=help_str, ) elif flag_type == str: if choices: @@ -249,7 +241,7 @@ def str2bool(v): type=str, choices=choices, default=default, - help=help_str + help=help_str, ) elif multi: parser.add_argument( @@ -257,27 +249,21 @@ def str2bool(v): type=str, action='append', default=default, - help=help_str + help=help_str, ) else: - parser.add_argument( - '--' + flag_name, type=str, default=default, help=help_str - ) + parser.add_argument('--' + flag_name, type=str, default=default, help=help_str) elif flag_type in (list, dict): parser.add_argument( '--' + flag_name, type=lambda s: ast.literal_eval(s), default=default, - help=help_str + help=help_str, ) elif flag_type in (int, float): - parser.add_argument( - '--' + flag_name, type=flag_type, default=default, help=help_str - ) + parser.add_argument('--' + flag_name, type=flag_type, default=default, help=help_str) else: - parser.add_argument( - '--' + flag_name, type=str, default=default, help=help_str - ) + parser.add_argument('--' + flag_name, type=str, default=default, help=help_str) return parser diff --git a/easy_rec/python/utils/load_class.py b/easy_rec/python/utils/load_class.py index dd66e2240..1f89f3cc7 100644 --- a/easy_rec/python/utils/load_class.py +++ b/easy_rec/python/utils/load_class.py @@ -51,7 +51,6 @@ def load_by_path(path): def _get_methods(aClass): - def should_track(func_name): return func_name == '__init__' or func_name[0] != '_' @@ -73,9 +72,7 @@ def _get_method_declare(aMethod): return sig_str else: spec = inspect.getargspec(aMethod) - args = inspect.formatargspec( - spec.args, spec.varargs, spec.keywords, spec.defaults - ) + args = inspect.formatargspec(spec.args, spec.varargs, spec.keywords, spec.defaults) return '%s%s' % (name, args) except TypeError: return '%s(cls, ...)' % name @@ -109,10 +106,7 @@ def check_class(cls, impl_cls, function_names=None): missing[name + '()'] = 'method signature differs' if len(missing) > 0: - raise Exception( - 'incompatible Implementation-implementation %s: %s' % - (impl_cls.__class__.__name__, missing) - ) + raise Exception('incompatible Implementation-implementation %s: %s' % (impl_cls.__class__.__name__, missing)) def import_pkg(pkg_info, prefix_to_remove=None): @@ -145,6 +139,7 @@ def import_pkg(pkg_info, prefix_to_remove=None): __import__(module_path) except Exception as e: import traceback + logging.error(traceback.format_exc()) raise ValueError('import module %s failed: %s' % (module_path, str(e))) @@ -172,8 +167,8 @@ def auto_import(user_path=None): if parent_dir != '': for idx in range(len(pre_defined_dirs)): pre_defined_dirs[idx] = ( - os.path.join(parent_dir, - pre_defined_dirs[idx][0]), pre_defined_dirs[idx][1] + os.path.join(parent_dir, pre_defined_dirs[idx][0]), + pre_defined_dirs[idx][1], ) prefix_to_remove = parent_dir + '/' @@ -197,17 +192,20 @@ def auto_import(user_path=None): def register_class(class_map, class_name, cls): - assert class_name not in class_map or class_map[class_name] == cls, \ - 'confilict class %s , %s is already register to be %s' % ( - cls, class_name, str(class_map[class_name])) + assert class_name not in class_map or class_map[class_name] == cls, ( + 'confilict class %s , %s is already register to be %s' + % ( + cls, + class_name, + str(class_map[class_name]), + ) + ) logging.debug('register class %s' % class_name) class_map[class_name] = cls def get_register_class_meta(class_map, have_abstract_class=True): - class RegisterABCMeta(ABCMeta): - def __new__(mcs, name, bases, attrs): newclass = super(RegisterABCMeta, mcs).__new__(mcs, name, bases, attrs) register_class(class_map, name, newclass) @@ -217,10 +215,7 @@ def create_class(cls, name): if name in class_map: return class_map[name] else: - raise Exception( - 'Class %s is not registered. Available ones are %s' % - (name, list(class_map.keys())) - ) + raise Exception('Class %s is not registered. Available ones are %s' % (name, list(class_map.keys()))) setattr(newclass, 'create_class', create_class) return newclass @@ -250,7 +245,5 @@ def load_keras_layer(name): return pydoc.locate(path), False except pydoc.ErrorDuringImport: print('load keras layer %s failed' % name) - logging.error( - 'load keras layer %s failed: %s' % (name, traceback.format_exc()) - ) + logging.error('load keras layer %s failed: %s' % (name, traceback.format_exc())) return None, False diff --git a/easy_rec/python/utils/meta_graph_editor.py b/easy_rec/python/utils/meta_graph_editor.py index bd12e9280..e4131bf73 100644 --- a/easy_rec/python/utils/meta_graph_editor.py +++ b/easy_rec/python/utils/meta_graph_editor.py @@ -7,17 +7,22 @@ from google.protobuf import text_format from tensorflow.python.framework import ops from tensorflow.python.platform.gfile import GFile + # from tensorflow.python.saved_model import constants from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model.loader_impl import SavedModelLoader -from easy_rec.python.utils import conditional, constant, embedding_utils, proto_util # NOQA +from easy_rec.python.utils import ( # NOQA + conditional, + constant, + embedding_utils, + proto_util, +) EMBEDDING_INITIALIZERS = 'embedding_initializers' class MetaGraphEditor: - def __init__( self, lookup_lib_path, @@ -34,7 +39,7 @@ def __init__( meta_graph_def=None, norm_name_to_ids=None, incr_update_params=None, - debug_dir='' + debug_dir='', ): self._lookup_op = tf.load_op_library(lookup_lib_path) self._debug_dir = debug_dir @@ -48,13 +53,11 @@ def __init__( assert meta_graph_def, 'either saved_model_dir or meta_graph_def must be set' tf.reset_default_graph() from tensorflow.python.framework import meta_graph - meta_graph.import_scoped_meta_graph_with_return_elements( - meta_graph_def, clear_devices=True - ) + + meta_graph.import_scoped_meta_graph_with_return_elements(meta_graph_def, clear_devices=True) # tf.train.import_meta_graph(meta_graph_def) self._meta_graph_version = meta_graph_def.meta_info_def.meta_graph_version - self._signature_def = meta_graph_def.signature_def[ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] + self._signature_def = meta_graph_def.signature_def[signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] if self._verbose: debug_out_path = os.path.join(self._debug_dir, 'meta_graph_raw.txt') @@ -163,11 +166,13 @@ def _get_share_embed_name(self, x, embed_names): embed_name_sub = '/'.join(tmp_toks) if tmp_name == embed_name_sub: assert not sel_embed_name, 'confusions encountered: %s %s' % ( - x, ','.join(embed_names) + x, + ','.join(embed_names), ) sel_embed_name = embed_name assert sel_embed_name, '%s not find in shared_embeddings: %s' % ( - tmp_name, ','.join(embed_names) + tmp_name, + ','.join(embed_names), ) return sel_embed_name @@ -184,28 +189,25 @@ def _find_embed_combiners(self, norm_embed_names): combiner_map = { 'SparseSegmentSum': 'sum', 'SparseSegmentMean': 'mean', - 'SparseSegmentSqrtN': 'sqrtn' + 'SparseSegmentSqrtN': 'sqrtn', } for node in self._meta_graph_def.graph_def.node: if node.op in combiner_map: norm_name, _ = proto_util.get_norm_embed_name(node.name) embed_combiners[norm_name] = combiner_map[node.op] - embed_combine_node_cts[norm_name - ] = embed_combine_node_cts.get(norm_name, 0) + 1 + embed_combine_node_cts[norm_name] = embed_combine_node_cts.get(norm_name, 0) + 1 elif node.op == 'RealDiv' and len(node.input) == 2: # for tag feature with weights, and combiner == mean if 'SegmentSum' in node.input[0] and 'SegmentSum' in node.input[1]: norm_name, _ = proto_util.get_norm_embed_name(node.name) embed_combiners[norm_name] = 'mean' - embed_combine_node_cts[ - norm_name] = embed_combine_node_cts.get(norm_name, 0) + 1 + embed_combine_node_cts[norm_name] = embed_combine_node_cts.get(norm_name, 0) + 1 elif node.op == 'SegmentSum': norm_name, _ = proto_util.get_norm_embed_name(node.name) # avoid overwrite RealDiv results if norm_name not in embed_combiners: embed_combiners[norm_name] = 'sum' - embed_combine_node_cts[norm_name - ] = embed_combine_node_cts.get(norm_name, 0) + 1 + embed_combine_node_cts[norm_name] = embed_combine_node_cts.get(norm_name, 0) + 1 return [embed_combiners[x] for x in norm_embed_names] def _find_lookup_indices_values_shapes(self): @@ -231,13 +233,9 @@ def _get_output_shape(graph_def, input_name): if '_embedding_weights/SparseReshape' in node.name: if node.op == 'SparseReshape': # embed_name, _ = proto_util.get_norm_embed_name(node.name, self._verbose) - fea_name, _ = proto_util.get_norm_embed_name( - node.name, self._verbose - ) + fea_name, _ = proto_util.get_norm_embed_name(node.name, self._verbose) for tmp_input in node.input: - tmp_shape = _get_output_shape( - self._meta_graph_def.graph_def, tmp_input - ) + tmp_shape = _get_output_shape(self._meta_graph_def.graph_def, tmp_input) if '_embedding_weights/Cast' in tmp_input: continue elif len(tmp_shape.dim) == 2: @@ -245,9 +243,7 @@ def _get_output_shape(graph_def, input_name): elif len(tmp_shape.dim) == 1: shapes[fea_name] = tmp_input elif node.op == 'Identity': - fea_name, _ = proto_util.get_norm_embed_name( - node.name, self._verbose - ) + fea_name, _ = proto_util.get_norm_embed_name(node.name, self._verbose) values[fea_name] = node.input[0] return indices, values, shapes @@ -282,17 +278,15 @@ def _find_embed_names_and_dims(self, norm_embed_names): embed_is_kv = {} for node in self._meta_graph_def.graph_def.node: if 'embedding_weights' in node.name and node.op in [ - 'VariableV2', 'KvVarHandleOp' + 'VariableV2', + 'KvVarHandleOp', ]: tmp = node.attr['shape'].shape.dim[-1].size tmp2 = 1 for x in node.attr['shape'].shape.dim[:-1]: tmp2 = tmp2 * x.size - embed_name, _ = proto_util.get_norm_embed_name( - node.name, self._verbose - ) - assert embed_name is not None,\ - 'fail to get_norm_embed_name(%s)' % node.name + embed_name, _ = proto_util.get_norm_embed_name(node.name, self._verbose) + assert embed_name is not None, 'fail to get_norm_embed_name(%s)' % node.name embed_dims[embed_name] = tmp embed_sizes[embed_name] = tmp2 embed_is_kv[embed_name] = 1 if node.op == 'KvVarHandleOp' else 0 @@ -325,10 +319,7 @@ def find_lookup_inputs(self): weights = self._find_lookup_weights() for fea in shapes.keys(): - logging.info( - 'Lookup Input[%s]: indices=%s values=%s shapes=%s' % - (fea, indices[fea], values[fea], shapes[fea]) - ) + logging.info('Lookup Input[%s]: indices=%s values=%s shapes=%s' % (fea, indices[fea], values[fea], shapes[fea])) graph = tf.get_default_graph() @@ -356,33 +347,36 @@ def _get_tensor_by_name(tensor_name): self._embed_combiners = self._find_embed_combiners(values.keys()) # get embedding dimensions - self._embed_names, self._embed_dims, self._embed_sizes, self._embed_is_kv\ - = self._find_embed_names_and_dims(values.keys()) + ( + self._embed_names, + self._embed_dims, + self._embed_sizes, + self._embed_is_kv, + ) = self._find_embed_names_and_dims(values.keys()) if not self._embed_name_to_ids: embed_name_uniq = list(set(self._embed_names)) - self._embed_name_to_ids = { - t: tid - for tid, t in enumerate(embed_name_uniq) - } - self._embed_ids = [ - int(self._embed_name_to_ids[x]) for x in self._embed_names - ] + self._embed_name_to_ids = {t: tid for tid, t in enumerate(embed_name_uniq)} + self._embed_ids = [int(self._embed_name_to_ids[x]) for x in self._embed_names] - self._is_cache_from_redis = [ - proto_util.is_cache_from_redis(x, self._redis_cache_names) - for x in self._embed_names - ] + self._is_cache_from_redis = [proto_util.is_cache_from_redis(x, self._redis_cache_names) for x in self._embed_names] # normalized feature names self._feature_names = list(values.keys()) - return lookup_input_indices, lookup_input_values, lookup_input_shapes,\ - lookup_input_weights + return ( + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + ) def add_lookup_op( - self, lookup_input_indices, lookup_input_values, lookup_input_shapes, - lookup_input_weights + self, + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, ): logging.info('add custom lookup operation to lookup embeddings from redis') self._lookup_outs = [None for i in range(len(lookup_input_values))] @@ -403,7 +397,7 @@ def add_lookup_op( embedding_dims=self._embed_dims[i:i_1], embedding_names=self._embed_ids[i:i_1], cache=self._is_cache_from_redis, - version=self._meta_graph_version + version=self._meta_graph_version, )[0] meta_graph_def = tf.train.export_meta_graph() @@ -411,16 +405,15 @@ def add_lookup_op( if self._verbose: debug_path = os.path.join(self._debug_dir, 'graph_raw.txt') with GFile(debug_path, 'w') as fout: - fout.write( - text_format.MessageToString( - self._meta_graph_def.graph_def, as_utf8=True - ) - ) + fout.write(text_format.MessageToString(self._meta_graph_def.graph_def, as_utf8=True)) return meta_graph_def def add_oss_lookup_op( - self, lookup_input_indices, lookup_input_values, lookup_input_shapes, - lookup_input_weights + self, + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, ): logging.info('add custom lookup operation to lookup embeddings from oss') place_on_cpu = os.getenv('place_embedding_on_cpu') @@ -464,7 +457,7 @@ def add_oss_lookup_op( embedding_ids=self._embed_ids, embedding_is_kv=self._embed_is_kv, shared_name='embedding_lookup_res', - name='embedding_lookup_fused/lookup' + name='embedding_lookup_fused/lookup', ) N = np.max([int(x) for x in self._embed_ids]) + 1 @@ -473,8 +466,7 @@ def add_oss_lookup_op( uniq_embed_combiners = ['mean' for x in range(N)] uniq_embed_is_kvs = [0 for x in range(N)] for embed_id, embed_combiner, embed_is_kv, embed_dim in zip( - self._embed_ids, self._embed_combiners, self._embed_is_kv, - self._embed_dims + self._embed_ids, self._embed_combiners, self._embed_is_kv, self._embed_dims ): uniq_embed_combiners[embed_id] = embed_combiner uniq_embed_is_kvs[embed_id] = embed_is_kv @@ -491,7 +483,7 @@ def add_oss_lookup_op( embedding_is_kv=uniq_embed_is_kvs, N=N, shared_name='embedding_lookup_res', - name='embedding_lookup_fused/init' + name='embedding_lookup_fused/init', ) ops.add_to_collection(EMBEDDING_INITIALIZERS, lookup_init_op) @@ -502,11 +494,10 @@ def add_oss_lookup_op( embedding_update = self._lookup_op.embedding_update( message=message_ph, shared_name='embedding_lookup_res', - name='embedding_lookup_fused/embedding_update' + name='embedding_lookup_fused/embedding_update', ) self._embedding_update_inputs['incr_update/sparse/message'] = message_ph - self._embedding_update_outputs['incr_update/sparse/embedding_update' - ] = embedding_update + self._embedding_update_outputs['incr_update/sparse/embedding_update'] = embedding_update # dense variables are updated one by one dense_name_to_ids = embedding_utils.get_dense_name_to_ids() @@ -514,9 +505,7 @@ def add_oss_lookup_op( dense_var_id = dense_name_to_ids[x.op.name] dense_input_name = 'incr_update/dense/%d/input' % dense_var_id dense_output_name = 'incr_update/dense/%d/output' % dense_var_id - dense_update_input = tf.placeholder( - tf.float32, x.get_shape(), name=dense_input_name - ) + dense_update_input = tf.placeholder(tf.float32, x.get_shape(), name=dense_input_name) self._dense_update_inputs[dense_input_name] = dense_update_input dense_assign_op = tf.assign(x, dense_update_input) self._dense_update_outputs[dense_output_name] = dense_assign_op @@ -526,11 +515,7 @@ def add_oss_lookup_op( if self._verbose: debug_path = os.path.join(self._debug_dir, 'graph_raw.txt') with GFile(debug_path, 'w') as fout: - fout.write( - text_format.MessageToString( - self._meta_graph_def.graph_def, as_utf8=True - ) - ) + fout.write(text_format.MessageToString(self._meta_graph_def.graph_def, as_utf8=True)) return meta_graph_def def bytes2str(self, x): @@ -548,16 +533,13 @@ def clear_meta_graph_embeding(self, meta_graph_def): def _clear_embedding_in_meta_collect(meta_graph_def, collect_name): tmp_vals = [ - x for x in meta_graph_def.collection_def[collect_name].bytes_list.value + x + for x in meta_graph_def.collection_def[collect_name].bytes_list.value if 'embedding_weights' not in self.bytes2str(x) ] - meta_graph_def.collection_def[collect_name].bytes_list.ClearField( - 'value' - ) + meta_graph_def.collection_def[collect_name].bytes_list.ClearField('value') for tmp_v in tmp_vals: - meta_graph_def.collection_def[collect_name].bytes_list.value.append( - tmp_v - ) + meta_graph_def.collection_def[collect_name].bytes_list.value.append(tmp_v) _clear_embedding_in_meta_collect(meta_graph_def, 'model_variables') _clear_embedding_in_meta_collect(meta_graph_def, 'trainable_variables') @@ -565,10 +547,16 @@ def _clear_embedding_in_meta_collect(meta_graph_def, collect_name): # clear Kv(pai embedding variable) ops in meta_info_def.stripped_op_list.op kept_ops = [ - x for x in meta_graph_def.meta_info_def.stripped_op_list.op - if x.name not in [ - 'InitializeKvVariableOp', 'KvResourceGather', 'KvResourceImportV2', - 'KvVarHandleOp', 'KvVarIsInitializedOp', 'ReadKvVariableOp' + x + for x in meta_graph_def.meta_info_def.stripped_op_list.op + if x.name + not in [ + 'InitializeKvVariableOp', + 'KvResourceGather', + 'KvResourceImportV2', + 'KvVarHandleOp', + 'KvVarIsInitializedOp', + 'ReadKvVariableOp', ] ] meta_graph_def.meta_info_def.stripped_op_list.ClearField('op') @@ -585,8 +573,7 @@ def clear_meta_collect(self, meta_graph_def): for key in meta_graph_def.collection_def: val = meta_graph_def.collection_def[key] if val.HasField('node_list'): - if 'embedding_weights' in val.node_list.value[ - 0] and 'easy_rec' not in val.node_list.value[0]: + if 'embedding_weights' in val.node_list.value[0] and 'easy_rec' not in val.node_list.value[0]: drop_meta_collects.append(key) elif key == 'saved_model_assets': drop_meta_collects.append(key) @@ -594,7 +581,6 @@ def clear_meta_collect(self, meta_graph_def): meta_graph_def.collection_def.pop(key) def remove_embedding_weights_and_update_lookup_outputs(self): - def _should_drop(name): if '_embedding_weights' in name: if self._verbose: @@ -602,9 +588,7 @@ def _should_drop(name): return True logging.info('remove embedding_weights node in graph_def.node') - logging.info( - 'and replace the old embedding_lookup outputs with new lookup_op outputs' - ) + logging.info('and replace the old embedding_lookup outputs with new lookup_op outputs') for tid, node in enumerate(self._all_graph_nodes): # drop the nodes @@ -613,21 +597,16 @@ def _should_drop(name): else: for i in range(len(node.input)): if _should_drop(node.input[i]): - input_name, _ = proto_util.get_norm_embed_name( - node.input[i], self._verbose - ) + input_name, _ = proto_util.get_norm_embed_name(node.input[i], self._verbose) print('REPLACE:' + node.input[i] + '=>' + input_name) - input_name = self._lookup_outs[ - self._feature_names.index(input_name)].name + input_name = self._lookup_outs[self._feature_names.index(input_name)].name if input_name.endswith(':0'): input_name = input_name.replace(':0', '') node.input[i] = input_name # drop by ids def _drop_by_ids(self, tmp_obj, key, drop_ids): - keep_vals = [ - x for i, x in enumerate(getattr(tmp_obj, key)) if i not in drop_ids - ] + keep_vals = [x for i, x in enumerate(getattr(tmp_obj, key)) if i not in drop_ids] tmp_obj.ClearField(key) getattr(tmp_obj, key).extend(keep_vals) @@ -650,30 +629,19 @@ def clear_save_restore(self): if self._restore_tensor_node: drop_ids = [] - for tmp_id, tmp_name in enumerate( - self._restore_tensor_node.attr['value'].tensor.string_val - ): + for tmp_id, tmp_name in enumerate(self._restore_tensor_node.attr['value'].tensor.string_val): if 'embedding_weights' in self.bytes2str(tmp_name): drop_ids.append(tmp_id) - self._drop_by_ids( - self._restore_tensor_node.attr['value'].tensor, 'string_val', drop_ids - ) - keep_node_num = len( - self._restore_tensor_node.attr['value'].tensor.string_val - ) + self._drop_by_ids(self._restore_tensor_node.attr['value'].tensor, 'string_val', drop_ids) + keep_node_num = len(self._restore_tensor_node.attr['value'].tensor.string_val) logging.info( - 'update self._restore_tensor_node: string_val keep_num = %d drop_num = %d' - % (keep_node_num, len(drop_ids)) + 'update self._restore_tensor_node: string_val keep_num = %d drop_num = %d' % (keep_node_num, len(drop_ids)) ) - self._restore_tensor_node.attr['value'].tensor.tensor_shape.dim[ - 0].size = keep_node_num - self._restore_tensor_node.attr['_output_shapes'].list.shape[0].dim[ - 0].size = keep_node_num + self._restore_tensor_node.attr['value'].tensor.tensor_shape.dim[0].size = keep_node_num + self._restore_tensor_node.attr['_output_shapes'].list.shape[0].dim[0].size = keep_node_num - logging.info( - 'update save/RestoreV2, drop tensor_shapes, _output_shapes, related to embedding_weights' - ) + logging.info('update save/RestoreV2, drop tensor_shapes, _output_shapes, related to embedding_weights') self._restore_shard_node = None for node_id, node in enumerate(self._all_graph_nodes): if not self._all_graph_node_flags[tid]: @@ -691,17 +659,14 @@ def clear_save_restore(self): self._restore_all_node.append(node) def clear_save_assign(self): - logging.info( - 'update save/Assign, drop tensor_shapes, _output_shapes, related to embedding_weights' - ) + logging.info('update save/Assign, drop tensor_shapes, _output_shapes, related to embedding_weights') # edit save/Assign drop_save_assigns = [] all_kv_drop = [] for tid, node in enumerate(self._all_graph_nodes): if not self._all_graph_node_flags[tid]: continue - if node.op == 'Assign' and 'save/Assign' in node.name and \ - 'embedding_weights' in node.input[0]: + if node.op == 'Assign' and 'save/Assign' in node.name and 'embedding_weights' in node.input[0]: drop_save_assigns.append('^' + node.name) self._all_graph_node_flags[tid] = False elif 'embedding_weights/ConcatPartitions/concat' in node.name: @@ -727,20 +692,16 @@ def clear_save_assign(self): # update node(save/Assign_[0-N])'s input[1] by the position of # node.input[0] in save/RestoreV2/tensor_names # the outputs of save/RestoreV2 is connected to save/Assign - tmp_id = [ - self.bytes2str(x) - for x in self._restore_tensor_node.attr['value'].tensor.string_val - ].index(node.input[0]) + tmp_id = [self.bytes2str(x) for x in self._restore_tensor_node.attr['value'].tensor.string_val].index( + node.input[0] + ) if tmp_id != 0: tmp_input2 = 'save/RestoreV2:%d' % tmp_id else: tmp_input2 = 'save/RestoreV2' if tmp_input2 != node.input[1]: if self._verbose: - logging.info( - "update save/Assign[%s]'s input from %s to %s" % - (node.name, node.input[1], tmp_input2) - ) + logging.info("update save/Assign[%s]'s input from %s to %s" % (node.name, node.input[1], tmp_input2)) node.input[1] = tmp_input2 # save/restore_all need save/restore_shard as input @@ -770,9 +731,7 @@ def clear_save_v2(self): save/SaveV2 input: [ save/SaveV2/tensor_names, save/SaveV2/shape_and_slices ] edit save/SaveV2 save/SaveV2/shape_and_slices save/SaveV2/tensor_names. """ - logging.info( - 'update save/SaveV2 input shape, _output_shapes, tensor_shape' - ) + logging.info('update save/SaveV2 input shape, _output_shapes, tensor_shape') save_drop_ids = [] for tid, node in enumerate(self._all_graph_nodes): if not self._all_graph_node_flags[tid]: @@ -790,31 +749,21 @@ def clear_save_v2(self): for node in self._all_graph_nodes: if node.name == 'save/SaveV2/shape_and_slices' and node.op == 'Const': # _output_shapes # size # string_val - node.attr['_output_shapes'].list.shape[0].dim[0].size -= len( - save_drop_ids - ) - node.attr['value'].tensor.tensor_shape.dim[0].size -= len( - save_drop_ids - ) - self._drop_by_ids( - node.attr['value'].tensor, 'string_val', save_drop_ids - ) + node.attr['_output_shapes'].list.shape[0].dim[0].size -= len(save_drop_ids) + node.attr['value'].tensor.tensor_shape.dim[0].size -= len(save_drop_ids) + self._drop_by_ids(node.attr['value'].tensor, 'string_val', save_drop_ids) elif node.name == 'save/SaveV2/tensor_names': # tensor_names may not have the same order as save/SaveV2/shape_and_slices tmp_drop_ids = [ - tmp_id for tmp_id, tmp_val in - enumerate(node.attr['value'].tensor.string_val) + tmp_id + for tmp_id, tmp_val in enumerate(node.attr['value'].tensor.string_val) if 'embedding_weights' in self.bytes2str(tmp_val) ] # attr['value'].tensor.string_val # tensor_shape # size assert len(save_drop_ids) == len(save_drop_ids) - node.attr['_output_shapes'].list.shape[0].dim[0].size -= len( - tmp_drop_ids - ) + node.attr['_output_shapes'].list.shape[0].dim[0].size -= len(tmp_drop_ids) node.attr['value'].tensor.tensor_shape.dim[0].size -= len(tmp_drop_ids) - self._drop_by_ids( - node.attr['value'].tensor, 'string_val', tmp_drop_ids - ) + self._drop_by_ids(node.attr['value'].tensor, 'string_val', tmp_drop_ids) def clear_initialize(self): """Clear initialization ops. @@ -835,15 +784,13 @@ def clear_initialize(self): self._all_graph_node_flags[tid] = False elif 'embedding_weights' in node.name and node.op == 'VariableV2': self._all_graph_node_flags[tid] = False - elif 'embedding_weights' in node.name and node.name.endswith( - '/read' - ) and node.op == 'Identity': + elif 'embedding_weights' in node.name and node.name.endswith('/read') and node.op == 'Identity': self._all_graph_node_flags[tid] = False elif 'embedding_weights' in node.name and node.op == 'Identity': node_toks = node.name.split('/') node_tok = node_toks[-1] if 'embedding_weights_' in node_tok: - node_tok = node_tok[len('embedding_weights_'):] + node_tok = node_tok[len('embedding_weights_') :] try: int(node_tok) self._all_graph_node_flags[tid] = False @@ -855,16 +802,13 @@ def clear_embedding_variable(self): for tid, node in enumerate(self._all_graph_nodes): if not self._all_graph_node_flags[tid]: continue - if node.op in [ - 'ReadKvVariableOp', 'KvVarIsInitializedOp', 'KvVarHandleOp' - ]: + if node.op in ['ReadKvVariableOp', 'KvVarIsInitializedOp', 'KvVarHandleOp']: self._all_graph_node_flags[tid] = False # there maybe some nodes depend on the dropped nodes, they are dropped as well def drop_dependent_nodes(self): drop_names = [ - tmp_node.name for tid, tmp_node in enumerate(self._all_graph_nodes) - if not self._all_graph_node_flags[tid] + tmp_node.name for tid, tmp_node in enumerate(self._all_graph_nodes) if not self._all_graph_node_flags[tid] ] while True: more_drop_names = [] @@ -872,10 +816,7 @@ def drop_dependent_nodes(self): if not self._all_graph_node_flags[tid]: continue if len(tmp_node.input) > 0 and tmp_node.input[0] in drop_names: - logging.info( - 'drop dependent node: %s depend on %s' % - (tmp_node.name, tmp_node.input[0]) - ) + logging.info('drop dependent node: %s depend on %s' % (tmp_node.name, tmp_node.input[0])) self._all_graph_node_flags[tid] = False more_drop_names.append(tmp_node.name) drop_names = more_drop_names @@ -884,13 +825,19 @@ def drop_dependent_nodes(self): def edit_graph(self): # the main entrance - lookup_input_indices, lookup_input_values, lookup_input_shapes,\ - lookup_input_weights = self.find_lookup_inputs() + ( + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + ) = self.find_lookup_inputs() # add lookup op to the graph self._meta_graph_def = self.add_lookup_op( - lookup_input_indices, lookup_input_values, lookup_input_shapes, - lookup_input_weights + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, ) self.clear_meta_graph_embeding(self._meta_graph_def) @@ -918,10 +865,7 @@ def edit_graph(self): self._meta_graph_def.graph_def.ClearField('node') self._meta_graph_def.graph_def.node.extend( - [ - x for tid, x in enumerate(self._all_graph_nodes) - if self._all_graph_node_flags[tid] - ] + [x for tid, x in enumerate(self._all_graph_nodes) if self._all_graph_node_flags[tid]] ) logging.info('old node number = %d' % self._old_node_num) @@ -933,19 +877,23 @@ def edit_graph(self): fout.write(text_format.MessageToString(self.graph_def, as_utf8=True)) debug_dump_path = os.path.join(self._debug_dir, 'meta_graph.txt') with GFile(debug_dump_path, 'w') as fout: - fout.write( - text_format.MessageToString(self._meta_graph_def, as_utf8=True) - ) + fout.write(text_format.MessageToString(self._meta_graph_def, as_utf8=True)) def edit_graph_for_oss(self): # the main entrance - lookup_input_indices, lookup_input_values, lookup_input_shapes,\ - lookup_input_weights = self.find_lookup_inputs() + ( + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + ) = self.find_lookup_inputs() # add lookup op to the graph self._meta_graph_def = self.add_oss_lookup_op( - lookup_input_indices, lookup_input_values, lookup_input_shapes, - lookup_input_weights + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, ) self.clear_meta_graph_embeding(self._meta_graph_def) @@ -973,10 +921,7 @@ def edit_graph_for_oss(self): self._meta_graph_def.graph_def.ClearField('node') self._meta_graph_def.graph_def.node.extend( - [ - x for tid, x in enumerate(self._all_graph_nodes) - if self._all_graph_node_flags[tid] - ] + [x for tid, x in enumerate(self._all_graph_nodes) if self._all_graph_node_flags[tid]] ) logging.info('old node number = %d' % self._old_node_num) @@ -988,6 +933,4 @@ def edit_graph_for_oss(self): fout.write(text_format.MessageToString(self.graph_def, as_utf8=True)) debug_dump_path = os.path.join(self._debug_dir, 'meta_graph.txt') with GFile(debug_dump_path, 'w') as fout: - fout.write( - text_format.MessageToString(self._meta_graph_def, as_utf8=True) - ) + fout.write(text_format.MessageToString(self._meta_graph_def, as_utf8=True)) diff --git a/easy_rec/python/utils/multi_optimizer.py b/easy_rec/python/utils/multi_optimizer.py index fbfad1fff..0c2b5ad28 100644 --- a/easy_rec/python/utils/multi_optimizer.py +++ b/easy_rec/python/utils/multi_optimizer.py @@ -6,7 +6,6 @@ class MultiOptimizer(optimizer.Optimizer): - def __init__(self, opts, grouped_vars, use_locking=False): """Combine multiple optimizers for optimization, such as WideAndDeep. @@ -23,9 +22,7 @@ def __init__(self, opts, grouped_vars, use_locking=False): def compute_gradients(self, loss, variables, **kwargs): grad_and_vars = [] for gid, opt in enumerate(self._opts): - grad_and_vars.extend( - opt.compute_gradients(loss, self._grouped_vars[gid], **kwargs) - ) + grad_and_vars.extend(opt.compute_gradients(loss, self._grouped_vars[gid], **kwargs)) return grad_and_vars def apply_gradients(self, grads_and_vars, global_step=None, name=None): diff --git a/easy_rec/python/utils/odps_util.py b/easy_rec/python/utils/odps_util.py index 4ad4d8981..c6ef3cb30 100644 --- a/easy_rec/python/utils/odps_util.py +++ b/easy_rec/python/utils/odps_util.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Common functions used for odps input.""" + from tensorflow.python.framework import dtypes from easy_rec.python.protos.dataset_pb2 import DatasetConfig @@ -11,7 +12,7 @@ def is_type_compatiable(odps_type, input_type): type_map = { 'bigint': DatasetConfig.INT64, 'string': DatasetConfig.STRING, - 'double': DatasetConfig.DOUBLE + 'double': DatasetConfig.DOUBLE, } tmp_type = type_map[odps_type] if tmp_type == input_type: @@ -32,7 +33,7 @@ def odps_type_to_input_type(odps_type): odps_type_map = { 'bigint': DatasetConfig.INT64, 'string': DatasetConfig.STRING, - 'double': DatasetConfig.DOUBLE + 'double': DatasetConfig.DOUBLE, } assert odps_type in odps_type_map, 'only support [bigint, string, double]' input_type = odps_type_map[odps_type] @@ -63,9 +64,14 @@ def check_input_field_and_types(data_config): type_map = {x: y for x, y in zip(selected_cols, selected_types)} for x, y in zip(input_fields, input_field_types): tmp_type = type_map[x] - assert is_type_compatiable(tmp_type, y), \ - 'feature[%s] type error: odps %s is not compatible with input_type %s' % ( - x, tmp_type, DatasetConfig.FieldType.Name(y)) + assert is_type_compatiable(tmp_type, y), ( + 'feature[%s] type error: odps %s is not compatible with input_type %s' + % ( + x, + tmp_type, + DatasetConfig.FieldType.Name(y), + ) + ) def odps_type_2_tf_type(odps_type): diff --git a/easy_rec/python/utils/pai_util.py b/easy_rec/python/utils/pai_util.py index e59acabe8..26c5a4ce5 100644 --- a/easy_rec/python/utils/pai_util.py +++ b/easy_rec/python/utils/pai_util.py @@ -54,9 +54,9 @@ def process_config(configs, task_index=0, worker_num=1): """ configs = configs.split(',') if len(configs) > 1: - assert len(configs) == worker_num, \ - 'number of configs must be equal to number of workers,' + \ - ' when number of configs > 1' + assert len(configs) == worker_num, ( + 'number of configs must be equal to number of workers,' + ' when number of configs > 1' + ) config = configs[task_index] else: config = configs[0] @@ -76,9 +76,7 @@ def process_config(configs, task_index=0, worker_num=1): def test(): - f = download( - 'https://easy-rec.oss-cn-hangzhou.aliyuncs.com/config/MultiTower/dwd_avazu_ctr_deepmodel.config' - ) + f = download('https://easy-rec.oss-cn-hangzhou.aliyuncs.com/config/MultiTower/dwd_avazu_ctr_deepmodel.config') assert f == 'dwd_avazu_ctr_deepmodel.config' diff --git a/easy_rec/python/utils/proto_util.py b/easy_rec/python/utils/proto_util.py index 80e819a91..f3b6efa08 100644 --- a/easy_rec/python/utils/proto_util.py +++ b/easy_rec/python/utils/proto_util.py @@ -31,14 +31,13 @@ def get_norm_embed_name(name, verbose=False): for i in range(0, len(name_toks) - 1): if name_toks[i + 1].startswith('embedding_weights:'): var_id = name_toks[i + 1].replace('embedding_weights:', '') - tmp_name = '/'.join(name_toks[:i + 1]) + tmp_name = '/'.join(name_toks[: i + 1]) if var_id != '0': tmp_name = tmp_name + '_' + var_id if verbose: logging.info('norm %s to %s' % (name, tmp_name)) return tmp_name, 0 - if i > 1 and name_toks[i + 1].startswith('part_') and \ - name_toks[i] == 'embedding_weights': + if i > 1 and name_toks[i + 1].startswith('part_') and name_toks[i] == 'embedding_weights': tmp_name = '/'.join(name_toks[:i]) part_id = name_toks[i + 1].replace('part_', '') part_toks = part_id.split(':') @@ -51,9 +50,8 @@ def get_norm_embed_name(name, verbose=False): # input_layer/app_category_embedding/app_category_embedding_weights/SparseReshape # => input_layer/app_category_embedding for i in range(0, len(name_toks) - 1): - if name_toks[i + 1].endswith('_embedding_weights') or \ - '_embedding_weights_' in name_toks[i + 1]: - tmp_name = '/'.join(name_toks[:i + 1]) + if name_toks[i + 1].endswith('_embedding_weights') or '_embedding_weights_' in name_toks[i + 1]: + tmp_name = '/'.join(name_toks[: i + 1]) if verbose: logging.info('norm %s to %s' % (name, tmp_name)) return tmp_name, 0 @@ -61,7 +59,7 @@ def get_norm_embed_name(name, verbose=False): # => input_layer/app_category_embedding for i in range(0, len(name_toks) - 1): if name_toks[i + 1] == 'embedding_weights': - tmp_name = '/'.join(name_toks[:i + 1]) + tmp_name = '/'.join(name_toks[: i + 1]) if verbose: logging.info('norm %s to %s' % (name, tmp_name)) return tmp_name, 0 @@ -85,8 +83,6 @@ def is_cache_from_redis(name, redis_cache_names): for y in redis_cache_names: for k in tok: if k.startswith(y): - logging.info( - 'embedding %s will be cached[specified by %s]' % (name, y) - ) + logging.info('embedding %s will be cached[specified by %s]' % (name, y)) return True return False diff --git a/easy_rec/python/utils/restore_filter.py b/easy_rec/python/utils/restore_filter.py index 55ed7378c..734b76bff 100644 --- a/easy_rec/python/utils/restore_filter.py +++ b/easy_rec/python/utils/restore_filter.py @@ -31,7 +31,6 @@ def keep(self, var_name): class KeywordFilter(Filter): - def __init__(self, pattern, exclusive=False): """Init KeywordFilter. @@ -51,7 +50,6 @@ def keep(self, var_name): class CombineFilter(Filter): - def __init__(self, filters, logical=Logical.AND): """Init CombineFilter. diff --git a/easy_rec/python/utils/shape_utils.py b/easy_rec/python/utils/shape_utils.py index 438fb4331..24a202215 100644 --- a/easy_rec/python/utils/shape_utils.py +++ b/easy_rec/python/utils/shape_utils.py @@ -14,6 +14,7 @@ # limitations under the License. # ============================================================================== """Utils used to manipulate tensor shapes.""" + from __future__ import absolute_import, division, print_function import six @@ -61,9 +62,7 @@ def merge_shape(t, shape_list): the tensor t with shape updated """ t_shape = t.get_shape().as_list() - assert len(shape_list) == len( - t_shape - ), 'input shape size should be the same of the tensor' + assert len(shape_list) == len(t_shape), 'input shape size should be the same of the tensor' for idx, size in enumerate(shape_list): if size is not None: t_shape[idx] = size @@ -89,8 +88,9 @@ def pad_tensor(t, length): t_d0 = t_shape[0] pad_d0 = tf.expand_dims(length - t_d0, 0) pad_shape = tf.cond( - tf.greater(t_rank, 1), lambda: tf.concat([pad_d0, t_shape[1:]], 0), - lambda: tf.expand_dims(length - t_d0, 0) + tf.greater(t_rank, 1), + lambda: tf.concat([pad_d0, t_shape[1:]], 0), + lambda: tf.expand_dims(length - t_d0, 0), ) padded_t = tf.concat([t, tf.zeros(pad_shape, dtype=t.dtype)], 0) if not _is_tensor(length): @@ -146,18 +146,10 @@ def pad_nd(tensor, output_shape): """ tensor_shape = tf.shape(tensor) - trailing_paddings = [ - shape - tensor_shape[i] if shape is not None else 0 - for i, shape in enumerate(output_shape) - ] - paddings = tf.stack( - [tf.zeros(len(trailing_paddings), dtype=tf.int32), trailing_paddings], - axis=1 - ) + trailing_paddings = [shape - tensor_shape[i] if shape is not None else 0 for i, shape in enumerate(output_shape)] + paddings = tf.stack([tf.zeros(len(trailing_paddings), dtype=tf.int32), trailing_paddings], axis=1) padded_tensor = tf.pad(tensor, paddings=paddings) - output_static_shape = [ - dim if not isinstance(dim, tf.Tensor) else None for dim in output_shape - ] + output_static_shape = [dim if not isinstance(dim, tf.Tensor) else None for dim in output_shape] padded_tensor.set_shape(output_static_shape) return padded_tensor @@ -175,12 +167,10 @@ def pad_or_clip_nd(tensor, output_shape): """ tensor_shape = tf.shape(tensor) clip_size = [ - tf.where(tensor_shape[i] - shape > 0, shape, -1) - if shape is not None else -1 for i, shape in enumerate(output_shape) + tf.where(tensor_shape[i] - shape > 0, shape, -1) if shape is not None else -1 + for i, shape in enumerate(output_shape) ] - clipped_tensor = tf.slice( - tensor, begin=tf.zeros(len(clip_size), dtype=tf.int32), size=clip_size - ) + clipped_tensor = tf.slice(tensor, begin=tf.zeros(len(clip_size), dtype=tf.int32), size=clip_size) # Pad tensor if the shape of clipped tensor is smaller than the expected # shape. @@ -237,17 +227,16 @@ def check_min_image_dim(min_dim, image_tensor): shape_assert = tf.Assert( tf.logical_and( tf.greater_equal(tf.shape(image_tensor)[1], min_dim), - tf.greater_equal(tf.shape(image_tensor)[2], min_dim) + tf.greater_equal(tf.shape(image_tensor)[2], min_dim), ), - ['image size must be >= {} in both height and width.'.format(min_dim)] + ['image size must be >= {} in both height and width.'.format(min_dim)], ) with tf.control_dependencies([shape_assert]): return tf.identity(image_tensor) if image_height < min_dim or image_width < min_dim: raise ValueError( - 'image size must be >= %d in both height and width; image dim = %d,%d' % - (min_dim, image_height, image_width) + 'image size must be >= %d in both height and width; image dim = %d,%d' % (min_dim, image_height, image_width) ) return image_tensor @@ -273,10 +262,7 @@ def assert_shape_equal(shape_a, shape_b): Raises: ValueError: When shapes are both static and unequal. """ - if ( - all(isinstance(dim, int) for dim in shape_a) - and all(isinstance(dim, int) for dim in shape_b) - ): + if all(isinstance(dim, int) for dim in shape_a) and all(isinstance(dim, int) for dim in shape_b): if shape_a != shape_b: raise ValueError('Unequal shapes {}, {}'.format(shape_a, shape_b)) else: @@ -307,9 +293,7 @@ def assert_shape_equal_along_first_dimension(shape_a, shape_b): """ if isinstance(shape_a[0], int) and isinstance(shape_b[0], int): if shape_a[0] != shape_b[0]: - raise ValueError( - 'Unequal first dimension {}, {}'.format(shape_a[0], shape_b[0]) - ) + raise ValueError('Unequal first dimension {}, {}'.format(shape_a[0], shape_b[0])) else: return tf.no_op() else: @@ -335,8 +319,9 @@ def assert_box_normalized(boxes, maximum_normalized_coordinate=1.1): return tf.Assert( tf.logical_and( tf.less_equal(box_maximum, maximum_normalized_coordinate), - tf.greater_equal(box_minimum, 0) - ), [boxes] + tf.greater_equal(box_minimum, 0), + ), + [boxes], ) @@ -361,7 +346,7 @@ def get_shape_list(tensor, expected_rank=None, name=None): assert_rank(tensor, expected_rank, name) shape = tensor.shape.as_list() non_static_indexes = [] - for (index, dim) in enumerate(shape): + for index, dim in enumerate(shape): if dim is None: non_static_indexes.append(index) if not non_static_indexes: @@ -396,20 +381,18 @@ def assert_rank(tensor, expected_rank, name=None): scope_name = tf.get_variable_scope().name raise ValueError( 'For the tensor `%s` in scope `%s`, the actual rank ' - '`%d` (shape = %s) is not equal to the expected rank `%s`' % - (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank)) + '`%d` (shape = %s) is not equal to the expected rank `%s`' + % (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank)) ) def truncate_sequence(seq_emb, seq_len, limited_len): - def truncate(seq_embed, seq_length): - seq_embed = tf.slice( - seq_embed, [0, 0, 0], [shape[0], limited_len, shape[2]] - ) + seq_embed = tf.slice(seq_embed, [0, 0, 0], [shape[0], limited_len, shape[2]]) seq_length = tf.where( tf.greater(seq_length, limited_len), - tf.ones_like(seq_length) * limited_len, seq_length + tf.ones_like(seq_length) * limited_len, + seq_length, ) return seq_embed, seq_length @@ -420,8 +403,9 @@ def keep(seq_embed, seq_length): max_seq_len = shape[1] return tf.cond( - max_seq_len > limited_len, lambda: truncate(seq_emb, seq_len), - lambda: keep(seq_emb, seq_len) + max_seq_len > limited_len, + lambda: truncate(seq_emb, seq_len), + lambda: keep(seq_emb, seq_len), ) @@ -435,16 +419,10 @@ def padding(): def truncate(): sliced = tf.slice(seq_emb, [0, 0, 0], [-1, fixed_len, -1]) - length = tf.where( - seq_len < fixed_len, seq_len, - tf.ones_like(seq_len) * fixed_len - ) if seq_len is not None else None + length = tf.where(seq_len < fixed_len, seq_len, tf.ones_like(seq_len) * fixed_len) if seq_len is not None else None return sliced, length def keep(): return seq_emb, seq_len - return tf.cond( - padding_length > 0, padding, - lambda: tf.cond(padding_length < 0, truncate, keep) - ) + return tf.cond(padding_length > 0, padding, lambda: tf.cond(padding_length < 0, truncate, keep)) diff --git a/easy_rec/python/utils/test_utils.py b/easy_rec/python/utils/test_utils.py index abe7c4284..069844354 100644 --- a/easy_rec/python/utils/test_utils.py +++ b/easy_rec/python/utils/test_utils.py @@ -4,6 +4,7 @@ isort:skip_file """ + from future import standard_library standard_library.install_aliases() @@ -36,9 +37,7 @@ def get_hdfs_tmp_dir(test_dir): """Create a randomly of directory in HDFS.""" - tmp_name = ''.join( - [random.choice(string.ascii_letters + string.digits) for i in range(8)] - ) + tmp_name = ''.join([random.choice(string.ascii_letters + string.digits) for i in range(8)]) assert isinstance(test_dir, str) test_rand_dir = os.path.join(test_dir, tmp_name) gfile.MkDir(test_rand_dir) @@ -50,9 +49,7 @@ def proc_wait(proc, timeout=1200): while proc.poll() is None and time.time() - t0 < timeout: time.sleep(1) if proc.poll() is None: - logging.warning( - 'proc[pid=%d] timeout[%d], will kill the proc' % (proc.pid, timeout) - ) + logging.warning('proc[pid=%d] timeout[%d], will kill the proc' % (proc.pid, timeout)) proc.terminate() while proc.poll() is None: time.sleep(1) @@ -61,9 +58,7 @@ def proc_wait(proc, timeout=1200): def get_tmp_dir(): max_retry = 5 while max_retry > 0: - tmp_name = ''.join( - [random.choice(string.ascii_letters + string.digits) for i in range(12)] - ) + tmp_name = ''.join([random.choice(string.ascii_letters + string.digits) for i in range(12)]) if os.environ.get('TEST_DIR', '') != '': global TEST_DIR TEST_DIR = os.environ['TEST_DIR'] @@ -103,9 +98,7 @@ def run_cmd(cmd_str, log_file, env=None): cmd_str = cmd_str.replace('\r', ' ').replace('\n', ' ') logging.info('RUNCMD: %s > %s 2>&1 ' % (cmd_str, log_file)) with open(log_file, 'w') as lfile: - proc = subprocess.Popen( - cmd_str, stdout=lfile, stderr=subprocess.STDOUT, shell=True, env=env - ) + proc = subprocess.Popen(cmd_str, stdout=lfile, stderr=subprocess.STDOUT, shell=True, env=env) if six.PY2: # for debug purpose proc.args = cmd_str @@ -164,12 +157,8 @@ def _replace_data_for_test(data_path): return data_path -def _load_config_for_test( - pipeline_config_path, test_dir, total_steps=50, num_epochs=0 -): - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path - ) +def _load_config_for_test(pipeline_config_path, test_dir, total_steps=50, num_epochs=0): + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) train_config = pipeline_config.train_config eval_config = pipeline_config.eval_config data_config = pipeline_config.data_config @@ -184,9 +173,7 @@ def _load_config_for_test( def _load_config_for_distribute_eval(pipeline_config_path, test_dir): - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path - ) + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) pipeline_config.model_dir = test_dir logging.info('test_model_dir %s' % pipeline_config.model_dir) return pipeline_config @@ -198,7 +185,7 @@ def test_datahub_train_eval( test_dir, process_pipeline_func=None, total_steps=50, - post_check_func=None + post_check_func=None, ): gpus = get_available_gpus() if len(gpus) > 0: @@ -214,9 +201,7 @@ def test_datahub_train_eval( if isinstance(pipeline_config_path, EasyRecConfig): pipeline_config = pipeline_config_path else: - pipeline_config = _load_config_for_test( - pipeline_config_path, test_dir, total_steps - ) + pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, total_steps) pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -239,14 +224,12 @@ def test_datahub_train_eval( pipeline_config = process_pipeline_func(pipeline_config) config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') - train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path %s' % \ - test_pipeline_config_path + train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path %s' % test_pipeline_config_path proc = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, 'master')) proc_wait(proc, timeout=TEST_TIME_OUT) if proc.returncode != 0: logging.warning( - 'train %s failed[pid=%d][code=%d][args=%s]' % - (test_pipeline_config_path, proc.pid, proc.returncode, proc.args) + 'train %s failed[pid=%d][code=%d][args=%s]' % (test_pipeline_config_path, proc.pid, proc.returncode, proc.args) ) return False if post_check_func: @@ -255,9 +238,7 @@ def test_datahub_train_eval( def _Load_config_for_test_eval(pipeline_config_path): - pipeline_config = config_util.get_configs_from_pipeline_file( - pipeline_config_path - ) + pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) return pipeline_config @@ -271,7 +252,7 @@ def test_single_train_eval( check_mode=False, fine_tune_checkpoint=None, extra_cmd_args=None, - timeout=-1 + timeout=-1, ): gpus = get_available_gpus() if len(gpus) > 0: @@ -287,9 +268,7 @@ def test_single_train_eval( if isinstance(pipeline_config_path, EasyRecConfig): pipeline_config = pipeline_config_path else: - pipeline_config = _load_config_for_test( - pipeline_config_path, test_dir, total_steps - ) + pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, total_steps) pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -301,7 +280,7 @@ def test_single_train_eval( test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path=' + test_pipeline_config_path if hyperparam_str: - train_cmd += ' --edit_config_json=\'%s\'' % hyperparam_str + train_cmd += " --edit_config_json='%s'" % hyperparam_str if fine_tune_checkpoint: train_cmd += ' --fine_tune_checkpoint %s' % fine_tune_checkpoint if check_mode: @@ -342,9 +321,7 @@ def test_single_pre_check(pipeline_config_path, test_dir): config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') - train_cmd = 'python -m easy_rec.python.tools.pre_check --pipeline_config_path %s ' % ( - test_pipeline_config_path - ) + train_cmd = 'python -m easy_rec.python.tools.pre_check --pipeline_config_path %s ' % (test_pipeline_config_path) proc = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, 'master')) proc_wait(proc, timeout=TEST_TIME_OUT) @@ -362,7 +339,9 @@ def test_single_predict(test_dir, input_path, output_path, saved_model_dir): set_gpu_id(None) predict_cmd = 'python -m easy_rec.python.predict --input_path %s --output_path %s --saved_model_dir %s' % ( - input_path, output_path, saved_model_dir + input_path, + output_path, + saved_model_dir, ) proc = run_cmd(predict_cmd, '%s/log_%s.txt' % (test_dir, 'master')) @@ -377,8 +356,10 @@ def test_feature_selection(pipeline_config): model_dir = pipeline_config.model_dir pipeline_config_path = os.path.join(model_dir, 'pipeline.config') output_dir = os.path.join(model_dir, 'feature_selection') - cmd = 'python -m easy_rec.python.tools.feature_selection --config_path %s ' \ - '--output_dir %s --topk 5 --visualize true' % (pipeline_config_path, output_dir) + cmd = ( + 'python -m easy_rec.python.tools.feature_selection --config_path %s ' + '--output_dir %s --topk 5 --visualize true' % (pipeline_config_path, output_dir) + ) proc = run_cmd(cmd, os.path.join(model_dir, 'log_feature_selection.txt')) proc_wait(proc, timeout=TEST_TIME_OUT) if proc.returncode != 0: @@ -391,20 +372,18 @@ def yaml_replace( train_yaml_path, pipline_config_path, test_pipeline_config_path, - test_export_dir=None + test_export_dir=None, ): with open(train_yaml_path, 'r', encoding='utf-8') as _file: sample = _file.read() x = yaml.load(sample) _command = x['app']['command'] if test_export_dir is not None: - _command = _command.replace( - pipline_config_path, test_pipeline_config_path - ).replace('{EXPOERT_DIR}', test_export_dir) - else: - _command = _command.replace( - pipline_config_path, test_pipeline_config_path + _command = _command.replace(pipline_config_path, test_pipeline_config_path).replace( + '{EXPOERT_DIR}', test_export_dir ) + else: + _command = _command.replace(pipline_config_path, test_pipeline_config_path) x['app']['command'] = _command with open(train_yaml_path, 'w', encoding='utf-8') as _file: @@ -417,9 +396,8 @@ def test_hdfs_train_eval( test_dir, process_pipeline_func=None, hyperparam_str='', - total_steps=2000 + total_steps=2000, ): - gpus = get_available_gpus() if len(gpus) > 0: set_gpu_id(gpus[0]) @@ -429,9 +407,7 @@ def test_hdfs_train_eval( logging.info('train_yaml_path %s' % train_yaml_path) if 'TF_CONFIG' in os.environ: del os.environ['TF_CONFIG'] - pipeline_config = _load_config_for_test( - pipeline_config_path, test_dir, total_steps - ) + pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, total_steps) logging.info('model_dir in pipeline_config has been modified') pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -441,9 +417,7 @@ def test_hdfs_train_eval( pipeline_config = process_pipeline_func(pipeline_config) config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') - yaml_replace( - train_yaml_path, pipeline_config_path, test_pipeline_config_path - ) + yaml_replace(train_yaml_path, pipeline_config_path, test_pipeline_config_path) logging.info('test_pipeline_config_path is %s' % test_pipeline_config_path) train_cmd = 'el_submit -yaml %s' % train_yaml_path proc = subprocess.Popen(train_cmd.split(), stderr=subprocess.STDOUT) @@ -459,9 +433,8 @@ def test_hdfs_eval( eval_yaml_path, test_dir, process_pipeline_func=None, - hyperparam_str='' + hyperparam_str='', ): - gpus = get_available_gpus() if len(gpus) > 0: set_gpu_id(gpus[0]) @@ -493,9 +466,8 @@ def test_hdfs_export( export_yaml_path, test_dir, process_pipeline_func=None, - hyperparam_str='' + hyperparam_str='', ): - gpus = get_available_gpus() if len(gpus) > 0: set_gpu_id(gpus[0]) @@ -513,8 +485,10 @@ def test_hdfs_export( test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') test_export_path = os.path.join(test_dir, 'export_dir') yaml_replace( - export_yaml_path, pipeline_config_path, test_pipeline_config_path, - test_export_path + export_yaml_path, + pipeline_config_path, + test_pipeline_config_path, + test_export_path, ) logging.info('test_pipeline_config_path is %s' % test_pipeline_config_path) eval_cmd = 'el_submit -yaml %s' % export_yaml_path @@ -552,8 +526,10 @@ def _get_ports(num_worker): if 'ports' in os.environ: ports = os.environ['ports'] port_arr = [int(x) for x in ports.split(',')] - assert len(port_arr) >= num_worker, 'not enough ports: %s, required: %d'\ - % (ports, num_worker) + assert len(port_arr) >= num_worker, 'not enough ports: %s, required: %d' % ( + ports, + num_worker, + ) return port_arr[:num_worker] else: return get_ports_base(num_worker) @@ -565,7 +541,7 @@ def _ps_worker_train( num_worker, num_evaluator=0, fit_on_eval=False, - fit_on_eval_steps=None + fit_on_eval_steps=None, ): gpus = get_available_gpus() # not enough gpus, run on cpu only @@ -576,7 +552,7 @@ def _ps_worker_train( cluster = { chief_or_master: ['localhost:%d' % ports[0]], 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], - 'ps': ['localhost:%d' % ports[-1]] + 'ps': ['localhost:%d' % ports[-1]], } tf_config = {'cluster': cluster} procs = {} @@ -588,9 +564,7 @@ def _ps_worker_train( train_cmd += ' --fit_on_eval' if fit_on_eval_steps is not None: train_cmd += ' --fit_on_eval_steps ' + str(int(fit_on_eval_steps)) - procs[chief_or_master] = run_cmd( - train_cmd, '%s/log_%s.txt' % (test_dir, chief_or_master) - ) + procs[chief_or_master] = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, chief_or_master)) tf_config['task'] = {'type': 'ps', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') @@ -601,27 +575,17 @@ def _ps_worker_train( os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id(gpus[idx + 1]) worker_name = 'worker_%d' % idx - procs[worker_name] = run_cmd( - train_cmd, '%s/log_%s.txt' % (test_dir, worker_name) - ) + procs[worker_name] = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, worker_name)) if num_evaluator > 0: tf_config['task'] = {'type': 'evaluator', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') - procs['evaluator'] = run_cmd( - train_cmd, '%s/log_%s.txt' % (test_dir, 'evaluator') - ) + procs['evaluator'] = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, 'evaluator')) return procs -def _ps_worker_distribute_eval( - pipeline_config_path, - checkpoint_path, - test_dir, - num_worker, - num_evaluator=0 -): +def _ps_worker_distribute_eval(pipeline_config_path, checkpoint_path, test_dir, num_worker, num_evaluator=0): gpus = get_available_gpus() # not enough gpus, run on cpu only if len(gpus) < num_worker: @@ -631,7 +595,7 @@ def _ps_worker_distribute_eval( cluster = { chief_or_master: ['localhost:%d' % ports[0]], 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], - 'ps': ['localhost:%d' % ports[-1]] + 'ps': ['localhost:%d' % ports[-1]], } tf_config = {'cluster': cluster} procs = {} @@ -640,34 +604,24 @@ def _ps_worker_distribute_eval( os.environ[constant.SORT_COL_BY_NAME] = '1' set_gpu_id(gpus[0]) train_cmd = 'python -m easy_rec.python.eval --pipeline_config_path {} --checkpoint_path {} \ - --distribute_eval True --eval_result_path distribute_eval_result.txt'.format( - pipeline_config_path, checkpoint_path - ) - procs[chief_or_master] = run_cmd( - train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, chief_or_master) - ) + --distribute_eval True --eval_result_path distribute_eval_result.txt'.format(pipeline_config_path, checkpoint_path) + procs[chief_or_master] = run_cmd(train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, chief_or_master)) tf_config['task'] = {'type': 'ps', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') - procs['ps'] = run_cmd( - train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, 'ps') - ) + procs['ps'] = run_cmd(train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, 'ps')) for idx in range(num_worker - 1): tf_config['task'] = {'type': 'worker', 'index': idx} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id(gpus[idx + 1]) worker_name = 'worker_%d' % idx - procs[worker_name] = run_cmd( - train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, worker_name) - ) + procs[worker_name] = run_cmd(train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, worker_name)) if num_evaluator > 0: tf_config['task'] = {'type': 'evaluator', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') - procs['evaluator'] = run_cmd( - train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, 'evaluator') - ) + procs['evaluator'] = run_cmd(train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, 'evaluator')) return procs @@ -678,11 +632,7 @@ def _multi_worker_mirror_train(pipeline_config_path, test_dir, num_worker): if len(gpus) < num_worker: gpus = [None] * num_worker ports = _get_ports(num_worker) - tf_config = { - 'cluster': { - 'worker': ['localhost:%d' % ports[i] for i in range(num_worker)] - } - } + tf_config = {'cluster': {'worker': ['localhost:%d' % ports[i] for i in range(num_worker)]}} procs = {} train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path %s' % pipeline_config_path for idx in range(num_worker): @@ -690,9 +640,7 @@ def _multi_worker_mirror_train(pipeline_config_path, test_dir, num_worker): os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id(gpus[idx]) worker_name = 'worker_%d' % idx - procs[worker_name] = run_cmd( - train_cmd, '%s/log_%s.txt' % (test_dir, worker_name) - ) + procs[worker_name] = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, worker_name)) return procs @@ -707,7 +655,9 @@ def _multi_worker_hvd_train(pipeline_config_path, test_dir, num_worker): ports = _get_ports(num_worker) hosts = ','.join(['localhost:%d' % ports[i] for i in range(num_worker)]) train_cmd = 'horovodrun -np %d --hosts %s python -m easy_rec.python.train_eval --pipeline_config_path %s' % ( - num_worker, hosts, pipeline_config_path + num_worker, + hosts, + pipeline_config_path, ) proc = run_cmd(train_cmd, '%s/log_hvd.txt' % test_dir) proc_wait(proc, timeout=1200) @@ -722,12 +672,10 @@ def test_distributed_train_eval( edit_config_json=None, use_hvd=False, fit_on_eval=False, - num_epoch=0 + num_epoch=0, ): logging.info('testing pipeline config %s' % pipeline_config_path) - pipeline_config = _load_config_for_test( - pipeline_config_path, test_dir, total_steps, num_epoch - ) + pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, total_steps, num_epoch) if edit_config_json is not None: config_util.edit_config(pipeline_config, edit_config_json) @@ -735,10 +683,9 @@ def test_distributed_train_eval( pipeline_config.train_config.sync_replicas = False if pipeline_config.train_config.train_distribute not in [ DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.SokStrategy, ]: - pipeline_config.train_config.train_distribute =\ - DistributionStrategy.HorovodStrategy + pipeline_config.train_config.train_distribute = DistributionStrategy.HorovodStrategy train_config = pipeline_config.train_config config_util.save_pipeline_config(pipeline_config, test_dir) @@ -757,13 +704,11 @@ def test_distributed_train_eval( num_worker, num_evaluator, fit_on_eval, - fit_on_eval_steps=int(total_steps // 2) + fit_on_eval_steps=int(total_steps // 2), ) elif train_config.train_distribute == DistributionStrategy.MultiWorkerMirroredStrategy: num_worker = 2 - procs = _multi_worker_mirror_train( - test_pipeline_config_path, test_dir, num_worker - ) + procs = _multi_worker_mirror_train(test_pipeline_config_path, test_dir, num_worker) else: raise NotImplementedError @@ -822,36 +767,22 @@ def test_distribute_eval_test(cur_eval_path, test_dir): return False single_data = read_data_from_json_path(single_work_eval_path) distribute_data = read_data_from_json_path(distribute_eval_path) - single_ret = { - k: single_data[k] - for k in single_data.keys() if 'loss' not in k and 'step' not in k - } - distribute_ret = { - k: distribute_data[k] - for k in distribute_data.keys() if 'loss' not in k - } + single_ret = {k: single_data[k] for k in single_data.keys() if 'loss' not in k and 'step' not in k} + distribute_ret = {k: distribute_data[k] for k in distribute_data.keys() if 'loss' not in k} difference_num = 0.00001 for k in single_ret.keys(): - if (abs(single_ret[k] - distribute_ret[k]) > difference_num): + if abs(single_ret[k] - distribute_ret[k]) > difference_num: logging.error( - 'distribute_eval difference[%.8f] large than threshold[%.8f]' % - (abs(single_ret[k] - distribute_ret[k]), difference_num) + 'distribute_eval difference[%.8f] large than threshold[%.8f]' + % (abs(single_ret[k] - distribute_ret[k]), difference_num) ) return False return True -def test_distributed_eval( - pipeline_config_path, - checkpoint_path, - test_dir, - total_steps=50, - num_evaluator=0 -): +def test_distributed_eval(pipeline_config_path, checkpoint_path, test_dir, total_steps=50, num_evaluator=0): logging.info('testing pipeline config %s' % pipeline_config_path) - pipeline_config = _load_config_for_distribute_eval( - pipeline_config_path, test_dir - ) + pipeline_config = _load_config_for_distribute_eval(pipeline_config_path, test_dir) train_config = pipeline_config.train_config config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') @@ -863,8 +794,11 @@ def test_distributed_eval( if train_config.train_distribute == DistributionStrategy.NoStrategy: num_worker = 2 procs = _ps_worker_distribute_eval( - test_pipeline_config_path, checkpoint_path, test_dir, num_worker, - num_evaluator + test_pipeline_config_path, + checkpoint_path, + test_dir, + num_worker, + num_evaluator, ) else: raise NotImplementedError diff --git a/easy_rec/python/utils/tf_utils.py b/easy_rec/python/utils/tf_utils.py index c82495e7f..5ed647404 100644 --- a/easy_rec/python/utils/tf_utils.py +++ b/easy_rec/python/utils/tf_utils.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Common functions used for odps input.""" + import json import os @@ -29,7 +30,7 @@ def get_tf_type(field_type): DatasetConfig.STRING: tf.string, DatasetConfig.BOOL: tf.bool, DatasetConfig.FLOAT: tf.float32, - DatasetConfig.DOUBLE: tf.double + DatasetConfig.DOUBLE: tf.double, } assert field_type in type_map, 'invalid type: %s' % field_type return type_map[field_type] @@ -42,7 +43,7 @@ def get_col_type(tf_type): tf.string: 'STRING', tf.float32: 'FLOAT', tf.double: 'DOUBLE', - tf.bool: 'BOOLEAN' + tf.bool: 'BOOLEAN', } assert tf_type in type_map, 'invalid type: %s' % tf_type return type_map[tf_type] diff --git a/examples/data/amazon_books_data/process_amazon.py b/examples/data/amazon_books_data/process_amazon.py index 4f6467f67..0da6c044e 100644 --- a/examples/data/amazon_books_data/process_amazon.py +++ b/examples/data/amazon_books_data/process_amazon.py @@ -11,7 +11,7 @@ header=None, names=title, engine='python', - encoding='ISO-8859-1' + encoding='ISO-8859-1', ) print('Reading test data...') test = pd.read_table( @@ -20,7 +20,7 @@ header=None, names=title, engine='python', - encoding='ISO-8859-1' + encoding='ISO-8859-1', ) print('Start processing train data...') @@ -89,24 +89,16 @@ def gen_neg(): test_set_df = pd.DataFrame(test_set) print('Start writing amazon_train_data...') -train_set_df.to_csv( - r'amazon_train_data', index=False, sep='\t', mode='a', header=False -) +train_set_df.to_csv(r'amazon_train_data', index=False, sep='\t', mode='a', header=False) print('Start writing amazon_test_data...') -test_set_df.to_csv( - r'amazon_test_data', index=False, sep='\t', mode='a', header=False -) +test_set_df.to_csv(r'amazon_test_data', index=False, sep='\t', mode='a', header=False) print('Negative Sampling') train_book = train[['BookID']].drop_duplicates() test_book = test[['BookID']].drop_duplicates() negative_book = pd.concat([train_book, test_book]).drop_duplicates() -df_ones = pd.DataFrame( - 1, index=negative_book.index, columns=negative_book.columns -) +df_ones = pd.DataFrame(1, index=negative_book.index, columns=negative_book.columns) negative_book_data = pd.concat([negative_book, df_ones, negative_book], axis=1) new_header = ['id:int64', 'weight:float', 'feature:string'] -negative_book_data.to_csv( - r'negative_book_data', index=False, sep='\t', mode='a', header=new_header -) +negative_book_data.to_csv(r'negative_book_data', index=False, sep='\t', mode='a', header=new_header) print('Done.') diff --git a/examples/data/criteo/process_criteo_kaggle.py b/examples/data/criteo/process_criteo_kaggle.py index 68cb5aecf..f7671abdd 100644 --- a/examples/data/criteo/process_criteo_kaggle.py +++ b/examples/data/criteo/process_criteo_kaggle.py @@ -5,18 +5,12 @@ target_columns = ['label'] columns = target_columns + dense_features + category_features -data_train = pd.read_csv( - 'criteo_kaggle_display/train.txt', sep='\t', names=columns -) +data_train = pd.read_csv('criteo_kaggle_display/train.txt', sep='\t', names=columns) samples_num = data_train.shape[0] print('samples_num:', samples_num, round(samples_num * 0.9)) train_num = int(round(samples_num * 0.9)) -data_train[:train_num].to_csv( - r'criteo_train_data', index=False, sep='\t', mode='a', header=False -) -data_train[train_num:].to_csv( - r'criteo_test_data', index=False, sep='\t', mode='a', header=False -) +data_train[:train_num].to_csv(r'criteo_train_data', index=False, sep='\t', mode='a', header=False) +data_train[train_num:].to_csv(r'criteo_test_data', index=False, sep='\t', mode='a', header=False) print('Done.') diff --git a/examples/data/movielens_1m/process_ml_1m.py b/examples/data/movielens_1m/process_ml_1m.py index f10e4161e..bc3d64abd 100644 --- a/examples/data/movielens_1m/process_ml_1m.py +++ b/examples/data/movielens_1m/process_ml_1m.py @@ -16,7 +16,7 @@ def process_data(): header=None, names=users_title, engine='python', - encoding='ISO-8859-1' + encoding='ISO-8859-1', ) users = users.filter(regex='UserID|Gender|Age|JobID|ZipCode') # process the gender and age of user @@ -35,20 +35,14 @@ def process_data(): header=None, names=movies_title, engine='python', - encoding='ISO-8859-1' + encoding='ISO-8859-1', ) # split the title and year in Feature:'Title' pattern = re.compile(r'^(.*)\((\d+)\)$') - title_map = { - val: pattern.match(val).group(1) - for ii, val in enumerate(set(movies['Title'])) - } - year_map = { - val: pattern.match(val).group(2) - for ii, val in enumerate(set(movies['Title'])) - } + title_map = {val: pattern.match(val).group(1) for ii, val in enumerate(set(movies['Title']))} + year_map = {val: pattern.match(val).group(2) for ii, val in enumerate(set(movies['Title']))} movies['Year'] = movies['Title'].map(year_map) movies['Title'] = movies['Title'].map(title_map) @@ -61,7 +55,7 @@ def process_data(): header=None, names=ratings_title, engine='python', - encoding='ISO-8859-1' + encoding='ISO-8859-1', ) ratings = ratings.filter(regex='UserID|MovieID|ratings') # ratings of 4 and 5 are viewed as positive samples [label:1] @@ -89,10 +83,6 @@ def process_data(): # split train set and test set, and write to file print('Start writing to file.') -data_new[:665110].to_csv( - r'movies_train_data', index=False, sep='\t', mode='a', header=False -) -data_new[665110:].to_csv( - r'movies_test_data', index=False, sep='\t', mode='a', header=False -) +data_new[:665110].to_csv(r'movies_train_data', index=False, sep='\t', mode='a', header=False) +data_new[665110:].to_csv(r'movies_test_data', index=False, sep='\t', mode='a', header=False) print('Done.') diff --git a/git-lfs/git_lfs.py b/git-lfs/git_lfs.py index 3be516028..edeaeb55d 100644 --- a/git-lfs/git_lfs.py +++ b/git-lfs/git_lfs.py @@ -13,15 +13,13 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s[%(lineno)d] : %(message)s', - level=logging.INFO + level=logging.INFO, ) try: import oss2 except ImportError: - logging.error( - 'please install python_oss from https://github.com/aliyun/aliyun-oss-python-sdk.git' - ) + logging.error('please install python_oss from https://github.com/aliyun/aliyun-oss-python-sdk.git') sys.exit(1) git_bin_path = '.git_bin_path' @@ -50,11 +48,12 @@ def load_git_url(): for line_str in fin: line_str = line_str.strip() line_json = json.loads(line_str) - git_bin_url_map[line_json['leaf_path'] - ] = (line_json['sig'], line_json['remote_path']) + git_bin_url_map[line_json['leaf_path']] = ( + line_json['sig'], + line_json['remote_path'], + ) except Exception as ex: logging.warning('exception: %s' % str(ex)) - pass return git_bin_url_map @@ -65,7 +64,9 @@ def save_git_url(git_bin_url_map): for key in keys: val = git_bin_url_map[key] tmp_str = '{"leaf_path": "%s", "sig": "%s", "remote_path": "%s"}' % ( - key, val[0], val[1] + key, + val[0], + val[1], ) fout.write('%s\n' % tmp_str) @@ -106,9 +107,7 @@ def load_git_bin(): line_json = json.loads(line_str) file_arr[line_json['leaf_name']] = line_json['leaf_file'] except Exception as ex: - logging.warning( - '%s is corrupted : %s' % (git_bin_path, traceback.format_exc(ex)) - ) + logging.warning('%s is corrupted : %s' % (git_bin_path, traceback.format_exc(ex))) return file_arr @@ -121,7 +120,8 @@ def save_git_bin(git_arr): leaf_files.sort() # make sure that leaf_name is in front of leaf_file tmp_str = '{"leaf_name": "%s", "leaf_file": %s}' % ( - leaf_path, json.dumps(leaf_files) + leaf_path, + json.dumps(leaf_files), ) fout.write('%s\n' % tmp_str) @@ -224,9 +224,7 @@ def get_yes_no(msg): if __name__ == '__main__': if len(sys.argv) < 2: - logging.error( - 'usage: python git_lfs.py [pull] [push] [add filename] [resolve_conflict]' - ) + logging.error('usage: python git_lfs.py [pull] [push] [add filename] [resolve_conflict]') sys.exit(1) home_directory = os.path.expanduser('~') with open('.git_oss_config_pub', 'r') as fin: @@ -243,9 +241,7 @@ def get_yes_no(msg): if line_str.startswith('#'): continue line_str = line_str.replace('~/', home_directory + '/') - line_str = line_str.replace( - '${TMPDIR}/', os.environ.get('TMPDIR', '/tmp/') - ) + line_str = line_str.replace('${TMPDIR}/', os.environ.get('TMPDIR', '/tmp/')) line_str = line_str.replace('${PROJECT_NAME}', get_proj_name()) line_tok = [x.strip() for x in line_str.split('=') if x != ''] if line_tok[0] == 'host': @@ -257,18 +253,13 @@ def get_yes_no(msg): elif line_tok[0] == 'git_oss_private_config': git_oss_private_path = line_tok[1] if git_oss_private_path.startswith('~/'): - git_oss_private_path = os.path.join( - home_directory, git_oss_private_path[2:] - ) + git_oss_private_path = os.path.join(home_directory, git_oss_private_path[2:]) elif line_tok[0] == 'git_oss_cache_dir': git_oss_cache_dir = line_tok[1] elif line_tok[0] == 'accl_endpoint': accl_endpoint = line_tok[1] - logging.info( - 'git_oss_data_dir=%s, host=%s, bucket_name=%s' % - (git_oss_data_dir, host, bucket_name) - ) + logging.info('git_oss_data_dir=%s, host=%s, bucket_name=%s' % (git_oss_data_dir, host, bucket_name)) logging.info('git_oss_cache_dir: %s' % git_oss_cache_dir) @@ -289,10 +280,7 @@ def get_yes_no(msg): oss_auth = oss2.Auth(accessid, accesskey) oss_bucket = oss2.Bucket(oss_auth, host, bucket_name) else: - logging.info( - 'git_oss_private_path[%s] is not found, read-only mode' % - git_oss_private_path - ) + logging.info('git_oss_private_path[%s] is not found, read-only mode' % git_oss_private_path) # pull only mode oss_auth = None oss_bucket = None @@ -363,14 +351,12 @@ def get_yes_no(msg): local_sig = '' update = False - if len(sys.argv - ) > 2 and (sys.argv[2] == '-f' or sys.argv[2] == '--force'): + if len(sys.argv) > 2 and (sys.argv[2] == '-f' or sys.argv[2] == '--force'): update = True else: if has_conflict(leaf_path, leaf_files): update = get_yes_no( - 'update %s using remote file[remote_sig=%s local_sig=%s]?[N/Y]' % - (leaf_path, remote_sig, local_sig) + 'update %s using remote file[remote_sig=%s local_sig=%s]?[N/Y]' % (leaf_path, remote_sig, local_sig) ) else: update = True @@ -393,13 +379,9 @@ def get_yes_no(msg): if sys.platform.startswith('linux'): subprocess.check_output(['wget', url, '-O', tar_tmp_path]) elif sys.platform.startswith('darwin'): - subprocess.check_output( - ['curl', url, '--output', tar_tmp_path] - ) + subprocess.check_output(['curl', url, '--output', tar_tmp_path]) elif sys.platform.startswith('win'): - subprocess.check_output( - ['curl', url, '--output', tar_tmp_path] - ) + subprocess.check_output(['curl', url, '--output', tar_tmp_path]) else: in_cache = True logging.info('%s is in cache' % file_name_with_sig) @@ -411,10 +393,7 @@ def get_yes_no(msg): logging.warning('cache invalid, will download from remote') os.remove(tar_tmp_path) continue - logging.warning( - 'download failed, local_sig(%s) != remote_sig(%s)' % - (local_sig, remote_sig) - ) + logging.warning('download failed, local_sig(%s) != remote_sig(%s)' % (local_sig, remote_sig)) except subprocess.CalledProcessError as ex: logging.error('exception: %s' % str(ex)) except oss2.exceptions.RequestError as ex: @@ -441,7 +420,6 @@ def get_yes_no(msg): bin_file_map = load_git_bin() except Exception as ex: logging.warning('load_git_bin exception: %s' % traceback.format_exc(ex)) - pass leaf_dirs = list_leafs(add_path) any_new = False for leaf_path, leaf_files in leaf_dirs: @@ -470,7 +448,6 @@ def get_yes_no(msg): bin_file_map = load_git_bin() except Exception as ex: logging.warning('load_git_bin exception: %s' % traceback.format_exc(ex)) - pass leaf_dirs = list_leafs(del_path) any_update = False for leaf_path, leaf_files in leaf_dirs: @@ -520,10 +497,7 @@ def get_yes_no(msg): else: git_objs[leaf_name] = leaf_file else: - logging.warning( - 'invalid state: merge_start = %d, line_str = %s' % - (merge_start, line_str) - ) + logging.warning('invalid state: merge_start = %d, line_str = %s' % (merge_start, line_str)) save_git_bin(git_objs) git_bin_url_map = {} @@ -539,18 +513,14 @@ def get_yes_no(msg): elif merge_start in [0, 1, 2]: line_json = json.loads(line_str) if line_json['leaf_path'] in git_objs: - git_bin_url_map[line_json['leaf_path'] - ] = (line_json['sig'], line_json['remote_path']) + git_bin_url_map[line_json['leaf_path']] = ( + line_json['sig'], + line_json['remote_path'], + ) else: - logging.warning( - 'invalid state: merge_start = %d, line_str = %s' % - (merge_start, line_str) - ) + logging.warning('invalid state: merge_start = %d, line_str = %s' % (merge_start, line_str)) save_git_url(git_bin_url_map) logging.info('all conflicts fixed.') else: logging.warning('invalid cmd: %s' % sys.argv[1]) - logging.warning( - 'choices are: %s' % - ','.join(['push', 'pull', 'add', 'remove', 'resolve_conflict']) - ) + logging.warning('choices are: %s' % ','.join(['push', 'pull', 'add', 'remove', 'resolve_conflict'])) diff --git a/pai_jobs/run.py b/pai_jobs/run.py index db6e162a7..a9c920041 100644 --- a/pai_jobs/run.py +++ b/pai_jobs/run.py @@ -1,8 +1,12 @@ +#!/usr/bin/env python # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. +"""Entry script of pai -name easyrec command.""" + from __future__ import print_function import logging + # use few threads to avoid oss error import os import time @@ -14,15 +18,23 @@ import easy_rec from easy_rec.python.inference.odps_predictor import ODPSPredictor from easy_rec.python.inference.vector_retrieve import VectorRetrieve +from easy_rec.python.main import _train_and_evaluate_impl as train_and_evaluate_impl from easy_rec.python.tools.pre_check import run_check - -from easy_rec.python.utils import config_util, constant, estimator_utils, fg_util, hpo_util, pai_util # NOQA -from easy_rec.python.utils.distribution_utils import DistributionStrategyMap, set_distribution_config # NOQA +from easy_rec.python.utils import ( # NOQA + config_util, + constant, + estimator_utils, + fg_util, + hpo_util, + pai_util, +) +from easy_rec.python.utils.distribution_utils import ( # NOQA + DistributionStrategyMap, + set_distribution_config, + set_tf_config_and_get_train_worker_num, +) os.environ['IS_ON_PAI'] = '1' - -from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_train_worker_num # NOQA - os.environ['OENV_MultiWriteThreadsNum'] = '4' os.environ['OENV_MultiCopyThreadsNum'] = '4' @@ -34,202 +46,118 @@ logging.error('failed to import tfio: %s' % str(ex)) tf.disable_eager_execution() -from easy_rec.python.main import _train_and_evaluate_impl as train_and_evaluate_impl # NOQA +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) - -tf.app.flags.DEFINE_string( - 'worker_hosts', '', 'Comma-separated list of hostname:port pairs' -) -tf.app.flags.DEFINE_string( - 'ps_hosts', '', 'Comma-separated list of hostname:port pairs' -) +tf.app.flags.DEFINE_string('worker_hosts', '', 'Comma-separated list of hostname:port pairs') +tf.app.flags.DEFINE_string('ps_hosts', '', 'Comma-separated list of hostname:port pairs') tf.app.flags.DEFINE_string('job_name', '', 'task type, ps/worker') tf.app.flags.DEFINE_integer('task_index', 0, 'Index of task within the job') tf.app.flags.DEFINE_string('config', '', 'EasyRec config file path') -tf.app.flags.DEFINE_string( - 'cmd', 'train', 'command type, train/evaluate/export' -) +tf.app.flags.DEFINE_string('cmd', 'train', 'command type, train/evaluate/export') tf.app.flags.DEFINE_string('tables', '', 'tables passed by pai command') # flags for train -tf.app.flags.DEFINE_integer( - 'num_gpus_per_worker', 1, 'number of gpu to use in training' -) -tf.app.flags.DEFINE_boolean( - 'with_evaluator', False, 'whether a evaluator is necessary' -) +tf.app.flags.DEFINE_integer('num_gpus_per_worker', 1, 'number of gpu to use in training') +tf.app.flags.DEFINE_boolean('with_evaluator', False, 'whether a evaluator is necessary') tf.app.flags.DEFINE_string( - 'eval_method', 'none', 'default to none, choices are [none: not evaluate,' + - 'master: evaluate on master, separate: evaluate on a separate task]' + 'eval_method', + 'none', + 'default to none, choices are [none: not evaluate,' + + 'master: evaluate on master, separate: evaluate on a separate task]', ) -tf.app.flags.DEFINE_string( - 'distribute_strategy', '', 'training distribute strategy' -) +tf.app.flags.DEFINE_string('distribute_strategy', '', 'training distribute strategy') tf.app.flags.DEFINE_string('edit_config_json', '', 'edit config json string') tf.app.flags.DEFINE_string('train_tables', '', 'tables used for train') tf.app.flags.DEFINE_string('eval_tables', '', 'tables used for evaluation') tf.app.flags.DEFINE_string('boundary_table', '', 'tables used for boundary') tf.app.flags.DEFINE_string('sampler_table', '', 'tables used for sampler') -tf.app.flags.DEFINE_string( - 'fine_tune_checkpoint', None, 'finetune checkpoint path' -) -tf.app.flags.DEFINE_string( - 'query_table', '', 'table used for retrieve vector neighbours' -) -tf.app.flags.DEFINE_string( - 'doc_table', '', 'table used for be retrieved as indexed vectors' -) +tf.app.flags.DEFINE_string('fine_tune_checkpoint', None, 'finetune checkpoint path') +tf.app.flags.DEFINE_string('query_table', '', 'table used for retrieve vector neighbours') +tf.app.flags.DEFINE_string('doc_table', '', 'table used for be retrieved as indexed vectors') +tf.app.flags.DEFINE_enum('knn_distance', 'inner_product', ['l2', 'inner_product'], 'type of knn distance') +tf.app.flags.DEFINE_integer('knn_num_neighbours', None, 'top n neighbours to be retrieved') +tf.app.flags.DEFINE_integer('knn_feature_dims', None, 'number of feature dimensions') tf.app.flags.DEFINE_enum( - 'knn_distance', 'inner_product', ['l2', 'inner_product'], - 'type of knn distance' -) -tf.app.flags.DEFINE_integer( - 'knn_num_neighbours', None, 'top n neighbours to be retrieved' -) -tf.app.flags.DEFINE_integer( - 'knn_feature_dims', None, 'number of feature dimensions' -) -tf.app.flags.DEFINE_enum( - 'knn_index_type', 'ivfflat', + 'knn_index_type', + 'ivfflat', ['flat', 'ivfflat', 'ivfpq', 'gpu_flat', 'gpu_ivfflat', 'gpu_ivfpg'], - 'knn index type' -) -tf.app.flags.DEFINE_string( - 'knn_feature_delimiter', ',', 'delimiter for feature vectors' -) -tf.app.flags.DEFINE_integer( - 'knn_nlist', 5, 'number of split part on each worker' + 'knn index type', ) +tf.app.flags.DEFINE_string('knn_feature_delimiter', ',', 'delimiter for feature vectors') +tf.app.flags.DEFINE_integer('knn_nlist', 5, 'number of split part on each worker') +tf.app.flags.DEFINE_integer('knn_nprobe', 2, 'number of probe part on each worker') tf.app.flags.DEFINE_integer( - 'knn_nprobe', 2, 'number of probe part on each worker' -) -tf.app.flags.DEFINE_integer( - 'knn_compress_dim', 8, - 'number of dimensions after compress for `ivfpq` and `gpu_ivfpq`' + 'knn_compress_dim', + 8, + 'number of dimensions after compress for `ivfpq` and `gpu_ivfpq`', ) # flags used for evaluate & export tf.app.flags.DEFINE_string( - 'checkpoint_path', '', 'checkpoint to be evaluated or exported ' - 'if not specified, use the latest checkpoint ' - 'in train_config.model_dir' + 'checkpoint_path', + '', + 'checkpoint to be evaluated or exported ' 'if not specified, use the latest checkpoint ' 'in train_config.model_dir', ) # flags used for evaluate -tf.app.flags.DEFINE_string( - 'eval_result_path', 'eval_result.txt', 'eval result metric file' -) -tf.app.flags.DEFINE_bool( - 'distribute_eval', False, - 'use distribute parameter server for train and eval.' -) +tf.app.flags.DEFINE_string('eval_result_path', 'eval_result.txt', 'eval result metric file') +tf.app.flags.DEFINE_bool('distribute_eval', False, 'use distribute parameter server for train and eval.') # flags used for export -tf.app.flags.DEFINE_string( - 'export_dir', '', 'directory where model should be exported to' -) +tf.app.flags.DEFINE_string('export_dir', '', 'directory where model should be exported to') tf.app.flags.DEFINE_bool('clear_export', False, 'remove export_dir if exists') -tf.app.flags.DEFINE_string( - 'export_done_file', '', 'a flag file to signal that export model is done' -) -tf.app.flags.DEFINE_integer( - 'max_wait_ckpt_ts', 0, 'max wait time in seconds for checkpoints' -) -tf.app.flags.DEFINE_boolean( - 'continue_train', True, 'use the same model to continue train or not' -) +tf.app.flags.DEFINE_string('export_done_file', '', 'a flag file to signal that export model is done') +tf.app.flags.DEFINE_integer('max_wait_ckpt_ts', 0, 'max wait time in seconds for checkpoints') +tf.app.flags.DEFINE_boolean('continue_train', True, 'use the same model to continue train or not') # flags used for predict -tf.app.flags.DEFINE_string( - 'saved_model_dir', '', 'directory where saved_model.pb exists' -) +tf.app.flags.DEFINE_string('saved_model_dir', '', 'directory where saved_model.pb exists') tf.app.flags.DEFINE_string('outputs', '', 'output tables') +tf.app.flags.DEFINE_string('all_cols', '', 'union of (selected_cols, reserved_cols), separated with , ') tf.app.flags.DEFINE_string( - 'all_cols', '', 'union of (selected_cols, reserved_cols), separated with , ' + 'all_col_types', + '', + 'column data types, for build record defaults, separated with ,', ) +tf.app.flags.DEFINE_string('selected_cols', '', 'columns to keep from input table, they are separated with ,') +tf.app.flags.DEFINE_string('reserved_cols', '', 'columns to keep from input table, they are separated with ,') tf.app.flags.DEFINE_string( - 'all_col_types', '', - 'column data types, for build record defaults, separated with ,' -) -tf.app.flags.DEFINE_string( - 'selected_cols', '', - 'columns to keep from input table, they are separated with ,' -) -tf.app.flags.DEFINE_string( - 'reserved_cols', '', - 'columns to keep from input table, they are separated with ,' -) -tf.app.flags.DEFINE_string( - 'output_cols', None, - 'output columns, such as: score float. multiple columns are separated by ,' + 'output_cols', + None, + 'output columns, such as: score float. multiple columns are separated by ,', ) tf.app.flags.DEFINE_integer('batch_size', 1024, 'predict batch size') -tf.app.flags.DEFINE_string( - 'profiling_file', None, - 'time stat file which can be viewed using chrome tracing' -) +tf.app.flags.DEFINE_string('profiling_file', None, 'time stat file which can be viewed using chrome tracing') tf.app.flags.DEFINE_string('redis_url', None, 'export to redis url, host:port') tf.app.flags.DEFINE_string('redis_passwd', None, 'export to redis passwd') tf.app.flags.DEFINE_integer('redis_threads', 5, 'export to redis threads') -tf.app.flags.DEFINE_integer( - 'redis_batch_size', 1024, 'export to redis batch_size' -) -tf.app.flags.DEFINE_integer( - 'redis_timeout', 600, 'export to redis time_out in seconds' -) -tf.app.flags.DEFINE_integer( - 'redis_expire', 24, 'export to redis expire time in hour' -) -tf.app.flags.DEFINE_string( - 'redis_embedding_version', '', 'redis embedding version' -) +tf.app.flags.DEFINE_integer('redis_batch_size', 1024, 'export to redis batch_size') +tf.app.flags.DEFINE_integer('redis_timeout', 600, 'export to redis time_out in seconds') +tf.app.flags.DEFINE_integer('redis_expire', 24, 'export to redis expire time in hour') +tf.app.flags.DEFINE_string('redis_embedding_version', '', 'redis embedding version') tf.app.flags.DEFINE_integer('redis_write_kv', 1, 'whether write kv ') -tf.app.flags.DEFINE_string( - 'oss_path', None, 'write embed objects to oss folder, oss://bucket/folder' -) +tf.app.flags.DEFINE_string('oss_path', None, 'write embed objects to oss folder, oss://bucket/folder') tf.app.flags.DEFINE_string('oss_endpoint', None, 'oss endpoint') tf.app.flags.DEFINE_string('oss_ak', None, 'oss ak') tf.app.flags.DEFINE_string('oss_sk', None, 'oss sk') -tf.app.flags.DEFINE_integer( - 'oss_threads', 10, '# threads access oss at the same time' -) -tf.app.flags.DEFINE_integer( - 'oss_timeout', 10, 'connect to oss, time_out in seconds' -) +tf.app.flags.DEFINE_integer('oss_threads', 10, '# threads access oss at the same time') +tf.app.flags.DEFINE_integer('oss_timeout', 10, 'connect to oss, time_out in seconds') tf.app.flags.DEFINE_integer('oss_expire', 24, 'oss expire time in hours') -tf.app.flags.DEFINE_integer( - 'oss_write_kv', 1, 'whether to write embedding to oss' -) -tf.app.flags.DEFINE_string( - 'oss_embedding_version', '', 'oss embedding version' -) +tf.app.flags.DEFINE_integer('oss_write_kv', 1, 'whether to write embedding to oss') +tf.app.flags.DEFINE_string('oss_embedding_version', '', 'oss embedding version') tf.app.flags.DEFINE_bool('verbose', False, 'print more debug information') -tf.app.flags.DEFINE_bool( - 'place_embedding_on_cpu', False, - 'whether to place embedding variables on cpu' -) +tf.app.flags.DEFINE_bool('place_embedding_on_cpu', False, 'whether to place embedding variables on cpu') # for automl hyper parameter tuning tf.app.flags.DEFINE_string('model_dir', None, 'model directory') -tf.app.flags.DEFINE_bool( - 'clear_model', False, 'remove model directory if exists' -) -tf.app.flags.DEFINE_string( - 'hpo_param_path', None, 'hyperparameter tuning param path' -) -tf.app.flags.DEFINE_string( - 'hpo_metric_save_path', None, 'hyperparameter save metric path' -) +tf.app.flags.DEFINE_bool('clear_model', False, 'remove model directory if exists') +tf.app.flags.DEFINE_string('hpo_param_path', None, 'hyperparameter tuning param path') +tf.app.flags.DEFINE_string('hpo_metric_save_path', None, 'hyperparameter save metric path') tf.app.flags.DEFINE_string('asset_files', None, 'extra files to add to export') tf.app.flags.DEFINE_bool('check_mode', False, 'is use check mode') tf.app.flags.DEFINE_string('fg_json_path', None, '') -tf.app.flags.DEFINE_bool( - 'enable_avx_str_split', False, 'enable avx str split to speedup' -) +tf.app.flags.DEFINE_bool('enable_avx_str_split', False, 'enable avx str split to speedup') FLAGS = tf.app.flags.FLAGS @@ -246,23 +174,14 @@ def set_selected_cols(pipeline_config, selected_cols, all_cols, all_col_types): if all_cols: all_cols_arr = all_cols.split(',') all_col_types_arr = all_col_types.split(',') - all_col_types_map = { - x.strip(): y.strip() - for x, y in zip(all_cols_arr, all_col_types_arr) - } + all_col_types_map = {x.strip(): y.strip() for x, y in zip(all_cols_arr, all_col_types_arr)} selected_cols_arr = [x.strip() for x in selected_cols.split(',')] selected_col_types = [all_col_types_map[x] for x in selected_cols_arr] selected_col_types = ','.join(selected_col_types) pipeline_config.data_config.selected_col_types = selected_col_types - print( - '[run.py] data_config.selected_cols = "%s"' % - pipeline_config.data_config.selected_cols - ) - print( - '[run.py] data_config.selected_col_types = "%s"' % - pipeline_config.data_config.selected_col_types - ) + print('[run.py] data_config.selected_cols = "%s"' % pipeline_config.data_config.selected_cols) + print('[run.py] data_config.selected_col_types = "%s"' % pipeline_config.data_config.selected_col_types) def _wait_ckpt(ckpt_path, max_wait_ts): @@ -275,9 +194,7 @@ def _wait_ckpt(ckpt_path, max_wait_ts): logging.info('wait for checkpoint in directory[%s]' % ckpt_path) time.sleep(30) else: - logging.info( - 'find checkpoint[%s] in directory[%s]' % (tmp_ckpt, ckpt_path) - ) + logging.info('find checkpoint[%s] in directory[%s]' % (tmp_ckpt, ckpt_path)) break else: while time.time() - start_ts < max_wait_ts: @@ -293,9 +210,7 @@ def main(argv): pai_util.set_on_pai() if FLAGS.enable_avx_str_split: constant.enable_avx_str_split() - logging.info( - 'will enable avx str split: %s' % constant.is_avx_str_split_enabled() - ) + logging.info('will enable avx str split: %s' % constant.is_avx_str_split_enabled()) if FLAGS.distribute_eval: os.environ['distribute_eval'] = 'True' @@ -310,14 +225,16 @@ def main(argv): num_gpus_per_worker = FLAGS.num_gpus_per_worker worker_hosts = FLAGS.worker_hosts.split(',') num_worker = len(worker_hosts) - assert FLAGS.distribute_strategy in DistributionStrategyMap, \ - 'invalid distribute_strategy [%s], available ones are %s' % ( - FLAGS.distribute_strategy, ','.join(DistributionStrategyMap.keys())) + assert FLAGS.distribute_strategy in DistributionStrategyMap, ( + 'invalid distribute_strategy [%s], available ones are %s' + % ( + FLAGS.distribute_strategy, + ','.join(DistributionStrategyMap.keys()), + ) + ) if FLAGS.config: - config = pai_util.process_config( - FLAGS.config, FLAGS.task_index, len(FLAGS.worker_hosts.split(',')) - ) + config = pai_util.process_config(FLAGS.config, FLAGS.task_index, len(FLAGS.worker_hosts.split(','))) pipeline_config = config_util.get_configs_from_pipeline_file(config, False) # should be in front of edit_config_json step @@ -334,22 +251,17 @@ def main(argv): pipeline_config.model_dir = FLAGS.model_dir pipeline_config.model_dir = pipeline_config.model_dir.strip() print('[run.py] update model_dir to %s' % pipeline_config.model_dir) - assert pipeline_config.model_dir.startswith( - 'oss://' - ), 'invalid model_dir format: %s' % pipeline_config.model_dir + assert pipeline_config.model_dir.startswith('oss://'), 'invalid model_dir format: %s' % pipeline_config.model_dir if FLAGS.asset_files: - pipeline_config.export_config.asset_files.extend( - FLAGS.asset_files.split(',') - ) + pipeline_config.export_config.asset_files.extend(FLAGS.asset_files.split(',')) if FLAGS.config: if not pipeline_config.model_dir.endswith('/'): pipeline_config.model_dir += '/' if FLAGS.clear_model: - if gfile.IsDirectory(pipeline_config.model_dir - ) and estimator_utils.is_chief(): + if gfile.IsDirectory(pipeline_config.model_dir) and estimator_utils.is_chief(): gfile.DeleteRecursively(pipeline_config.model_dir) if FLAGS.max_wait_ckpt_ts > 0: @@ -363,10 +275,9 @@ def main(argv): if not FLAGS.train_tables and FLAGS.tables: tables = FLAGS.tables.split(',') - assert len( - tables - ) >= 2, 'at least 2 tables must be specified, but only[%d]: %s' % ( - len(tables), FLAGS.tables + assert len(tables) >= 2, 'at least 2 tables must be specified, but only[%d]: %s' % ( + len(tables), + FLAGS.tables, ) if FLAGS.train_tables: @@ -392,9 +303,7 @@ def main(argv): if FLAGS.boundary_table: logging.info('Load boundary_table: %s' % FLAGS.boundary_table) - config_util.add_boundaries_to_config( - pipeline_config, FLAGS.boundary_table - ) + config_util.add_boundaries_to_config(pipeline_config, FLAGS.boundary_table) if FLAGS.sampler_table: pipeline_config.data_config.negative_sampler.input_path = FLAGS.sampler_table @@ -402,8 +311,10 @@ def main(argv): if FLAGS.train_tables or FLAGS.tables: # parse selected_cols set_selected_cols( - pipeline_config, FLAGS.selected_cols, FLAGS.all_cols, - FLAGS.all_col_types + pipeline_config, + FLAGS.selected_cols, + FLAGS.all_cols, + FLAGS.all_col_types, ) else: pipeline_config.data_config.selected_cols = '' @@ -422,9 +333,7 @@ def main(argv): print('[run.py] with_evaluator %s' % str(FLAGS.with_evaluator)) print('[run.py] eval_method %s' % FLAGS.eval_method) - assert FLAGS.eval_method in [ - 'none', 'master', 'separate' - ], 'invalid evalaute_method: %s' % FLAGS.eval_method + assert FLAGS.eval_method in ['none', 'master', 'separate'], 'invalid evalaute_method: %s' % FLAGS.eval_method # with_evaluator is depreciated, keeped for compatibility if FLAGS.with_evaluator: @@ -436,32 +345,28 @@ def main(argv): FLAGS.task_index, FLAGS.job_name, distribute_strategy=distribute_strategy, - eval_method=FLAGS.eval_method - ) - set_distribution_config( - pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy + eval_method=FLAGS.eval_method, ) + set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy) logging.info('run.py check_mode: %s .' % FLAGS.check_mode) train_and_evaluate_impl( pipeline_config, continue_train=FLAGS.continue_train, - check_mode=FLAGS.check_mode + check_mode=FLAGS.check_mode, ) if FLAGS.hpo_metric_save_path: hpo_util.save_eval_metrics( pipeline_config.model_dir, metric_save_path=FLAGS.hpo_metric_save_path, - has_evaluator=(FLAGS.eval_method == 'separate') + has_evaluator=(FLAGS.eval_method == 'separate'), ) elif FLAGS.cmd == 'evaluate': check_param('config') # TODO: support multi-worker evaluation if not FLAGS.distribute_eval: - assert len( - FLAGS.worker_hosts.split(',') - ) == 1, 'evaluate only need 1 worker' + assert len(FLAGS.worker_hosts.split(',')) == 1, 'evaluate only need 1 worker' config_util.auto_expand_share_feature_configs(pipeline_config) if FLAGS.eval_tables: @@ -475,17 +380,17 @@ def main(argv): FLAGS.worker_hosts, FLAGS.task_index, FLAGS.job_name, - eval_method='none' - ) - set_distribution_config( - pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy + eval_method='none', ) + set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy) if FLAGS.eval_tables or FLAGS.tables: # parse selected_cols set_selected_cols( - pipeline_config, FLAGS.selected_cols, FLAGS.all_cols, - FLAGS.all_col_types + pipeline_config, + FLAGS.selected_cols, + FLAGS.all_cols, + FLAGS.all_col_types, ) else: pipeline_config.data_config.selected_cols = '' @@ -496,17 +401,13 @@ def main(argv): logging.info('will_use_distribute_eval') distribute_eval = os.environ.get('distribute_eval') logging.info('distribute_eval = {}'.format(distribute_eval)) - easy_rec.distribute_evaluate( - pipeline_config, FLAGS.checkpoint_path, None, FLAGS.eval_result_path - ) + easy_rec.distribute_evaluate(pipeline_config, FLAGS.checkpoint_path, None, FLAGS.eval_result_path) else: os.environ['distribute_eval'] = 'False' logging.info('will_use_eval') distribute_eval = os.environ.get('distribute_eval') logging.info('distribute_eval = {}'.format(distribute_eval)) - easy_rec.evaluate( - pipeline_config, FLAGS.checkpoint_path, None, FLAGS.eval_result_path - ) + easy_rec.evaluate(pipeline_config, FLAGS.checkpoint_path, None, FLAGS.eval_result_path) elif FLAGS.cmd == 'export': check_param('export_dir') check_param('config') @@ -555,7 +456,7 @@ def main(argv): FLAGS.worker_hosts, FLAGS.task_index, FLAGS.job_name, - eval_method='none' + eval_method='none', ) assert len(FLAGS.worker_hosts.split(',')) == 1, 'export only need 1 worker' @@ -571,8 +472,12 @@ def main(argv): extra_params = redis_params extra_params.update(oss_params) export_out_dir = easy_rec.export( - export_dir, pipeline_config, FLAGS.checkpoint_path, FLAGS.asset_files, - FLAGS.verbose, **extra_params + export_dir, + pipeline_config, + FLAGS.checkpoint_path, + FLAGS.asset_files, + FLAGS.verbose, + **extra_params, ) if FLAGS.export_done_file: flag_file = os.path.join(export_out_dir, FLAGS.export_done_file) @@ -582,12 +487,8 @@ def main(argv): elif FLAGS.cmd == 'predict': check_param('tables') check_param('saved_model_dir') - logging.info( - 'will use the following columns as model input: %s' % FLAGS.selected_cols - ) - logging.info( - 'will copy the following columns to output: %s' % FLAGS.reserved_cols - ) + logging.info('will use the following columns as model input: %s' % FLAGS.selected_cols) + logging.info('will copy the following columns to output: %s' % FLAGS.reserved_cols) profiling_file = FLAGS.profiling_file if FLAGS.task_index == 0 else None if profiling_file is not None: @@ -597,12 +498,10 @@ def main(argv): fg_json_path=FLAGS.fg_json_path, profiling_file=profiling_file, all_cols=FLAGS.all_cols, - all_col_types=FLAGS.all_col_types + all_col_types=FLAGS.all_col_types, ) input_table, output_table = FLAGS.tables, FLAGS.outputs - logging.info( - 'input_table = %s, output_table = %s' % (input_table, output_table) - ) + logging.info('input_table = %s, output_table = %s' % (input_table, output_table)) worker_num = len(FLAGS.worker_hosts.split(',')) predictor.predict_impl( input_table, @@ -611,7 +510,7 @@ def main(argv): output_cols=FLAGS.output_cols, batch_size=FLAGS.batch_size, slice_id=FLAGS.task_index, - slice_num=worker_num + slice_num=worker_num, ) elif FLAGS.cmd == 'export_checkpoint': check_param('export_dir') @@ -621,7 +520,7 @@ def main(argv): FLAGS.worker_hosts, FLAGS.task_index, FLAGS.job_name, - eval_method='none' + eval_method='none', ) assert len(FLAGS.worker_hosts.split(',')) == 1, 'export only need 1 woker' config_util.auto_expand_share_feature_configs(pipeline_config) @@ -630,20 +529,23 @@ def main(argv): export_path=FLAGS.export_dir + '/model', checkpoint_path=FLAGS.checkpoint_path, asset_files=FLAGS.asset_files, - verbose=FLAGS.verbose + verbose=FLAGS.verbose, ) elif FLAGS.cmd == 'vector_retrieve': check_param('knn_distance') assert FLAGS.knn_feature_dims is not None, '`knn_feature_dims` should not be None' assert FLAGS.knn_num_neighbours is not None, '`knn_num_neighbours` should not be None' - query_table, doc_table, output_table = FLAGS.query_table, FLAGS.doc_table, FLAGS.outputs + query_table, doc_table, output_table = ( + FLAGS.query_table, + FLAGS.doc_table, + FLAGS.outputs, + ) if not query_table: tables = FLAGS.tables.split(',') - assert len( - tables - ) >= 1, 'at least 1 tables must be specified, but only[%d]: %s' % ( - len(tables), FLAGS.tables + assert len(tables) >= 1, 'at least 1 tables must be specified, but only[%d]: %s' % ( + len(tables), + FLAGS.tables, ) query_table = tables[0] doc_table = tables[1] if len(tables) > 1 else query_table @@ -659,16 +561,14 @@ def main(argv): index_type=FLAGS.knn_index_type, nlist=FLAGS.knn_nlist, nprobe=FLAGS.knn_nprobe, - m=FLAGS.knn_compress_dim + m=FLAGS.knn_compress_dim, ) worker_hosts = FLAGS.worker_hosts.split(',') knn(FLAGS.knn_num_neighbours, FLAGS.task_index, len(worker_hosts)) elif FLAGS.cmd == 'check': run_check(pipeline_config, FLAGS.tables) else: - raise ValueError( - 'cmd should be one of train/evaluate/export/predict/export_checkpoint/vector_retrieve' - ) + raise ValueError('cmd should be one of train/evaluate/export/predict/export_checkpoint/vector_retrieve') if __name__ == '__main__': diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..0b02db173 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,52 @@ +[tool.black] +line-length = 120 +target-version = ['py38', 'py39', 'py310'] +skip-string-normalization = false +# 注意：Black 不支持 2 空格缩进！Black 强制使用 4 空格 + +[tool.ruff] +line-length = 120 +indent-width = 2 # 添加缩进配置 +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "W", # pycodestyle warnings + "I", # isort +] +ignore = [ + "E501", # line too long (handled by black) + "E203", # whitespace before ':' (conflicts with black) + "E111", # indentation is not a multiple of four + "E114", # indentation is not a multiple of four (comment) + "E117", # over-indented +] + +[tool.ruff.format] +indent-style = "space" +quote-style = "single" + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "W", # pycodestyle warnings + "I", # isort +] +ignore = [ + "E501", # line too long (handled by formatter) + "E203", # whitespace before ':' (conflicts with formatter) + "E111", # indentation is not a multiple of four + "E114", # indentation is not a multiple of four (comment) + "E117", # over-indented +] + +[tool.isort] +profile = "black" +line_length = 120 +indent = 2 # isort 的缩进配置 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +known_first_party = ["easy_rec"] diff --git a/scripts/ci_test_change_files.py b/scripts/ci_test_change_files.py index 509f5913f..8538ad4f1 100644 --- a/scripts/ci_test_change_files.py +++ b/scripts/ci_test_change_files.py @@ -11,18 +11,12 @@ logging.error(ex) sys.exit(2) -logging.basicConfig( - level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s' -) +logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( - '--pull_request_num', type=int, default=None, help='pull request number' - ) - parser.add_argument( - '--exclude_dirs', nargs='*', type=str, help='the directory to be ignored' - ) + parser.add_argument('--pull_request_num', type=int, default=None, help='pull request number') + parser.add_argument('--exclude_dirs', nargs='*', type=str, help='the directory to be ignored') args = parser.parse_args() diff --git a/scripts/pre-commit b/scripts/pre-commit index 88a3e7804..93e8a05cc 100755 --- a/scripts/pre-commit +++ b/scripts/pre-commit @@ -18,9 +18,7 @@ os.environ.pop('__PYVENV_LAUNCHER__', None) # start templated INSTALL_PYTHON = '/apsarapangu/disk2/yancheng.lgq/miniconda3/envs/tf_1_15/bin/python' -ARGS = [ - 'hook-impl', '--config=.pre-commit-config.yaml', '--hook-type=pre-commit' -] +ARGS = ['hook-impl', '--config=.pre-commit-config.yaml', '--hook-type=pre-commit'] # end templated ARGS.extend(('--hook-dir', os.path.realpath(os.path.dirname(__file__)))) ARGS.append('--') diff --git a/setup.cfg b/setup.cfg index aebb3437c..79d3afea1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,40 +1,43 @@ -[easy_install] -index_url = http://mirrors.aliyun.com/pypi/simple/ - -[bdist_wheel] -universal = 1 - [isort] -line_length = 79 -multi_line_output = 7 -# force_single_line = true 这与 multi_line_output 冲突 +profile = black +line_length = 120 +indent = 2 +src_paths = easy_rec known_standard_library = setuptools known_first_party = easy_rec known_third_party = absl,common_io,distutils,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,scipy,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,tensorflow_probability,yaml -no_lines_before = LOCALFOLDER -default_section = THIRDPARTY -skip = easy_rec/python/protos +skip = easy_rec/python/protos, venv [yapf] based_on_style = pep8 -column_limit = 79 -indent_width = 2 +COLUMN_LIMIT = 120 +INDENT_WIDTH = 2 CONTINUATION_INDENT_WIDTH = 2 -CONTINUATION_ALIGN_STYLE = SPACE -DEDENT_CLOSING_BRACKETS = true -ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT = false -SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN = false +ALLOW_SPLIT_BEFORE_DICT_VALUE = False +BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF = True +SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET = False +NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS = True [flake8] -select = B,C,D,E,F,P,T4,W,B9 +show-source = true +statistics = true max-line-length = 120 -ignore = - E111,E114,E121,E124,E125,E126,E129,E251,W291,W503,W504, - # docstring missing error should be used when all docstrings are completed +extend-select = + B,C,D,E,F,P,T4,W,B9, + BLK,T10 +extend-ignore = + E111,E114,E121,E124,E125,E126,E129,E203,E251,E721,F401, + W291,W503,W504, D100,D101,D102,D103,D104,D105,D106,D107 per-file-ignores = - __init__.py: F401 - easy_rec/python/utils/test_utils.py: E402 - easy_rec/python/utils/io_util.py: E402 -exclude = docs/src,scripts,easy_rec/python/protos,*.pyi,.git + __init__.py:F401, + easy_rec/python/utils/test_utils.py:E402, + easy_rec/python/utils/io_util.py:E402, docstring-convention = google +enable-extensions = G + +[docformatter] +wrap_summaries = 120 +wrap_descriptions = 120 +blank = True +pre_summary_newline = False diff --git a/setup.py b/setup.py index b9f841c00..ede80ef6f 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ import codecs import os + from setuptools import find_packages, setup @@ -60,12 +61,12 @@ def parse_require_file(fpath): include_package_data=True, classifiers=[ 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3' + 'Programming Language :: Python :: 3', ], tests_require=parse_requirements('requirements/tests.txt'), install_requires=parse_requirements('requirements/runtime.txt'), extras_require={ 'all': parse_requirements('requirements.txt'), 'tests': parse_requirements('requirements/tests.txt'), - } + }, ) From 864e4aa59658f2d13f7ca6dd116755cc1aa0756e Mon Sep 17 00:00:00 2001 From: yangxudong Date: Thu, 13 Nov 2025 15:19:11 +0800 Subject: [PATCH 50/51] upgrade zero inflated lognormal loss, support export structure path, upgrade pre-commit hooks --- .github/workflows/code_style.yml | 2 +- .pre-commit-config.yaml | 41 +- README.md | 2 +- docs/post_fix.py | 4 +- docs/source/_ext/post_process.py | 10 +- docs/source/automl/finetune_config.md | 2 +- docs/source/automl/hpo_config.md | 136 +- docs/source/automl/hpo_emr.md | 8 +- docs/source/automl/hpo_res.md | 2 +- docs/source/automl/pai_nni_hpo.md | 6 +- docs/source/benchmark.md | 10 +- docs/source/component/backbone.md | 6 +- docs/source/component/component.md | 454 ++-- docs/source/conf.py | 75 +- docs/source/eval.md | 18 +- docs/source/export.md | 4 +- docs/source/faq.md | 2 +- docs/source/feature/fg.md | 36 +- docs/source/feature/fg_docs/ComboFeature.md | 10 +- docs/source/feature/fg_docs/IdFeature.md | 22 +- docs/source/feature/fg_docs/OverLapFeature.md | 34 +- docs/source/feature/fg_docs/RawFeature.md | 20 +- docs/source/feature/odl_sample.md | 2 +- docs/source/feature/rtp_fg.md | 16 +- docs/source/feature/rtp_native.md | 8 +- docs/source/intro.md | 2 +- docs/source/models/cmbf.md | 6 +- docs/source/models/dbmtl.md | 6 +- docs/source/models/dlrm.md | 2 +- docs/source/models/dssm.md | 2 +- docs/source/models/dssm_neg_sampler.md | 2 +- docs/source/models/loss.md | 44 +- docs/source/models/mind.md | 2 +- docs/source/models/multi_tower.md | 6 +- docs/source/models/uniter.md | 4 +- docs/source/online_train.md | 2 +- ...73\347\272\277\351\242\204\346\265\213.md" | 2 +- ...73\347\272\277\351\242\204\346\265\213.md" | 8 +- docs/source/quick_start/designer_tutorial.md | 78 +- docs/source/quick_start/local_tutorial.md | 22 +- docs/source/tf_on_yarn.md | 4 +- docs/source/vector_retrieve.md | 24 +- easy_rec/__init__.py | 12 +- .../python/builders/hyperparams_builder.py | 21 +- easy_rec/python/builders/loss_builder.py | 256 +- easy_rec/python/builders/optimizer_builder.py | 126 +- easy_rec/python/builders/strategy_builder.py | 16 +- easy_rec/python/compat/adam_s.py | 114 +- easy_rec/python/compat/array_ops.py | 31 +- easy_rec/python/compat/dynamic_variable.py | 179 +- easy_rec/python/compat/early_stopping.py | 291 +-- easy_rec/python/compat/embedding_ops.py | 100 +- .../python/compat/embedding_parallel_saver.py | 214 +- easy_rec/python/compat/estimator_train.py | 54 +- easy_rec/python/compat/exporter.py | 142 +- .../compat/feature_column/feature_column.py | 1653 +++++++------ .../feature_column/feature_column_v2.py | 2174 ++++++++++------- .../feature_column/sequence_feature_column.py | 252 +- .../python/compat/feature_column/utils.py | 41 +- easy_rec/python/compat/layers.py | 184 +- easy_rec/python/compat/optimizers.py | 289 ++- easy_rec/python/compat/queues.py | 117 +- easy_rec/python/compat/regularizers.py | 57 +- easy_rec/python/compat/sok_optimizer.py | 137 +- .../python/compat/sync_replicas_optimizer.py | 150 +- .../python/compat/weight_decay_optimizers.py | 166 +- .../distribute_metrics_impl_pai.py | 1753 ++++++------- .../distribute_metrics_impl_tf.py | 1833 +++++++------- easy_rec/python/core/learning_schedules.py | 129 +- easy_rec/python/core/metrics.py | 158 +- easy_rec/python/core/sampler.py | 525 ++-- easy_rec/python/eval.py | 65 +- easy_rec/python/export.py | 67 +- .../python/feature_column/feature_column.py | 379 +-- .../python/feature_column/feature_group.py | 4 +- easy_rec/python/hpo/emr_hpo.py | 151 +- easy_rec/python/hpo/generate_hpo_sql.py | 48 +- easy_rec/python/hpo/pai_hpo.py | 259 +- .../python/inference/client/client_demo.py | 36 +- .../inference/client/easyrec_request.py | 3 +- easy_rec/python/inference/csv_predictor.py | 77 +- .../inference/hive_parquet_predictor.py | 83 +- easy_rec/python/inference/hive_predictor.py | 80 +- easy_rec/python/inference/odps_predictor.py | 43 +- .../python/inference/parquet_predictor.py | 72 +- .../python/inference/parquet_predictor_v2.py | 72 +- easy_rec/python/inference/predictor.py | 189 +- easy_rec/python/inference/processor/test.py | 76 +- easy_rec/python/inference/vector_retrieve.py | 62 +- easy_rec/python/input/batch_tfrecord_input.py | 95 +- easy_rec/python/input/criteo_binary_reader.py | 119 +- easy_rec/python/input/criteo_input.py | 91 +- easy_rec/python/input/csv_input.py | 125 +- easy_rec/python/input/csv_input_ex.py | 67 +- easy_rec/python/input/csv_input_v2.py | 61 +- easy_rec/python/input/datahub_input.py | 159 +- easy_rec/python/input/dummy_input.py | 34 +- easy_rec/python/input/hive_input.py | 84 +- easy_rec/python/input/hive_parquet_input.py | 74 +- easy_rec/python/input/hive_rtp_input.py | 124 +- easy_rec/python/input/input.py | 679 +++-- easy_rec/python/input/kafka_dataset.py | 69 +- easy_rec/python/input/kafka_input.py | 157 +- easy_rec/python/input/load_parquet.py | 154 +- easy_rec/python/input/odps_input.py | 78 +- easy_rec/python/input/odps_input_v2.py | 88 +- easy_rec/python/input/odps_input_v3.py | 74 +- easy_rec/python/input/odps_rtp_input.py | 167 +- easy_rec/python/input/odps_rtp_input_v2.py | 36 +- easy_rec/python/input/parquet_input.py | 174 +- easy_rec/python/input/parquet_input_v2.py | 66 +- easy_rec/python/input/parquet_input_v3.py | 103 +- easy_rec/python/input/rtp_input.py | 173 +- easy_rec/python/input/rtp_input_v2.py | 94 +- easy_rec/python/input/tfrecord_input.py | 66 +- easy_rec/python/layers/backbone.py | 55 +- easy_rec/python/layers/capsule_layer.py | 85 +- easy_rec/python/layers/cmbf.py | 230 +- easy_rec/python/layers/common_layers.py | 92 +- easy_rec/python/layers/dnn.py | 51 +- easy_rec/python/layers/embed_input_layer.py | 8 +- easy_rec/python/layers/fm.py | 1 + easy_rec/python/layers/input_layer.py | 211 +- easy_rec/python/layers/keras/__init__.py | 42 +- easy_rec/python/layers/keras/activation.py | 23 +- easy_rec/python/layers/keras/attention.py | 80 +- .../python/layers/keras/auxiliary_loss.py | 6 +- easy_rec/python/layers/keras/blocks.py | 142 +- easy_rec/python/layers/keras/bst.py | 96 +- easy_rec/python/layers/keras/custom_ops.py | 119 +- easy_rec/python/layers/keras/data_augment.py | 42 +- easy_rec/python/layers/keras/din.py | 16 +- easy_rec/python/layers/keras/einsum_dense.py | 336 +-- easy_rec/python/layers/keras/embedding.py | 18 +- easy_rec/python/layers/keras/fibinet.py | 61 +- easy_rec/python/layers/keras/interaction.py | 156 +- easy_rec/python/layers/keras/layer_norm.py | 117 +- easy_rec/python/layers/keras/mask_net.py | 41 +- .../layers/keras/multi_head_attention.py | 276 ++- easy_rec/python/layers/keras/multi_task.py | 20 +- .../layers/keras/numerical_embedding.py | 89 +- easy_rec/python/layers/keras/ppnet.py | 137 +- easy_rec/python/layers/keras/transformer.py | 38 +- easy_rec/python/layers/layer_norm.py | 16 +- easy_rec/python/layers/mmoe.py | 36 +- easy_rec/python/layers/multihead_attention.py | 56 +- .../layers/multihead_cross_attention.py | 421 ++-- easy_rec/python/layers/senet.py | 29 +- easy_rec/python/layers/seq_input_layer.py | 80 +- .../python/layers/sequence_feature_layer.py | 212 +- easy_rec/python/layers/uniter.py | 173 +- easy_rec/python/layers/utils.py | 44 +- .../layers/variational_dropout_layer.py | 46 +- easy_rec/python/loss/circle_loss.py | 28 +- easy_rec/python/loss/contrastive_loss.py | 10 +- easy_rec/python/loss/f1_reweight_loss.py | 12 +- easy_rec/python/loss/focal_loss.py | 35 +- easy_rec/python/loss/jrc_loss.py | 46 +- easy_rec/python/loss/listwise_loss.py | 84 +- easy_rec/python/loss/multi_similarity.py | 31 +- easy_rec/python/loss/pairwise_loss.py | 169 +- .../loss/softmax_loss_with_negative_mining.py | 51 +- .../python/loss/zero_inflated_lognormal.py | 41 +- easy_rec/python/main.py | 456 ++-- easy_rec/python/model/autoint.py | 36 +- easy_rec/python/model/cmbf.py | 34 +- .../model/collaborative_metric_learning.py | 107 +- easy_rec/python/model/dat.py | 95 +- easy_rec/python/model/dbmtl.py | 85 +- easy_rec/python/model/dcn.py | 31 +- easy_rec/python/model/deepfm.py | 83 +- easy_rec/python/model/dlrm.py | 40 +- easy_rec/python/model/dropoutnet.py | 145 +- easy_rec/python/model/dssm.py | 112 +- easy_rec/python/model/dssm_senet.py | 98 +- easy_rec/python/model/dummy_model.py | 17 +- easy_rec/python/model/easy_rec_estimator.py | 451 ++-- easy_rec/python/model/easy_rec_model.py | 176 +- easy_rec/python/model/esmm.py | 150 +- easy_rec/python/model/fm.py | 40 +- easy_rec/python/model/match_model.py | 202 +- easy_rec/python/model/mind.py | 311 ++- easy_rec/python/model/mmoe.py | 51 +- easy_rec/python/model/multi_task_model.py | 186 +- easy_rec/python/model/multi_tower.py | 29 +- easy_rec/python/model/multi_tower_bst.py | 129 +- easy_rec/python/model/multi_tower_din.py | 71 +- easy_rec/python/model/multi_tower_recall.py | 46 +- easy_rec/python/model/pdn.py | 141 +- easy_rec/python/model/ple.py | 95 +- easy_rec/python/model/rank_model.py | 408 ++-- easy_rec/python/model/rocket_launching.py | 199 +- easy_rec/python/model/simple_multi_task.py | 31 +- easy_rec/python/model/uniter.py | 30 +- easy_rec/python/model/wide_and_deep.py | 68 +- easy_rec/python/ops/gen_kafka_ops.py | 128 +- easy_rec/python/ops/gen_str_avx_op.py | 4 +- easy_rec/python/ops/incr_record.py | 6 +- easy_rec/python/predict.py | 129 +- easy_rec/python/test/csv_input_test.py | 47 +- .../python/test/custom_early_stop_func.py | 5 +- easy_rec/python/test/dh_local_run.py | 52 +- easy_rec/python/test/embed_test.py | 13 +- easy_rec/python/test/emr_run.py | 67 +- easy_rec/python/test/eval_metric_test.py | 64 +- easy_rec/python/test/excel_convert_test.py | 29 +- easy_rec/python/test/export_test.py | 347 +-- easy_rec/python/test/fg_test.py | 29 +- easy_rec/python/test/hive_input_test.py | 41 +- easy_rec/python/test/hpo_test.py | 30 +- easy_rec/python/test/kafka_test.py | 144 +- easy_rec/python/test/local_incr_test.py | 85 +- easy_rec/python/test/loss_test.py | 96 +- easy_rec/python/test/odps_command.py | 26 +- easy_rec/python/test/odps_local_run.py | 37 +- easy_rec/python/test/odps_run.py | 166 +- easy_rec/python/test/odps_test_prepare.py | 39 +- easy_rec/python/test/odps_test_util.py | 54 +- easy_rec/python/test/pre_check_test.py | 18 +- easy_rec/python/test/predictor_test.py | 320 +-- easy_rec/python/test/rtp_convert_test.py | 66 +- easy_rec/python/test/run.py | 26 +- easy_rec/python/test/train_eval_test.py | 820 ++++--- easy_rec/python/test/util_test.py | 35 +- .../test/zero_inflated_lognormal_test.py | 15 +- .../python/tools/add_boundaries_to_config.py | 21 +- .../tools/add_feature_info_to_config.py | 47 +- .../python/tools/convert_config_format.py | 13 +- easy_rec/python/tools/convert_rtp_data.py | 7 +- easy_rec/python/tools/convert_rtp_fg.py | 122 +- .../python/tools/create_config_from_excel.py | 176 +- easy_rec/python/tools/criteo/convert_data.py | 54 +- easy_rec/python/tools/edit_lookup_graph.py | 55 +- easy_rec/python/tools/faiss_index_pai.py | 33 +- easy_rec/python/tools/feature_selection.py | 149 +- easy_rec/python/tools/hit_rate_ds.py | 107 +- easy_rec/python/tools/hit_rate_pai.py | 96 +- easy_rec/python/tools/pre_check.py | 57 +- easy_rec/python/tools/predict_and_chk.py | 67 +- easy_rec/python/tools/read_kafka.py | 19 +- easy_rec/python/tools/split_model_pai.py | 114 +- easy_rec/python/tools/split_pdn_model_pai.py | 114 +- easy_rec/python/tools/test_saved_model.py | 32 +- easy_rec/python/tools/view_saved_model.py | 10 +- easy_rec/python/tools/write_kafka.py | 30 +- easy_rec/python/train_eval.py | 168 +- easy_rec/python/utils/activation.py | 23 +- easy_rec/python/utils/check_utils.py | 56 +- easy_rec/python/utils/config_util.py | 145 +- easy_rec/python/utils/constant.py | 3 +- easy_rec/python/utils/convert_rtp_fg.py | 174 +- easy_rec/python/utils/dag.py | 13 +- easy_rec/python/utils/distribution_utils.py | 166 +- easy_rec/python/utils/ds_util.py | 6 +- easy_rec/python/utils/embedding_utils.py | 3 +- easy_rec/python/utils/estimator_utils.py | 315 ++- easy_rec/python/utils/export_big_model.py | 431 ++-- easy_rec/python/utils/expr_util.py | 8 +- easy_rec/python/utils/fg_util.py | 7 +- easy_rec/python/utils/hit_rate_utils.py | 101 +- easy_rec/python/utils/hive_utils.py | 40 +- easy_rec/python/utils/hpo_util.py | 7 +- easy_rec/python/utils/hvd_utils.py | 3 +- easy_rec/python/utils/input_utils.py | 37 +- easy_rec/python/utils/io_util.py | 74 +- easy_rec/python/utils/load_class.py | 35 +- easy_rec/python/utils/meta_graph_editor.py | 397 +-- easy_rec/python/utils/multi_optimizer.py | 4 +- easy_rec/python/utils/odps_util.py | 25 +- easy_rec/python/utils/pai_util.py | 8 +- easy_rec/python/utils/proto_util.py | 12 +- easy_rec/python/utils/restore_filter.py | 5 +- easy_rec/python/utils/shape_utils.py | 93 +- easy_rec/python/utils/test_utils.py | 290 ++- easy_rec/python/utils/tf_utils.py | 24 +- .../data/amazon_books_data/process_amazon.py | 36 +- examples/data/criteo/process_criteo_kaggle.py | 9 +- examples/data/movielens_1m/process_ml_1m.py | 52 +- examples/match_model/dssm.md | 2 +- examples/match_model/dssm_negative_sample.md | 2 +- git-lfs/git_lfs.py | 65 +- pai_jobs/deploy_ext.sh | 2 +- pai_jobs/run.py | 419 ++-- pyproject.toml | 52 - scripts/ci_test_change_files.py | 9 +- scripts/pre-commit | 4 +- setup.cfg | 53 +- setup.py | 42 +- 288 files changed, 18212 insertions(+), 15127 deletions(-) delete mode 100644 pyproject.toml diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 97f7291d8..0b9aeb3d3 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -23,7 +23,7 @@ jobs: PULL_REQUEST_NUM: ${{ github.event.pull_request.number }} run: | source ~/.bashrc - conda activate py310 + conda activate tf25_py3 pre-commit run -a if [ $? -eq 0 ] then diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6aecaa0c..a2132ed9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,53 +1,44 @@ repos: - repo: https://github.com/pycqa/flake8.git - rev: 7.3.0 + rev: 5.0.0 hooks: - id: flake8 additional_dependencies: [ - 'flake8-docstrings>=1.7.0' + 'flake8-docstrings==1.5.0' ] - - repo: https://github.com/pycqa/isort - rev: 7.0.0 + - repo: https://github.com/asottile/seed-isort-config + rev: v2.2.0 + hooks: + - id: seed-isort-config + - repo: https://github.com/timothycrosley/isort + rev: 4.3.21 hooks: - id: isort - args: ["--profile", "black", "--atomic"] - - - repo: https://github.com/google/yapf - rev: v0.43.0 + - repo: https://github.com/pre-commit/mirrors-yapf + rev: v0.30.0 hooks: - id: yapf - args: [--style=setup.cfg] - - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.8 - hooks: - - id: ruff - args: [--fix, --select=E231,E501,W503, --preview] - types: [python] - - id: ruff-format - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 + rev: v3.1.0 hooks: - id: trailing-whitespace args: ["--no-markdown-linebreak-ext"] - id: check-yaml - id: end-of-file-fixer - id: requirements-txt-fixer - #- id: double-quote-string-fixer + - id: double-quote-string-fixer - id: check-merge-conflict - id: mixed-line-ending args: ["--fix=lf"] - - repo: https://github.com/PyCQA/docformatter - rev: v1.7.7 + - repo: https://github.com/myint/docformatter + rev: v1.3.1 hooks: - id: docformatter args: ["--in-place", "--wrap-descriptions", "0", "--wrap-summaries", "0"] - repo: https://github.com/executablebooks/mdformat - rev: 0.7.22 + rev: 0.7.1 hooks: - id: mdformat additional_dependencies: [ - 'mdformat-tables', - 'mdformat-black' + 'mdformat-tables==0.4.0' ] diff --git a/README.md b/README.md index df14a6bee..2b75da77a 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Running Platform: - Flexible feature config and simple model config - [Build models by combining some components](docs/source/component/backbone.md) -- Efficient and robust feature generation[used in taobao] +- Efficient and robust feature generation\[used in taobao\] - Nice web interface in development ### It is smart diff --git a/docs/post_fix.py b/docs/post_fix.py index 726a32181..ce279bed8 100644 --- a/docs/post_fix.py +++ b/docs/post_fix.py @@ -9,5 +9,7 @@ with open(sys.argv[1], 'w') as fout: for line_str in lines: if '_static/searchtools.js' in line_str: - fout.write(' \n') + fout.write( + ' \n' + ) fout.write(line_str) diff --git a/docs/source/_ext/post_process.py b/docs/source/_ext/post_process.py index 85cb033c4..8f46285e6 100644 --- a/docs/source/_ext/post_process.py +++ b/docs/source/_ext/post_process.py @@ -11,12 +11,14 @@ def __init__(self, document, startnode=None): super(PostFixLink, self).__init__(document, startnode) def apply(self, **kwargs): + def _visit(node): if not node.children: return for child in node.children: if isinstance(child, nodes.Element): - if 'refuri' in child.attributes and '.md#' in child.attributes['refuri']: + if 'refuri' in child.attributes and '.md#' in child.attributes[ + 'refuri']: src = child.attributes['refuri'] dst = src.replace('.md#', '.html#') logging.info('[PostFixLink] replace %s to %s' % (src, dst)) @@ -30,7 +32,7 @@ def setup(app): app.add_post_transform(PostFixLink) return { - 'version': '0.1', - 'parallel_read_safe': True, - 'parallel_write_safe': True, + 'version': '0.1', + 'parallel_read_safe': True, + 'parallel_write_safe': True, } diff --git a/docs/source/automl/finetune_config.md b/docs/source/automl/finetune_config.md index b07eac9cd..8b7131059 100644 --- a/docs/source/automl/finetune_config.md +++ b/docs/source/automl/finetune_config.md @@ -109,7 +109,7 @@ metric_dict={'auc_is_like':0.25, 'auc_is_valid_play':0.5, 'auc_is_comment':0.25} 与begin训练的`差异点`: - 每个配置模块支持jinja模版渲染 -- 配置finetune日期{% set date_list = [20220616,20220617] %} +- 配置finetune日期{% set date_list = \[20220616,20220617\] %} - 配置finetune开始日期{% set date_begin = 20220616 %}，Dfine_tune_checkpoint开始日期和后续日期采取的model路径不一样 - 假设每天finetune： - {bizdate} 必须保留，将会在代码中根据当天日期进行替换 diff --git a/docs/source/automl/hpo_config.md b/docs/source/automl/hpo_config.md index a8c47e97a..33bbb725d 100644 --- a/docs/source/automl/hpo_config.md +++ b/docs/source/automl/hpo_config.md @@ -24,13 +24,13 @@ assessor: 支持将该组中的实验结果和同组中的所有历史进行比较，如果不满足比较标准（例如小于中位数），则停止该组超参数的运行。比如说设置最大运行次数max_trial_num， 实际使用量会显著小于max_trial_num，但具体数量就和实际跑的任务及随机到的超参有关系了。例如max_trial_num=50时，可能最终可能不到 25 次，并且差不多已经是完整探索了50组超参。 ``` -| PAIAssessor | 描述 | 值 | -| ------------- | ------------------------------------------------ | ----------------- | -| optimize_mode | 最大化优化的方向 | maximize/minimize | -| start_step | 从第几步开始进行早停判定 | 2 | -| moving_avg | 早停判断时，采用所有历史的滑动平均值作为判断标准 | True | +| PAIAssessor | 描述 | 值 | +| ------------- | ----------------------------- | ----------------- | +| optimize_mode | 最大化优化的方向 | maximize/minimize | +| start_step | 从第几步开始进行早停判定 | 2 | +| moving_avg | 早停判断时，采用所有历史的滑动平均值作为判断标准 | True | | proportion | 本次超参搜索的最优值和历史记录的proportion值比较 | 0.5 | -| patience | metric指标连续下降几次，就停止 | 10 | +| patience | metric指标连续下降几次，就停止 | 10 | ### 示例 @@ -98,37 +98,37 @@ metric_source_{{bizdate}}=oss://automl-nni/easyrec/finetune/{{bizdate}}_finetune ### 字段介绍 -| 配置模块 | 描述 | 是否可选 | -| --------------- | ------------------------------------------------------------------------------------------------- | -------- | -| platform_config | 用于标记任务执行的平台以及对应的执行命令 | 必选 | -| metric_config | 用于标记任务metric的获取来源、metric的key以及对应权重、metric类型、最终metric的方式 | 必选 | -| output_config | 如果使用服务版，可以配置output_config用来获取最优模型配置summary_path，用于配制tensorboard路径 | 可选 | -| schedule_config | 如果任务在指定时间内调度任务，则需要配置schedule_config,修改对应的schedule_config的值 | 可选 | -| params_config | 如果用户的参数是保存在文件中，则需要配置params_config, 用于标记需要修改参数的源文件路径和目标路径 | 可选 | -| oss_config | 如果任务需要使用OSS存储，则需要配置OSS config | 可选 | -| odps_config | 如果任务需要使用maxcompute平台执行任务，则需要配置odps config | 可选 | -| ts_config | 如果任务需要使用trainingservice平台执行任务，则需要配置ts config | 可选 | -| paiflow_config | 如果任务需要执行工作流任务，则需要配置paiflow_config,修改对应的paiflow_config的值 | 可选 | -| dlc_config | 如果任务需要执行dlc任务，则需要配置dlc_config,修改对应的dlc_config的值 | 可选 | -| monitor_config | 支持失败告警,最优metric更新时提醒 | 可选 | +| 配置模块 | 描述 | 是否可选 | +| --------------- | ----------------------------------------------------------------- | ---- | +| platform_config | 用于标记任务执行的平台以及对应的执行命令 | 必选 | +| metric_config | 用于标记任务metric的获取来源、metric的key以及对应权重、metric类型、最终metric的方式 | 必选 | +| output_config | 如果使用服务版，可以配置output_config用来获取最优模型配置summary_path，用于配制tensorboard路径 | 可选 | +| schedule_config | 如果任务在指定时间内调度任务，则需要配置schedule_config,修改对应的schedule_config的值 | 可选 | +| params_config | 如果用户的参数是保存在文件中，则需要配置params_config, 用于标记需要修改参数的源文件路径和目标路径 | 可选 | +| oss_config | 如果任务需要使用OSS存储，则需要配置OSS config | 可选 | +| odps_config | 如果任务需要使用maxcompute平台执行任务，则需要配置odps config | 可选 | +| ts_config | 如果任务需要使用trainingservice平台执行任务，则需要配置ts config | 可选 | +| paiflow_config | 如果任务需要执行工作流任务，则需要配置paiflow_config,修改对应的paiflow_config的值 | 可选 | +| dlc_config | 如果任务需要执行dlc任务，则需要配置dlc_config,修改对应的dlc_config的值 | 可选 | +| monitor_config | 支持失败告警,最优metric更新时提醒 | 可选 | ## platform_config -| platform_config | 描述 | 值 | -| --------------- | ----------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | -| name | 用于标记任务执行的平台 | DLC/MaxCompute/DataScience/LOCAL/PAI/PAIFLOW | -| cmdxx | 用于标记执行的命令，以cmd开头 | dlc submit pytorch --name=test_nni\_${exp_id}\_${trial_id} xxx | +| platform_config | 描述 | 值 | +| --------------- | ------------------------------------------------------------ | -------------------------------------------------------------- | +| name | 用于标记任务执行的平台 | DLC/MaxCompute/DataScience/LOCAL/PAI/PAIFLOW | +| cmdxx | 用于标记执行的命令，以cmd开头 | dlc submit pytorch --name=test_nni\_${exp_id}\_${trial_id} xxx | | resume | 1表示开启续跑模式；用于用户一次运行时，比如说第一行任务成功，第二行由于资源不足失败，可以开启续跑，从第二行命令开始运行 | 0/1 | ## metric_config -| metric_config | 描述 | 值 | -| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| metric_type | metric类型 | summary/table/api/json/stdout | -| metric_source | metric来源（可以为多个以metric_source开头的，具体可以看maxcompute_crossvalidation案例） | 对应为具体的路径或者job | -| final_mode | 如果任务运行过程中，存在很多中间metric，那么需要确定最终metric的计算方式 | final/best/avg | +| metric_config | 描述 | 值 | +| ---------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| metric_type | metric类型 | summary/table/api/json/stdout | +| metric_source | metric来源（可以为多个以metric_source开头的，具体可以看maxcompute_crossvalidation案例） | 对应为具体的路径或者job | +| final_mode | 如果任务运行过程中，存在很多中间metric，那么需要确定最终metric的计算方式 | final/best/avg | | source_list_final_mode | 可选，默认值为final_mode，可选值为final/best/avg,用于有多个metric_source时最终metric如何计算，具体可以看maxcompute_crossvalidation案例 | final/best/avg | -| metric_dict | 对应查询的key以及对应的权重;可以为负值 | metric_dict={'auc_is_like':0.25, 'auc_is_valid_play':0.5, 'auc_is_comment':0.25, 'loss_play_time':-0.25} metric=val(’auc_is_valid_play’)\*0.5+val(’auc_is_like’)\*0.25+val(’auc_is_comment’)\*0.25-val(’loss_play_time’)\*0.25 | +| metric_dict | 对应查询的key以及对应的权重;可以为负值 | metric_dict={'auc_is_like':0.25, 'auc_is_valid_play':0.5, 'auc_is_comment':0.25, 'loss_play_time':-0.25} metric=val(’auc_is_valid_play’)\*0.5+val(’auc_is_like’)\*0.25+val(’auc_is_comment’)\*0.25-val(’loss_play_time’)\*0.25 | - 如果metric_type=stdout类型，则metric_dict对应的key为正则表达式，value为对应的权重 @@ -191,31 +191,31 @@ metric_source=cmd1 ## output_config -| output_config | 描述 | 值 | -| ------------- | ------------------------------------------------------ | ---- | -| model_path | 如果使用服务版，可以配置model_path用来获取最优模型 | 路径 | -| summary_path | 如果使用单机版，可以配置summary用于本地查看TensorBoard | 路径 | +| output_config | 描述 | 值 | +| ------------- | ------------------------------------ | --- | +| model_path | 如果使用服务版，可以配置model_path用来获取最优模型 | 路径 | +| summary_path | 如果使用单机版，可以配置summary用于本地查看TensorBoard | 路径 | ## schedule_config -| schedule_config | 描述 | 值 | -| --------------- | ---------------------------------- | ---------------- | +| schedule_config | 描述 | 值 | +| --------------- | -------------------- | ---------------- | | day | 支持在指定时间范围内调度AutoML任务 | everyday/weekend | -| start_time | 指定调度开始时间 | 00:00-23:59 | -| end_time | 指定调度结束时间 | 00:00-23:59 | +| start_time | 指定调度开始时间 | 00:00-23:59 | +| end_time | 指定调度结束时间 | 00:00-23:59 | ## params_config 如果用户的参数是保存在文件中，则需要配置params_config, -| params_config | 描述 | 值 | -| -------------------------- | ------------------------------------------------------------------------------------ | ----------------- | +| params_config | 描述 | 值 | +| -------------------------- | ------------------------------------------------------ | ----------------- | | params_src_dst_filepath1xx | 用于标记需要修改参数的源文件路径和目标路径,可以为多个，以params_src_dst_filepath开头 | src_path,dst_path | -| params_src_dst_filepath2xx | xx | xx | +| params_src_dst_filepath2xx | xx | xx | ## oss_config -| oss_config | 描述 | 值 | +| oss_config | 描述 | 值 | | --------------- | -------- | -------------------------------------------------------------------------- | | endpoint | endpoint | [http://oss-cn-shanghai.aliyuncs.com](http://oss-cn-shanghai.aliyuncs.com) | | accessKeyID | ak | ak | @@ -224,28 +224,28 @@ metric_source=cmd1 ## odps_config -| odps_config | 描述 | 值 | -| ------------- | ------------ | ------------------------------------------------------------------------------------------ | -| access_id | ak | ak | -| access_key | sk | ak | -| project_name | project_name | xxx | +| odps_config | 描述 | 值 | +| ------------- | ------------ | ------------------------------------------------------------------------------------- | +| access_id | ak | ak | +| access_key | sk | ak | +| project_name | project_name | xxx | | end_point | end_point | 弹外: http://service.odps.aliyun.com/api 弹内：http://service-corp.odps.aliyun-inc.com/api | -| log_view_host | logview host | 弹外：http://logview.odps.aliyun.com 弹内：http://logview.alibaba-inc.com | -| role_arn | role_arn | acs:ram::xxx:role/aliyunserviceroleforpaiautoml | +| log_view_host | logview host | 弹外：http://logview.odps.aliyun.com 弹内：http://logview.alibaba-inc.com | +| role_arn | role_arn | acs:ram::xxx:role/aliyunserviceroleforpaiautoml | ## dlc_config -| dlc_config | 描述 | 值 | -| ---------- | ----------- | ----------------------------------------------------------------------- | -| access_id | ak | ak | -| access_key | sk | ak | +| dlc_config | 描述 | 值 | +| ---------- | ----------- | ----------------------------------------------------------------- | +| access_id | ak | ak | +| access_key | sk | ak | | end_point | end_point | 弹外：pai-dlc.cn-shanghai.aliyuncs.com 弹内：pai-dlc-share.aliyuncs.com | -| region | cn-shanghai | cn-shanghai | -| protocol | protocol | http/https | +| region | cn-shanghai | cn-shanghai | +| protocol | protocol | http/https | ## ts_config -| ts_config | 描述 | 值 | +| ts_config | 描述 | 值 | | ----------------- | --------- | ---------------------------- | | access_key_id | ak | ak | | access_key_secret | sk | ak | @@ -254,7 +254,7 @@ metric_source=cmd1 ## paiflow_config -| paiflow_config | 描述 | 值 | +| paiflow_config | 描述 | 值 | | ----------------- | ------------ | ------- | | access_key_id | ak | ak | | access_key_secret | sk | ak | @@ -267,24 +267,24 @@ metric_source=cmd1 - 点击阿里钉头像->机器人管理-自定义机器人->群组选择工作通知 - 点击阿里钉头像->机器人管理-自定义机器人->群组：选择对应的群号 -| monitor_config | 描述 | 值 | -| -------------- | ------------------------------------------------------------------------------- | ----------------------------------------------------- | -| url | url为创建自定义机器人对应的Webhook地址 | https://oapi.dingtalk.com/robot/send?access_token=xxx | -| keyword | 添加自定义机器人：自定义关键词 | monitor | -| at_mobiles | 在content里添加@人的手机号，且只有在群内的成员才可被@，非群内成员手机号会被脱敏 | ['11xx'] | -| at_user_ids | 被@人的用户userid。即工号 | [] | -| is_at_all | 是否@所有人 | True/False | +| monitor_config | 描述 | 值 | +| -------------- | -------------------------------------------- | ----------------------------------------------------- | +| url | url为创建自定义机器人对应的Webhook地址 | https://oapi.dingtalk.com/robot/send?access_token=xxx | +| keyword | 添加自定义机器人：自定义关键词 | monitor | +| at_mobiles | 在content里添加@人的手机号，且只有在群内的成员才可被@，非群内成员手机号会被脱敏 | \['11xx'\] | +| at_user_ids | 被@人的用户userid。即工号 | \[\] | +| is_at_all | 是否@所有人 | True/False | ## search_space.json -| search_space | 描述 | 值 | -| ------------ | ---------------------------------------------------------------------------------------------------------------------------- | --- | -| key | trial.ini中配置的搜索参数变量 | | +| search_space | 描述 | 值 | +| ------------ | --------------------------------------------------------------------------------------------------------- | --- | +| key | trial.ini中配置的搜索参数变量 | | | type | nni中定义的搜索类型，相关配置参考[NNI searchSpace参考手册](https://nni.readthedocs.io/en/v2.2/Tutorial/SearchSpaceSpec.html) | | - {”\_type”: “choice”, “\_value”: options}：从options中选取一个。 -- {”\_type”: “randint”, “\_value”: [lower, upper]}：\[low,upper)之间选择一个随机整数。 -- {”\_type”: “uniform”, “\_value”: [low, high]}：[low,upper]之间随机采样。 +- {”\_type”: “randint”, “\_value”: \[lower, upper\]}：\[low,upper)之间选择一个随机整数。 +- {”\_type”: “uniform”, “\_value”: \[low, high\]}：\[low,upper\]之间随机采样。 | | value | value是根据业务、经验设置相关搜索值 | | diff --git a/docs/source/automl/hpo_emr.md b/docs/source/automl/hpo_emr.md index 01fb4a00b..3d7172f5a 100644 --- a/docs/source/automl/hpo_emr.md +++ b/docs/source/automl/hpo_emr.md @@ -23,7 +23,7 @@ python -m easy_rec.python.hpo.emr_hpo --hyperparams hyperparams.json --config_p - --metric_name  调优的指标，默认是auc，其它可选指标\[参考../eval.md) - --max_parallel   同一时刻可以并行跑的实验数目，默认4 - --total_trial_num  总共跑多少组实验，默认6 -- --el_submit_params el_submit指定PS/Worker资源的一些参数，包括-t x -m x [-pn x -pc x -pm x] -wn x -wc x -wm x -wg x 默认值 +- --el_submit_params el_submit指定PS/Worker资源的一些参数，包括-t x -m x \[-pn x -pc x -pm x\] -wn x -wc x -wm x -wg x 默认值 ```bash -t standalone -m local -wn 1 -wc 6 -wm 20000 -wg 1 @@ -45,7 +45,7 @@ python -m easy_rec.python.hpo.emr_hpo --hyperparams hyperparams.json --config_p - name: easy_rec pipeline_config里面的参数名称，注意要用全路径 - feature_config.features\[**input_names[0]=field_name1**\].embedding_dim + feature_config.features\[**input_names\[0\]=field_name1**\].embedding_dim - 由于feature_config.features是一个数组，所以需要用到选择器，根据**属性值**选择部分特征: @@ -68,7 +68,7 @@ python -m easy_rec.python.hpo.emr_hpo --hyperparams hyperparams.json --config_p 有些参数的值是关联的，比如对于deepfm算法，所有的embedding_dim必须是一样的 -- name里面可以指定多个要调整的参数名称，用";"分割feature_config.features\[input_names[0]=field1\].embedding_dim;feature_config.features\[input_names[0]=field20\].embedding_dim +- name里面可以指定多个要调整的参数名称，用";"分割feature_config.features\[input_names\[0\]=field1\].embedding_dim;feature_config.features\[input_names\[0\]=field20\].embedding_dim - 如果name里面包含了多个参数名称，那么candidates也需要有多个参数值，用";"分割如"32;32" - candidates: 候选值 - type: 候选值类型, 支持Categorical, Integer, Real @@ -107,6 +107,6 @@ python -m easy_rec.python.hpo.emr_hpo --hyperparams hyperparams.json --config_p - 如果设置了--debug，那么将会保留本地临时目录: /tmp/emr_easy_rec_hpo_1600519258 - rewrite\_[0-4].json定义了每组实验的参数 + rewrite\_\[0-4\].json定义了每组实验的参数 ![image.png](../../images/automl/emr_json.png) ![image.png](../../images/automl/emr_json2.png) diff --git a/docs/source/automl/hpo_res.md b/docs/source/automl/hpo_res.md index a6aa23bca..ef3abe85d 100644 --- a/docs/source/automl/hpo_res.md +++ b/docs/source/automl/hpo_res.md @@ -1,6 +1,6 @@ ## 调优结果 -在运行实验后，可以在命令行界面中找到如下的Web界面地址 ：\[Your IP\]:[Your Port] +在运行实验后，可以在命令行界面中找到如下的Web界面地址 ：\[Your IP\]:\[Your Port\] ![image.png](../../images/automl/pai_nni_create.jpg) ### 查看概要页面 diff --git a/docs/source/automl/pai_nni_hpo.md b/docs/source/automl/pai_nni_hpo.md index 1aa5e8a75..8f5a75fe4 100644 --- a/docs/source/automl/pai_nni_hpo.md +++ b/docs/source/automl/pai_nni_hpo.md @@ -253,8 +253,8 @@ train_config { [NNI searchSpace参考手册](https://nni.readthedocs.io/en/v2.2/Tutorial/SearchSpaceSpec.html) - {"\_type": "choice", "\_value": options}：从options中选取一个。 -- {"\_type": "randint", "\_value": [lower, upper]}：\[low,upper)之间选择一个随机整数。 -- {"\_type": "uniform", "\_value": [low, high]}：[low,upper]之间随机采样。 +- {"\_type": "randint", "\_value": \[lower, upper\]}：\[low,upper)之间选择一个随机整数。 +- {"\_type": "uniform", "\_value": \[low, high\]}：\[low,upper\]之间随机采样。 #### 高级 @@ -324,7 +324,7 @@ nnictl create --config config.yml --port=8780 - 如果NNICTL一开始成功，后续突然不成功，可以清空ECS环境中的python进程，重试 - 例如/mnt/data/project/project/exp/$experiment_id/log/nnictl_stderr.log中无报错，但是Failed to establish a new connection: [Errno 111] Connection refused')) + 例如/mnt/data/project/project/exp/$experiment_id/log/nnictl_stderr.log中无报错，但是Failed to establish a new connection: \[Errno 111\] Connection refused')) 命令：ps -ef|grep python|grep -v grep|cut -c 9-15|xargs kill -15 diff --git a/docs/source/benchmark.md b/docs/source/benchmark.md index db54d4a57..8a2c5348e 100644 --- a/docs/source/benchmark.md +++ b/docs/source/benchmark.md @@ -62,11 +62,11 @@ - 测试结果 | model | global_step | ctr auc | masked cvr auc | ctcvr auc | 训练时间 | config | -| --------------- | ----------- | --------- | -------------- | --------- | -------- | -------------------------------------------------------------------------------------------------------------------- | -| SimpleMultiTask | 4100 | 0.592606 | | 0.6306802 | 1小时 | [simple_multi_task.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/simple_multi_task.config) | -| MMoE | 3100 | 0.5869702 | | 0.6330008 | 1小时 | [mmoe.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/mmoe.config) | -| ESMM | 800 | 0.5974812 | 0.6841141 | 0.6362526 | 3小时 | [esmm.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/esmm.config) | -| PLE | 3200 | 0.5874 | | 0.6159 | 2小时 | [ple.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/ple.config) | +| --------------- | ----------- | --------- | -------------- | --------- | ---- | -------------------------------------------------------------------------------------------------------------------- | +| SimpleMultiTask | 4100 | 0.592606 | | 0.6306802 | 1小时 | [simple_multi_task.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/simple_multi_task.config) | +| MMoE | 3100 | 0.5869702 | | 0.6330008 | 1小时 | [mmoe.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/mmoe.config) | +| ESMM | 800 | 0.5974812 | 0.6841141 | 0.6362526 | 3小时 | [esmm.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/esmm.config) | +| PLE | 3200 | 0.5874 | | 0.6159 | 2小时 | [ple.config](http://easyrec.oss-cn-beijing.aliyuncs.com/benchmark/multi_task/ple.config) | ### CENSUS diff --git a/docs/source/component/backbone.md b/docs/source/component/backbone.md index 74f006037..195661301 100644 --- a/docs/source/component/backbone.md +++ b/docs/source/component/backbone.md @@ -316,9 +316,9 @@ x3 = Cross()(x0, x2) MovieLens-1M数据集效果对比： -| Model | Epoch | AUC | -| ------------------- | ----- | ------ | -| DCN （内置） | 1 | 0.8576 | +| Model | Epoch | AUC | +| ----------------- | ----- | ------ | +| DCN （内置） | 1 | 0.8576 | | DCN_v2 （backbone） | 1 | 0.8770 | 备注：新实现的`Cross`组件对应了参数量更多的v2版本的DCN，而内置的DCN模型对应了v1版本的DCN。 diff --git a/docs/source/component/component.md b/docs/source/component/component.md index baab90584..8ef90b79e 100644 --- a/docs/source/component/component.md +++ b/docs/source/component/component.md @@ -2,61 +2,61 @@ ## 1.基础组件 -| 类名 | 功能 | 说明 | 示例 | -| ----------------- | ------------ | ------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | -| MLP | 多层感知机 | 可定制激活函数、initializer、Dropout、BN等 | [案例1](backbone.html#wide-deep) | -| Highway | 类似残差链接 | 可用来对预训练embedding做增量微调 | [highway network](../models/highway.html) | -| Gate | 门控 | 多个输入的加权求和 | [Cross Decoupling Network](../models/cdn.html#id2) | -| PeriodicEmbedding | 周期激活函数 | 数值特征Embedding | [案例5](backbone.html#dlrm-embedding) | -| AutoDisEmbedding | 自动离散化 | 数值特征Embedding | [dlrm_on_criteo_with_autodis.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/dlrm_on_criteo_with_autodis.config) | -| NaryDisEmbedding | N进制编码 | 数值特征Embedding | [dlrm_on_criteo_with_narydis.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/dlrm_on_criteo_with_narydis.config) | -| TextCNN | 文本卷积 | 提取文本序列的特征 | [text_cnn_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/text_cnn_on_movielens.config) | +| 类名 | 功能 | 说明 | 示例 | +| ----------------- | ------ | ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| MLP | 多层感知机 | 可定制激活函数、initializer、Dropout、BN等 | [案例1](backbone.html#wide-deep) | +| Highway | 类似残差链接 | 可用来对预训练embedding做增量微调 | [highway network](../models/highway.html) | +| Gate | 门控 | 多个输入的加权求和 | [Cross Decoupling Network](../models/cdn.html#id2) | +| PeriodicEmbedding | 周期激活函数 | 数值特征Embedding | [案例5](backbone.html#dlrm-embedding) | +| AutoDisEmbedding | 自动离散化 | 数值特征Embedding | [dlrm_on_criteo_with_autodis.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/dlrm_on_criteo_with_autodis.config) | +| NaryDisEmbedding | N进制编码 | 数值特征Embedding | [dlrm_on_criteo_with_narydis.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/dlrm_on_criteo_with_narydis.config) | +| TextCNN | 文本卷积 | 提取文本序列的特征 | [text_cnn_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/text_cnn_on_movielens.config) | **备注**：Gate组件的第一个输入是权重向量，后面的输入拼凑成一个列表，权重向量的长度应等于列表的长度 ## 2.特征交叉组件 -| 类名 | 功能 | 说明 | 示例 | -| -------------- | ---------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------- | -| FM | 二阶交叉 | DeepFM模型的组件 | [案例2](backbone.html#deepfm) | -| DotInteraction | 二阶内积交叉 | DLRM模型的组件 | [案例4](backbone.html#dlrm) | -| Cross | bit-wise交叉 | DCN v2模型的组件 | [案例3](backbone.html#dcn) | -| BiLinear | 双线性 | FiBiNet模型的组件 | [fibinet_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/fibinet_on_movielens.config) | -| FiBiNet | SENet & BiLinear | FiBiNet模型 | [fibinet_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/fibinet_on_movielens.config) | +| 类名 | 功能 | 说明 | 示例 | +| -------------- | ---------------- | ------------ | -------------------------------------------------------------------------------------------------------------------------- | +| FM | 二阶交叉 | DeepFM模型的组件 | [案例2](backbone.html#deepfm) | +| DotInteraction | 二阶内积交叉 | DLRM模型的组件 | [案例4](backbone.html#dlrm) | +| Cross | bit-wise交叉 | DCN v2模型的组件 | [案例3](backbone.html#dcn) | +| BiLinear | 双线性 | FiBiNet模型的组件 | [fibinet_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/fibinet_on_movielens.config) | +| FiBiNet | SENet & BiLinear | FiBiNet模型 | [fibinet_on_movielens.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/fibinet_on_movielens.config) | ## 3.特征重要度学习组件 -| 类名 | 功能 | 说明 | 示例 | -| --------- | ------------------------- | ----------------- | ----------------------------------------------------- | -| SENet | 建模特征重要度 | FiBiNet模型的组件 | [MMoE](../models/mmoe.html#id4) | -| MaskBlock | 建模特征重要度 | MaskNet模型的组件 | [Cross Decoupling Network](../models/cdn.html#id2) | -| MaskNet | 多个串行或并行的MaskBlock | MaskNet模型 | [DBMTL](../models/dbmtl.html#dbmtl-based-on-backbone) | -| PPNet | 参数个性化网络 | PPNet模型 | [PPNet](../models/ppnet.html#id2) | +| 类名 | 功能 | 说明 | 示例 | +| --------- | ----------------- | ------------ | ----------------------------------------------------- | +| SENet | 建模特征重要度 | FiBiNet模型的组件 | [MMoE](../models/mmoe.html#id4) | +| MaskBlock | 建模特征重要度 | MaskNet模型的组件 | [Cross Decoupling Network](../models/cdn.html#id2) | +| MaskNet | 多个串行或并行的MaskBlock | MaskNet模型 | [DBMTL](../models/dbmtl.html#dbmtl-based-on-backbone) | +| PPNet | 参数个性化网络 | PPNet模型 | [PPNet](../models/ppnet.html#id2) | ## 4. 序列特征编码组件 -| 类名 | 功能 | 说明 | 示例 | -| ------------------ | --------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------ | -| DIN | target attention | DIN模型的组件 | [DIN_backbone.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/din_backbone_on_taobao.config) | -| BST | transformer | BST模型的组件 | [BST_backbone.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/bst_backbone_on_taobao.config) | -| SeqAugment | 序列数据增强 | crop, mask, reorder | [CL4SRec](../models/cl4srec.html#id2) | -| Attention | Dot-product attention | Transformer模型的组件 | | -| MultiHeadAttention | Multi-head attention | Transformer模型的组件 | | -| TransformerBlock | Transformer layer | Transformer模型的组件 | | -| TransformerEncoder | Transformer encoder | Transformer模型的组件 | | -| TextEncoder | BERT 模型 | 类似BERT模型 | | +| 类名 | 功能 | 说明 | 示例 | +| ------------------ | --------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| DIN | target attention | DIN模型的组件 | [DIN_backbone.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/din_backbone_on_taobao.config) | +| BST | transformer | BST模型的组件 | [BST_backbone.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/bst_backbone_on_taobao.config) | +| SeqAugment | 序列数据增强 | crop, mask, reorder | [CL4SRec](../models/cl4srec.html#id2) | +| Attention | Dot-product attention | Transformer模型的组件 | | +| MultiHeadAttention | Multi-head attention | Transformer模型的组件 | | +| TransformerBlock | Transformer layer | Transformer模型的组件 | | +| TransformerEncoder | Transformer encoder | Transformer模型的组件 | | +| TextEncoder | BERT 模型 | 类似BERT模型 | | ## 5. 多目标学习组件 -| 类名 | 功能 | 说明 | 示例 | -| --------- | --------------------------- | -------------- | ----------------------------- | -| MMoE | Multiple Mixture of Experts | MMoE模型的组件 | [案例8](backbone.html#mmoe) | -| AITMTower | AITM模型的一个tower | AITM模型的组件 | [AITM](../models/aitm.md#id2) | +| 类名 | 功能 | 说明 | 示例 | +| --------- | --------------------------- | --------- | ----------------------------- | +| MMoE | Multiple Mixture of Experts | MMoE模型的组件 | [案例8](backbone.html#mmoe) | +| AITMTower | AITM模型的一个tower | AITM模型的组件 | [AITM](../models/aitm.md#id2) | ## 6. 辅助损失函数组件 -| 类名 | 功能 | 说明 | 示例 | -| ------------- | -------------------- | ------------------ | -------------------------- | +| 类名 | 功能 | 说明 | 示例 | +| ------------- | ---------- | --------- | ------------------------ | | AuxiliaryLoss | 用来计算辅助损失函数 | 常用在自监督学习中 | [案例7](backbone.html#id7) | # 组件详细参数 @@ -65,74 +65,74 @@ - MLP （多层感知机） -| 参数 | 类型 | 默认值 | 说明 | -| ----------------------- | ---- | ---------- | ------------------------------------- | -| hidden_units | list | | 各隐层单元数 | -| dropout_ratio | list | | 各隐层dropout rate | -| activation | str | relu | 每层的激活函数 | -| use_bn | bool | true | 是否使用batch normalization | -| use_final_bn | bool | true | 最后一层是否使用batch normalization | -| use_bias | bool | false | 是否使用偏置项 | -| use_final_bias | bool | false | 最后一层是否使用偏置项 | -| final_activation | str | relu | 最后一层的激活函数 | +| 参数 | 类型 | 默认值 | 说明 | +| ----------------------- | ---- | ---------- | --------------------------- | +| hidden_units | list | | 各隐层单元数 | +| dropout_ratio | list | | 各隐层dropout rate | +| activation | str | relu | 每层的激活函数 | +| use_bn | bool | true | 是否使用batch normalization | +| use_final_bn | bool | true | 最后一层是否使用batch normalization | +| use_bias | bool | false | 是否使用偏置项 | +| use_final_bias | bool | false | 最后一层是否使用偏置项 | +| final_activation | str | relu | 最后一层的激活函数 | | initializer | str | he_uniform | 权重初始化方法，参考keras Dense layer | | use_bn_after_activation | bool | false | 是否在激活函数之后做batch norm | - HighWay -| 参数 | 类型 | 默认值 | 说明 | -| -------------- | ------ | ------ | -------------------- | -| emb_size | uint32 | None | embedding维度 | -| activation | str | gelu | 激活函数 | -| dropout_rate | float | 0 | dropout rate | -| init_gate_bias | float | -3.0 | 门控网络的bias初始值 | -| num_layers | int | 1 | 网络层数 | +| 参数 | 类型 | 默认值 | 说明 | +| -------------- | ------ | ---- | ------------ | +| emb_size | uint32 | None | embedding维度 | +| activation | str | gelu | 激活函数 | +| dropout_rate | float | 0 | dropout rate | +| init_gate_bias | float | -3.0 | 门控网络的bias初始值 | +| num_layers | int | 1 | 网络层数 | - PeriodicEmbedding -| 参数 | 类型 | 默认值 | 说明 | -| ------------------ | ------ | ------ | -------------------------------------------------------------- | -| embedding_dim | uint32 | | embedding维度 | -| sigma | float | | 初始化自定义参数时的标准差，**效果敏感、小心调参** | -| add_linear_layer | bool | true | 是否在embedding之后添加额外的层 | -| linear_activation | str | relu | 额外添加的层的激活函数 | -| output_tensor_list | bool | false | 是否同时输出embedding列表 | -| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | +| 参数 | 类型 | 默认值 | 说明 | +| ------------------ | ------ | ----- | ------------------------------------------------- | +| embedding_dim | uint32 | | embedding维度 | +| sigma | float | | 初始化自定义参数时的标准差，**效果敏感、小心调参** | +| add_linear_layer | bool | true | 是否在embedding之后添加额外的层 | +| linear_activation | str | relu | 额外添加的层的激活函数 | +| output_tensor_list | bool | false | 是否同时输出embedding列表 | +| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | - AutoDisEmbedding -| 参数 | 类型 | 默认值 | 说明 | -| ------------------ | ------ | ------ | -------------------------------------------------------------- | -| embedding_dim | uint32 | | embedding维度 | -| num_bins | uint32 | | 虚拟分桶数量 | -| keep_prob | float | 0.8 | 残差链接的权重 | -| temperature | float | | softmax函数的温度系数 | -| output_tensor_list | bool | false | 是否同时输出embedding列表 | -| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | +| 参数 | 类型 | 默认值 | 说明 | +| ------------------ | ------ | ----- | ------------------------------------------------- | +| embedding_dim | uint32 | | embedding维度 | +| num_bins | uint32 | | 虚拟分桶数量 | +| keep_prob | float | 0.8 | 残差链接的权重 | +| temperature | float | | softmax函数的温度系数 | +| output_tensor_list | bool | false | 是否同时输出embedding列表 | +| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | - NaryDisEmbedding -| 参数 | 类型 | 默认值 | 说明 | -| ------------------ | ------ | ------ | ------------------------------------------------------------------------- | -| embedding_dim | uint32 | | embedding维度 | -| carries | list | | N-ary 数值特征需要编码的进制列表 | -| multiplier | float | 1.0 | 针对float类型的特征，放大`multiplier`倍再取整后进行进制编码 | -| intra_ary_pooling | string | sum | 同一进制的不同位的数字embedding如何聚合成最终的embedding, 可选：sum, mean | -| num_replicas | uint32 | 1 | 每个特征输出多少个embedding表征 | -| output_tensor_list | bool | false | 是否同时输出embedding列表 | -| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | +| 参数 | 类型 | 默认值 | 说明 | +| ------------------ | ------ | ----- | --------------------------------------------------- | +| embedding_dim | uint32 | | embedding维度 | +| carries | list | | N-ary 数值特征需要编码的进制列表 | +| multiplier | float | 1.0 | 针对float类型的特征，放大`multiplier`倍再取整后进行进制编码 | +| intra_ary_pooling | string | sum | 同一进制的不同位的数字embedding如何聚合成最终的embedding, 可选：sum, mean | +| num_replicas | uint32 | 1 | 每个特征输出多少个embedding表征 | +| output_tensor_list | bool | false | 是否同时输出embedding列表 | +| output_3d_tensor | bool | false | 是否同时输出3d tensor, `output_tensor_list=true`时该参数不生效 | 备注：该组件依赖自定义Tensorflow OP，可能在某些版本的TF上无法使用 - TextCNN -| 参数 | 类型 | 默认值 | 说明 | -| ------------------- | ------------ | ------ | -------------------- | -| num_filters | list | | 卷积核个数列表 | -| filter_sizes | list | | 卷积核步长列表 | -| activation | string | relu | 卷积操作的激活函数 | -| pad_sequence_length | uint32 | | 序列补齐或截断的长度 | -| mlp | MLP | | protobuf message | +| 参数 | 类型 | 默认值 | 说明 | +| ------------------- | ------------ | ---- | ---------------- | +| num_filters | list | | 卷积核个数列表 | +| filter_sizes | list | | 卷积核步长列表 | +| activation | string | relu | 卷积操作的激活函数 | +| pad_sequence_length | uint32 | | 序列补齐或截断的长度 | +| mlp | MLP | | protobuf message | 备注：pad_sequence_length 参数必须要配置，否则模型predict的分数可能不稳定 @@ -140,128 +140,128 @@ - FM -| 参数 | 类型 | 默认值 | 说明 | -| ----------- | ---- | ------ | -------------------------------------------------- | -| use_variant | bool | false | 是否使用FM的变体：所有二阶交叉项直接输出，而不求和 | +| 参数 | 类型 | 默认值 | 说明 | +| ----------- | ---- | ----- | -------------------------- | +| use_variant | bool | false | 是否使用FM的变体：所有二阶交叉项直接输出，而不求和 | - DotInteraction -| 参数 | 类型 | 默认值 | 说明 | -| ---------------- | ---- | ------ | -------------------------------------------------------------------- | -| self_interaction | bool | false | 是否运行特征自己与自己交叉 | -| skip_gather | bool | false | 一个优化开关，设置为true，可以提高运行速度，但需要占用更多的内存空间 | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------- | ---- | ----- | ------------------------------------ | +| self_interaction | bool | false | 是否运行特征自己与自己交叉 | +| skip_gather | bool | false | 一个优化开关，设置为true，可以提高运行速度，但需要占用更多的内存空间 | - Cross -| 参数 | 类型 | 默认值 | 说明 | -| ------------------ | ------ | ---------------- | -------------------------------------------------------------------------------------------------------------------------- | -| projection_dim | uint32 | None | 使用矩阵分解降低计算开销，把大的权重矩阵分解为两个小的矩阵相乘，projection_dim是第一个小矩阵的列数，也是第二个小矩阵的行数 | -| diag_scale | float | 0 | used to increase the diagonal of the kernel W by `diag_scale`, that is, W + diag_scale * I, where I is an identity matrix | -| use_bias | bool | true | whether to add a bias term for this layer. | -| kernel_initializer | string | truncated_normal | Initializer to use on the kernel matrix | -| bias_initializer | string | zeros | Initializer to use on the bias vector | -| kernel_regularizer | string | None | Regularizer to use on the kernel matrix | -| bias_regularizer | string | None | Regularizer to use on bias vector | +| 参数 | 类型 | 默认值 | 说明 | +| ------------------ | ------ | ---------------- | ------------------------------------------------------------------------------------------------------------------------- | +| projection_dim | uint32 | None | 使用矩阵分解降低计算开销，把大的权重矩阵分解为两个小的矩阵相乘，projection_dim是第一个小矩阵的列数，也是第二个小矩阵的行数 | +| diag_scale | float | 0 | used to increase the diagonal of the kernel W by `diag_scale`, that is, W + diag_scale * I, where I is an identity matrix | +| use_bias | bool | true | whether to add a bias term for this layer. | +| kernel_initializer | string | truncated_normal | Initializer to use on the kernel matrix | +| bias_initializer | string | zeros | Initializer to use on the bias vector | +| kernel_regularizer | string | None | Regularizer to use on the kernel matrix | +| bias_regularizer | string | None | Regularizer to use on bias vector | - Bilinear -| 参数 | 类型 | 默认值 | 说明 | -| ---------------- | ------ | ----------- | ---------------- | -| type | string | interaction | 双线性类型 | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------- | ------ | ----------- | ---------- | +| type | string | interaction | 双线性类型 | | use_plus | bool | true | 是否使用plus版本 | -| num_output_units | uint32 | | 输出size | +| num_output_units | uint32 | | 输出size | - FiBiNet -| 参数 | 类型 | 默认值 | 说明 | -| -------- | -------- | ------ | ---------------- | -| bilinear | Bilinear | | protobuf message | -| senet | SENet | | protobuf message | -| mlp | MLP | | protobuf message | +| 参数 | 类型 | 默认值 | 说明 | +| -------- | -------- | --- | ---------------- | +| bilinear | Bilinear | | protobuf message | +| senet | SENet | | protobuf message | +| mlp | MLP | | protobuf message | ## 3.特征重要度学习组件 - SENet -| 参数 | 类型 | 默认值 | 说明 | -| --------------------- | ------ | ------ | -------------------------- | -| reduction_ratio | uint32 | 4 | 隐层单元数量缩减倍数 | -| num_squeeze_group | uint32 | 2 | 压缩分组数量 | -| use_skip_connection | bool | true | 是否使用残差连接 | -| use_output_layer_norm | bool | true | 是否在输出层使用layer norm | +| 参数 | 类型 | 默认值 | 说明 | +| --------------------- | ------ | ---- | ------------------ | +| reduction_ratio | uint32 | 4 | 隐层单元数量缩减倍数 | +| num_squeeze_group | uint32 | 2 | 压缩分组数量 | +| use_skip_connection | bool | true | 是否使用残差连接 | +| use_output_layer_norm | bool | true | 是否在输出层使用layer norm | - MaskBlock -| 参数 | 类型 | 默认值 | 说明 | -| ---------------- | ------ | ------ | ------------------------------------------------------------- | -| output_size | uint32 | | 输出层单元数 | -| reduction_factor | float | | 隐层单元数缩减因子 | -| aggregation_size | uint32 | | 隐层单元数 | -| input_layer_norm | bool | true | 输入是否需要做layer norm | -| projection_dim | uint32 | | 用两个小矩阵相乘代替原来的输入-隐层权重矩阵，配置小矩阵的维数 | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------- | ------ | ---- | ------------------------------- | +| output_size | uint32 | | 输出层单元数 | +| reduction_factor | float | | 隐层单元数缩减因子 | +| aggregation_size | uint32 | | 隐层单元数 | +| input_layer_norm | bool | true | 输入是否需要做layer norm | +| projection_dim | uint32 | | 用两个小矩阵相乘代替原来的输入-隐层权重矩阵，配置小矩阵的维数 | - MaskNet -| 参数 | 类型 | 默认值 | 说明 | -| ------------ | ---- | ------ | ----------------- | -| mask_blocks | list | | MaskBlock结构列表 | -| use_parallel | bool | true | 是否使用并行模式 | -| mlp | MLP | 可选 | 顶部mlp | +| 参数 | 类型 | 默认值 | 说明 | +| ------------ | ---- | ---- | ------------- | +| mask_blocks | list | | MaskBlock结构列表 | +| use_parallel | bool | true | 是否使用并行模式 | +| mlp | MLP | 可选 | 顶部mlp | - PPNet -| 参数 | 类型 | 默认值 | 说明 | -| --------------- | ------ | ------ | --------------------------------------------------------------------------- | -| mlp | MLP | | mlp 配置 | -| gate_params | GateNN | | 参数个性化Gate网络的配置 | -| mode | string | eager | 配置参数个性化是作用在MLP的每个layer的输入上还是输出上，可选：[eager, lazy] | -| full_gate_input | bool | true | 是否需要添加stop_gradient之后的mlp的输入作为gate网络的输入 | +| 参数 | 类型 | 默认值 | 说明 | +| --------------- | ------ | ----- | -------------------------------------------------- | +| mlp | MLP | | mlp 配置 | +| gate_params | GateNN | | 参数个性化Gate网络的配置 | +| mode | string | eager | 配置参数个性化是作用在MLP的每个layer的输入上还是输出上，可选：\[eager, lazy\] | +| full_gate_input | bool | true | 是否需要添加stop_gradient之后的mlp的输入作为gate网络的输入 | 其中，GateNN的参数如下： -| 参数 | 类型 | 默认值 | 说明 | -| ------------ | ------ | ---------------------- | ----------------------------------------------------------------- | +| 参数 | 类型 | 默认值 | 说明 | +| ------------ | ------ | --------------- | ----------------------------------------- | | output_dim | uint32 | mlp前一层的输出units数 | Gate网络的输出维度，eager模式下必须要配置为mlp第一层的输入units数 | -| hidden_dim | uint32 | output_dim | 隐层单元数 | -| dropout_rate | float | 0.0 | 隐层dropout rate | -| activation | str | relu | 隐层的激活函数 | -| use_bn | bool | true | 隐层是否使用batch normalization | +| hidden_dim | uint32 | output_dim | 隐层单元数 | +| dropout_rate | float | 0.0 | 隐层dropout rate | +| activation | str | relu | 隐层的激活函数 | +| use_bn | bool | true | 隐层是否使用batch normalization | ## 4. 序列特征编码组件 - SeqAugment (序列数据增强) -| 参数 | 类型 | 默认值 | 说明 | -| ------------ | ----- | ------ | ----------------------- | -| mask_rate | float | 0.6 | 被mask掉的token比率 | -| crop_rate | float | 0.2 | 裁剪保留的token比率 | -| reorder_rate | float | 0.6 | shuffle的子序列长度占比 | +| 参数 | 类型 | 默认值 | 说明 | +| ------------ | ----- | --- | --------------- | +| mask_rate | float | 0.6 | 被mask掉的token比率 | +| crop_rate | float | 0.2 | 裁剪保留的token比率 | +| reorder_rate | float | 0.6 | shuffle的子序列长度占比 | - DIN -| 参数 | 类型 | 默认值 | 说明 | -| -------------------- | ------ | ------- | ----------------------------- | -| attention_dnn | MLP | | attention unit mlp | +| 参数 | 类型 | 默认值 | 说明 | +| -------------------- | ------ | ------- | ------------------------- | +| attention_dnn | MLP | | attention unit mlp | | need_target_feature | bool | true | 是否返回target item embedding | -| attention_normalizer | string | softmax | softmax or sigmoid | +| attention_normalizer | string | softmax | softmax or sigmoid | - BST -| 参数 | 类型 | 默认值 | 说明 | -| ---------------------------- | ------ | ------ | ----------------------------------------------- | -| hidden_size | int | | transformer 编码层单元数 | -| num_hidden_layers | int | | transformer层数 | -| num_attention_heads | int | | transformer head数 | -| intermediate_size | int | | transformer中间层单元数 | -| hidden_act | string | gelu | 隐藏层激活函数 | -| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | -| attention_probs_dropout_prob | float | 0.1 | attention层dropout rate | -| max_position_embeddings | int | 512 | 序列最大长度 | -| use_position_embeddings | bool | true | 是否使用位置编码 | -| initializer_range | float | 0.2 | 权重参数初始值的区间范围 | -| output_all_token_embeddings | bool | true | 是否输出所有token embedding | -| target_item_position | string | head | target item的插入位置，可选：head, tail, ignore | -| reserve_target_position | bool | true | 是否为target item保留一个位置 | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------------------- | ------ | ---- | -------------------------------------- | +| hidden_size | int | | transformer 编码层单元数 | +| num_hidden_layers | int | | transformer层数 | +| num_attention_heads | int | | transformer head数 | +| intermediate_size | int | | transformer中间层单元数 | +| hidden_act | string | gelu | 隐藏层激活函数 | +| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | +| attention_probs_dropout_prob | float | 0.1 | attention层dropout rate | +| max_position_embeddings | int | 512 | 序列最大长度 | +| use_position_embeddings | bool | true | 是否使用位置编码 | +| initializer_range | float | 0.2 | 权重参数初始值的区间范围 | +| output_all_token_embeddings | bool | true | 是否输出所有token embedding | +| target_item_position | string | head | target item的插入位置，可选：head, tail, ignore | +| reserve_target_position | bool | true | 是否为target item保留一个位置 | - Attention @@ -273,15 +273,15 @@ The calculation follows the steps: 1. Use scores to calculate a softmax distribution with shape (batch_size, Tq, Tv). 1. Use the softmax distribution to create a linear combination of value with shape (batch_size, Tq, dim). -| 参数 | 类型 | 默认值 | 说明 | -| ----------------------- | ------ | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| use_scale | bool | False | If True, will create a scalar variable to scale the attention scores. | -| scale_by_dim | bool | Fasle | whether to scale by dimension | -| score_mode | string | dot | Function to use to compute attention scores, one of {"dot", "concat"}. "dot" refers to the dot product between the query and key vectors. "concat" refers to the hyperbolic tangent of the concatenation of the query and key vectors. | -| dropout | float | 0.0 | Float between 0 and 1. Fraction of the units to drop for the attention scores. | -| seed | int | None | A Python integer to use as random seed incase of dropout. | -| return_attention_scores | bool | False | if True, returns the attention scores (after masking and softmax) as an additional output argument. | -| use_causal_mask | bool | False | Set to True for decoder self-attention. Adds a mask such that position i cannot attend to positions j > i. This prevents the flow of information from the future towards the past. | +| 参数 | 类型 | 默认值 | 说明 | +| ----------------------- | ------ | ----- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| use_scale | bool | False | If True, will create a scalar variable to scale the attention scores. | +| scale_by_dim | bool | Fasle | whether to scale by dimension | +| score_mode | string | dot | Function to use to compute attention scores, one of {"dot", "concat"}. "dot" refers to the dot product between the query and key vectors. "concat" refers to the hyperbolic tangent of the concatenation of the query and key vectors. | +| dropout | float | 0.0 | Float between 0 and 1. Fraction of the units to drop for the attention scores. | +| seed | int | None | A Python integer to use as random seed incase of dropout. | +| return_attention_scores | bool | False | if True, returns the attention scores (after masking and softmax) as an additional output argument. | +| use_causal_mask | bool | False | Set to True for decoder self-attention. Adds a mask such that position i cannot attend to positions j > i. This prevents the flow of information from the future towards the past. | > - inputs: List of the following tensors: > \- query: Query tensor of shape (batch_size, Tq, dim). @@ -293,83 +293,83 @@ The calculation follows the steps: - MultiHeadAttention -| 参数 | 类型 | 默认值 | 说明 | -| ----------------------- | ------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 参数 | 类型 | 默认值 | 说明 | +| ----------------------- | ------ | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | num_heads | uint32 | 无 | Number of attention heads. | -| key_dim | uint32 | | Size of each attention head for query and key. | -| value_dim | uint32 | | Size of each attention head for value. | -| dropout | float | 0.0 | Dropout probability. | -| use_bias | bool | true | whether the dense layers use bias vectors/matrices. | -| return_attention_scores | bool | false | whether the output should be (attention_output, attention_scores) | -| use_causal_mask | bool | false | whether to apply a causal mask to prevent tokens from attending to future tokens (e.g., used in a decoder Transformer). | -| output_shape | uint32 | | The expected shape of an output tensor, besides the batch and sequence dims. If not specified, projects back to the query feature dim (the query input's last dimension). | -| kernel_initializer | string | | Initializer for dense layer kernels. | -| bias_initializer | string | | Initializer for dense layer biases. | +| key_dim | uint32 | | Size of each attention head for query and key. | +| value_dim | uint32 | | Size of each attention head for value. | +| dropout | float | 0.0 | Dropout probability. | +| use_bias | bool | true | whether the dense layers use bias vectors/matrices. | +| return_attention_scores | bool | false | whether the output should be (attention_output, attention_scores) | +| use_causal_mask | bool | false | whether to apply a causal mask to prevent tokens from attending to future tokens (e.g., used in a decoder Transformer). | +| output_shape | uint32 | | The expected shape of an output tensor, besides the batch and sequence dims. If not specified, projects back to the query feature dim (the query input's last dimension). | +| kernel_initializer | string | | Initializer for dense layer kernels. | +| bias_initializer | string | | Initializer for dense layer biases. | - TransformerBlock Transformer encoder 的其中一个layer。 -| 参数 | 类型 | 默认值 | 说明 | -| ---------------------------- | ------ | ------ | ------------------------- | -| hidden_size | int | | transformer 编码层单元数 | -| num_attention_heads | int | | transformer head数 | -| intermediate_size | int | | transformer中间层单元数 | -| hidden_act | string | relu | 隐藏层激活函数 | -| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | -| attention_probs_dropout_prob | float | 0.0 | attention层的dropout rate | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------------------- | ------ | ---- | ----------------------- | +| hidden_size | int | | transformer 编码层单元数 | +| num_attention_heads | int | | transformer head数 | +| intermediate_size | int | | transformer中间层单元数 | +| hidden_act | string | relu | 隐藏层激活函数 | +| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | +| attention_probs_dropout_prob | float | 0.0 | attention层的dropout rate | - TransformerEncoder -| 参数 | 类型 | 默认值 | 说明 | -| ---------------------------- | ------ | ------ | --------------------------- | -| vocab_size | uint32 | | 词汇表大小 | -| hidden_size | uint32 | | transformer 编码层单元数 | -| num_hidden_layers | uint32 | | transformer层数 | -| num_attention_heads | uint32 | | transformer head数 | -| intermediate_size | uint32 | | transformer中间层单元数 | -| hidden_act | string | relu | 隐藏层激活函数 | -| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | -| attention_probs_dropout_prob | float | 0.0 | attention层的dropout rate | -| max_position_embeddings | uint32 | 512 | 序列最大长度 | -| output_all_token_embeddings | bool | true | 是否输出所有token embedding | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------------------- | ------ | ---- | ----------------------- | +| vocab_size | uint32 | | 词汇表大小 | +| hidden_size | uint32 | | transformer 编码层单元数 | +| num_hidden_layers | uint32 | | transformer层数 | +| num_attention_heads | uint32 | | transformer head数 | +| intermediate_size | uint32 | | transformer中间层单元数 | +| hidden_act | string | relu | 隐藏层激活函数 | +| hidden_dropout_prob | float | 0.1 | 隐藏层dropout rate | +| attention_probs_dropout_prob | float | 0.0 | attention层的dropout rate | +| max_position_embeddings | uint32 | 512 | 序列最大长度 | +| output_all_token_embeddings | bool | true | 是否输出所有token embedding | - TextEncoder BERT模型结构 -| 参数 | 类型 | 默认值 | 说明 | -| ---------------- | ------------------ | ------ | -------------------------------------------- | -| transformer | TransformerEncoder | 无 | transformer 子组件的配置 | -| separator | string | ' ' | 文本分隔符 | -| vocab_file | string | 无 | 词汇表文件路径，不设置时使用hash获得token id | -| default_token_id | int32 | 0 | Out of vocabulary 的token的默认id | +| 参数 | 类型 | 默认值 | 说明 | +| ---------------- | ------------------ | --- | ----------------------------- | +| transformer | TransformerEncoder | 无 | transformer 子组件的配置 | +| separator | string | ' ' | 文本分隔符 | +| vocab_file | string | 无 | 词汇表文件路径，不设置时使用hash获得token id | +| default_token_id | int32 | 0 | Out of vocabulary 的token的默认id | ## 5. 多任务学习组件 - MMoE -| 参数 | 类型 | 默认值 | 说明 | -| ---------- | ------ | ------ | --------------- | -| num_task | uint32 | | 任务数 | -| num_expert | uint32 | 0 | expert数量 | -| expert_mlp | MLP | 可选 | expert的mlp参数 | +| 参数 | 类型 | 默认值 | 说明 | +| ---------- | ------ | --- | ------------ | +| num_task | uint32 | | 任务数 | +| num_expert | uint32 | 0 | expert数量 | +| expert_mlp | MLP | 可选 | expert的mlp参数 | - AITMTower -| 参数 | 类型 | 默认值 | 说明 | -| ------------- | ------ | ------ | --------------------------------- | +| 参数 | 类型 | 默认值 | 说明 | +| ------------- | ------ | ---- | ------------------------------ | | project_dim | uint32 | 可选 | attention Query, Key, Value的维度 | -| stop_gradient | bool | True | 是否需要停用对依赖的输入的梯度 | -| transfer_mlp | MLP | | transfer的mlp参数 | +| stop_gradient | bool | True | 是否需要停用对依赖的输入的梯度 | +| transfer_mlp | MLP | | transfer的mlp参数 | ## 6. 计算辅助损失函数的组件 - AuxiliaryLoss -| 参数 | 类型 | 默认值 | 说明 | -| ----------- | ------ | ------ | ----------------------------------------------- | -| loss_type | string | | 损失函数类型，包括：l2_loss, nce_loss, info_nce | -| loss_weight | float | 1.0 | 损失函数权重 | -| temperature | float | 0.1 | info_nce & nec loss 的参数 | -| 其他 | | | 根据loss_type决定 | +| 参数 | 类型 | 默认值 | 说明 | +| ----------- | ------ | --- | ------------------------------------- | +| loss_type | string | | 损失函数类型，包括：l2_loss, nce_loss, info_nce | +| loss_weight | float | 1.0 | 损失函数权重 | +| temperature | float | 0.1 | info_nce & nec loss 的参数 | +| 其他 | | | 根据loss_type决定 | diff --git a/docs/source/conf.py b/docs/source/conf.py index d5d68dd8f..3945e9eed 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -41,19 +41,19 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', - 'sphinx.ext.napoleon', - 'recommonmark', - 'sphinx_markdown_tables', - 'post_process', + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon', + 'recommonmark', + 'sphinx_markdown_tables', + 'post_process', ] # Add any paths that contain templates here, relative to this directory. @@ -123,25 +123,26 @@ # -- Options for LaTeX output ------------------------------------------------ latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'easy_rec.tex', 'easy\\_rec Documentation', 'EasyRec Team', 'manual'), + (master_doc, 'easy_rec.tex', 'easy\\_rec Documentation', 'EasyRec Team', + 'manual'), ] # -- Options for manual page output ------------------------------------------ @@ -156,15 +157,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ( - master_doc, - 'easy_rec', - 'easy_rec Documentation', - author, - 'easy_rec', - 'One line description of project.', - 'Miscellaneous', - ), + ( + master_doc, + 'easy_rec', + 'easy_rec Documentation', + author, + 'easy_rec', + 'One line description of project.', + 'Miscellaneous', + ), ] # -- Options for Epub output ------------------------------------------------- @@ -197,6 +198,6 @@ todo_include_todos = True autodoc_default_options = { - 'member-order': 'bysource', - 'special-members': '__init__', + 'member-order': 'bysource', + 'special-members': '__init__', } diff --git a/docs/source/eval.md b/docs/source/eval.md index 816053808..06271ac7d 100644 --- a/docs/source/eval.md +++ b/docs/source/eval.md @@ -39,22 +39,22 @@ eval_config { **备注** 在多目标模型里, eval_config 的 metrics_set 无效，请使用 model 里面的 metrics_set 配置 -参见文档 [MODEL]->[多目标模型] 配置中的 model_config.[model].task_towers.metrics_set +参见文档 \[MODEL\]->\[多目标模型\] 配置中的 model_config.\[model\].task_towers.metrics_set ### Metric: | MetricClass | Example | 适用模型 | -| ------------------ | ------------------------- | ------------------------------------------------------------------------- | -| Accuracy | accuracy {} | 多分类模型LossType=CLASSIFICATION, num_class > 1 | +| ------------------ | ------------------------- | --------------------------------------------------------------------- | +| Accuracy | accuracy {} | 多分类模型LossType=CLASSIFICATION, num_class > 1 | | MeanAbsoluteError | mean_absolute_error {} | 回归模型LossType=L2_LOSS | -| RecallAtTopK | recall_at_topk {} | 多分类模型LossType=CLASSIFICATION, num_class > 1; CoMetricLearningI2I模型 | +| RecallAtTopK | recall_at_topk {} | 多分类模型LossType=CLASSIFICATION, num_class > 1; CoMetricLearningI2I模型 | | Max_F1 | max_f1 {} | 分类模型LossType=CLASSIFICATION | | MeanSquaredError | mean_squared_error{} | 回归模型LossType=L2_LOSS | -| AUC | auc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | -| GAUC | gauc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | -| SessionAUC | session_auc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | -| Precision | precision{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | -| Recall | recall{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | +| AUC | auc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | +| GAUC | gauc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | +| SessionAUC | session_auc{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | +| Precision | precision{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | +| Recall | recall{} | 二分类模型LossType=CLASSIFICATION, num_class = 1 | | AvgPrecisionAtTopK | precision_at_topk{topk=5} | CoMetricLearningI2I模型专用, LossType=CIRCLE_LOSS / MULTI_SIMILARITY_LOSS | **备注** diff --git a/docs/source/export.md b/docs/source/export.md index f5df579b4..fd09fbb5f 100644 --- a/docs/source/export.md +++ b/docs/source/export.md @@ -101,11 +101,11 @@ pai -name easy_rec_ext -project algo_public - --asset_files: 需要导出的asset文件路径, 可设置多个, 逗号分隔； - 如果需要导出到assets目录的子目录下，使用`${target_path}:${source_path}`的格式；（从版本0.8.7开始支持） - e.g. '--asset_files custom_fg_lib/fg.json:oss://${bucket}/path/to/fg.json' -- 模型导出之后可以使用(EasyRecProcessor)[./predict/在线预测.md]部署到PAI-EAS平台 +- 模型导出之后可以使用(EasyRecProcessor)\[./predict/在线预测.md\]部署到PAI-EAS平台 ### 双塔召回模型 -如果是双塔召回模型(如dssm, mind等), 模型导出之后, 一般还需要进行模型切分和索引构建, 才能使用(EasyRecProcessor)[./predict/在线预测.md]部署到PAI-EAS上. +如果是双塔召回模型(如dssm, mind等), 模型导出之后, 一般还需要进行模型切分和索引构建, 才能使用(EasyRecProcessor)\[./predict/在线预测.md\]部署到PAI-EAS上. #### 模型切分 diff --git a/docs/source/faq.md b/docs/source/faq.md index 55212b805..11ef98889 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -272,7 +272,7 @@ Original error: #### 输出user塔的embedding时，输出为空 -用tfResponse.getDoubleVals("user_emb")去打印结果出来的是否返回的是[]，空的数组 +用tfResponse.getDoubleVals("user_emb")去打印结果出来的是否返回的是\[\]，空的数组 需要使用 tfResponse.getStringVals("user_emb") diff --git a/docs/source/feature/fg.md b/docs/source/feature/fg.md index a3edff8ab..c69a1cfc5 100644 --- a/docs/source/feature/fg.md +++ b/docs/source/feature/fg.md @@ -55,21 +55,21 @@ FG模块在推荐系统架构中的位置如下图所示： FG支持的特征变换算子与EasyRec支持的特征（`Feature Column`）之间没有严格的对应关系，大致可以参加如下表格： -| FG 算子 | EasyRec Feature Column | -| :-------------------- | :------------------------------------ | -| id_feature | IdFeature 或 TagFeature | -| raw_feature | RawFeature | -| expr_feature | RawFeature | -| combo_feature | IdFeature 或 TagFeature | +| FG 算子 | EasyRec Feature Column | +| :-------------------- | :---------------------------------- | +| id_feature | IdFeature 或 TagFeature | +| raw_feature | RawFeature | +| expr_feature | RawFeature | +| combo_feature | IdFeature 或 TagFeature | | lookup_feature | RawFeature 或 IdFeature 或 TagFeature | | match_feature | RawFeature 或 IdFeature 或 TagFeature | -| overlap_feature | RawFeature | -| sequence_feature | SequenceFeature 或 TagFeature | -| bm25_feature | RawFeature | -| kv_dot_product | RawFeature | -| tokenize_feature | TagFeature | -| text_normalizer | IdFeature | -| regex_replace_feature | IdFeature | +| overlap_feature | RawFeature | +| sequence_feature | SequenceFeature 或 TagFeature | +| bm25_feature | RawFeature | +| kv_dot_product | RawFeature | +| tokenize_feature | TagFeature | +| text_normalizer | IdFeature | +| regex_replace_feature | IdFeature | 备注：**FG的执行结果输出给EasyRec模型，两种之间是串联的关系**。 @@ -111,7 +111,7 @@ pai -name easy_rec_ext - 参数说明: [请参考](../export.md#pai) - 注意事项: - - 请检查fg.config, 保证导出的模型是支持多个placeholder的输入[每个特征一个placeholder] + - 请检查fg.config, 保证导出的模型是支持多个placeholder的输入\[每个特征一个placeholder\] ``` export_config { @@ -123,7 +123,7 @@ pai -name easy_rec_ext - 如果有设置feature_config.features.max_partitions, 请加入下面的命令重置: - - -Dedit_config_json='{"feature_config.features[:].max_partitions":1}'进行修改, 可以获得更好的性能 + - -Dedit_config_json='{"feature_config.features\[:\].max_partitions":1}'进行修改, 可以获得更好的性能 #### 特征筛选 @@ -187,11 +187,11 @@ eascmd -i -k -e update ali_rec_rn - tables: item特征存储在hologres表里面, 支持分多个表存储 - key: 必填, itemId列的名字; - value: 可选，需要加载的列名, 多个列名之间用逗号(,)分割; - - condition: 可选，where子语句支持筛选item, 如itemId < 10000; + - condition: 可选，where子语句支持筛选item, 如itemId \< 10000; - timekey: 可选，用于item的增量更新，支持的格式: timestamp和int - static: 可选, 表示是静态特征，不用周期性更新 - 支持多个item表, 如果多张表有重复的列, 后面的表覆盖前面的表 - - "tables": [{"key":"table1", ...},{"key":"table2", ...}] + - "tables": \[{"key":"table1", ...},{"key":"table2", ...}\] - 如果多张表有重复的列，后面的表将覆盖前面的表 - hologres表里面每一列存储一个item特征,示例:
CPU利用率 QPS AVG RT TP99
优化前 89 33 288 362
优化后 93 226 34 57
@@ -202,7 +202,7 @@ eascmd -i -k -e update ali_rec_rn
- remote_type: Item特征数据源, 目前支持：`hologres`, `none` - hologres：通过SQL接口进行数据读取和写入，适用于海量数据的存储和查询 - - none: 不使用Item特征缓存，item特征通过请求传入，此时tables应设置为[] + - none: 不使用Item特征缓存，item特征通过请求传入，此时tables应设置为\[\] - storage: 将oss的模型目录mount到docker的指定目录下 - mount_path: docker内部的挂载路径, 与示例保持一致即可 - 配置了storage就不需要配置model_path了 diff --git a/docs/source/feature/fg_docs/ComboFeature.md b/docs/source/feature/fg_docs/ComboFeature.md index 5d706eafa..5e0495cdd 100644 --- a/docs/source/feature/fg_docs/ComboFeature.md +++ b/docs/source/feature/fg_docs/ComboFeature.md @@ -18,11 +18,11 @@ combo_feature是多个字段（或表达式）的组合（即笛卡尔积），i ^\]表示多值分隔符，注意这是一个符号，其ASCII编码是"\\x1D"，而不是两个符号 -| user:age_class的取值 | item:item_id的取值 | 输出的feature | -| -------------------- | ------------------ | ---------------------------------------------------------------------------------------------------------- | -| 123 | 45678 | comb_u_age_item_123_45678 | -| abc, bcd | 45678 | comb_u_age_item_abc_45678, comb_u_age_item_bcd_45678 | -| abc, bcd | 12345^\]45678 | comb_u_age_item_abc_12345, comb_u_age_item_abc_45678, comb_u_age_item_bcd_12345, comb_u_age_item_bcd_45678 | +| user:age_class的取值 | item:item_id的取值 | 输出的feature | +| ----------------- | --------------- | ---------------------------------------------------------------------------------------------------------- | +| 123 | 45678 | comb_u_age_item_123_45678 | +| abc, bcd | 45678 | comb_u_age_item_abc_45678, comb_u_age_item_bcd_45678 | +| abc, bcd | 12345^\]45678 | comb_u_age_item_abc_12345, comb_u_age_item_abc_45678, comb_u_age_item_bcd_12345, comb_u_age_item_bcd_45678 | 输出的feature个数等于 diff --git a/docs/source/feature/fg_docs/IdFeature.md b/docs/source/feature/fg_docs/IdFeature.md index b2b06e3f7..27e2f4c54 100644 --- a/docs/source/feature/fg_docs/IdFeature.md +++ b/docs/source/feature/fg_docs/IdFeature.md @@ -14,23 +14,23 @@ id_feature表示离散特征, 包含单值离散特征和多值离散特征. } ``` -| 字段名 | 含义 | -| ------------ | ------------------------------------------------------------------------------------------------------------ | -| feature_name | 必选项，feature_name会被当做最终输出的feature的前缀 | -| expression | 必选项，expression描述该feature所依赖的字段来源 | +| 字段名 | 含义 | +| ------------ | ----------------------------------------------------------------------------- | +| feature_name | 必选项，feature_name会被当做最终输出的feature的前缀 | +| expression | 必选项，expression描述该feature所依赖的字段来源 | | need_prefix | 可选项，true表示会拼上feature_name作为前缀，false表示不拼，默认为true，通常在shared_embedding的场景会用false | ## 示例: 下面以item侧的特征is_main作为案例来说明在不同配置下特征的输入和输出: -| 类型 | item:is_main的取值 | 输出的feature | -| ---------- | ------------------ | ---------------------------------- | -| int64_t | 100 | item_is_main_100 | -| double | 5.2 | item_is_main_5（小数部分会被截取） | -| string | abc | item_is_main_abc | -| 多值string | abc^\]bcd | item_is_main_abc^Citem_is_main_bcd | -| 多值int | 123^\]456 | item_is_main_123^Citem_is_main_456 | +| 类型 | item:is_main的取值 | 输出的feature | +| -------- | --------------- | ---------------------------------- | +| int64_t | 100 | item_is_main_100 | +| double | 5.2 | item_is_main_5（小数部分会被截取） | +| string | abc | item_is_main_abc | +| 多值string | abc^\]bcd | item_is_main_abc^Citem_is_main_bcd | +| 多值int | 123^\]456 | item_is_main_123^Citem_is_main_456 | - ^\]表示多值分隔符，注意这是一个符号，其ASCII编码是"\\x1D", 也可以写作"\\u001d" - ^C是FG encode之后输出的特征值的分隔符, 其ASCII编码是"\\x03" diff --git a/docs/source/feature/fg_docs/OverLapFeature.md b/docs/source/feature/fg_docs/OverLapFeature.md index 534e12ddb..b6396db61 100644 --- a/docs/source/feature/fg_docs/OverLapFeature.md +++ b/docs/source/feature/fg_docs/OverLapFeature.md @@ -6,16 +6,16 @@ 离线推荐使用1.3.56-SNAPSHOT这个版本。 ps: 写fg的时候注意维度，title的维度要大于或等于query的问题（简单来说就是如果title是user特征，那query也只能是user特征，user特征的batch size为1，商品特征的batch size为商品数） -| 方式 | 描述 | 备注 | -| ------------------- | ----------------------------------------------------------- | ------------------------------ | -| common_word | 计算query与title间重复term，并输出为fg_common1_common2 | 重复数不超过query term数 | -| diff_word | 计算query与title间不重复term，并输出为fg_diff1_diff2 | 不重复数不超过query term数 | -| query_common_ratio | 计算query与title间重复term数占query中term比例,乘以10取下整 | 取值为[0,10] | -| title_common_ratio | 计算query与title间重复term数占title中term比例,乘以100取下整 | 取值为[0,100] | -| is_contain | 计算query是否全部包含在title中，保持顺序 | 0表示未包含，1表示包含 | -| is_equal | 计算query是否与title完全相同 | 0表示不完全相同，1表示完全相同 | -| common_word_divided | 计算query与title间重复term，并输出为fg_common1, fg_common2 | 重复数不超过query term数 | -| diff_word_divided | 计算query与title间不重复term，并输出为fg_diff1, fg_diff2 | 重复数不超过query term数 | +| 方式 | 描述 | 备注 | +| ------------------- | ----------------------------------------------- | ------------------ | +| common_word | 计算query与title间重复term，并输出为fg_common1_common2 | 重复数不超过query term数 | +| diff_word | 计算query与title间不重复term，并输出为fg_diff1_diff2 | 不重复数不超过query term数 | +| query_common_ratio | 计算query与title间重复term数占query中term比例,乘以10取下整 | 取值为\[0,10\] | +| title_common_ratio | 计算query与title间重复term数占title中term比例,乘以100取下整 | 取值为\[0,100\] | +| is_contain | 计算query是否全部包含在title中，保持顺序 | 0表示未包含，1表示包含 | +| is_equal | 计算query是否与title完全相同 | 0表示不完全相同，1表示完全相同 | +| common_word_divided | 计算query与title间重复term，并输出为fg_common1, fg_common2 | 重复数不超过query term数 | +| diff_word_divided | 计算query与title间不重复term，并输出为fg_diff1, fg_diff2 | 重复数不超过query term数 | ## 配置方法 @@ -30,14 +30,14 @@ } ``` -| 字段名 | 含义 | -| ------------ | ------------------------------------------------------------------------------------------------- | -| feature_type | 必选项，描述改feature的类型 | -| feature_name | 必选项，feature_name会被当做最终输出的feature的前缀 | -| query | 必选项，query依赖的表, attr1是一个多值string, 多值string的分隔符使用chr(29) | -| title | 必选项，title依赖的表, attr2是一个多值string | +| 字段名 | 含义 | +| ------------ | -------------------------------------------------------------------------------------- | +| feature_type | 必选项，描述改feature的类型 | +| feature_name | 必选项，feature_name会被当做最终输出的feature的前缀 | +| query | 必选项，query依赖的表, attr1是一个多值string, 多值string的分隔符使用chr(29) | +| title | 必选项，title依赖的表, attr2是一个多值string | | method | 可填common_word, diff_word, query_common_ratio, title_common_ratio, is_contain， 对应上图五种方式 | -| separator | 输出结果中的分割字符，不填写我们默认为\_ ，但也可以用户自己定制，具体看例子 | +| separator | 输出结果中的分割字符，不填写我们默认为\_ ，但也可以用户自己定制，具体看例子 | ## 例子 diff --git a/docs/source/feature/fg_docs/RawFeature.md b/docs/source/feature/fg_docs/RawFeature.md index f81b3ced1..9159d315d 100644 --- a/docs/source/feature/fg_docs/RawFeature.md +++ b/docs/source/feature/fg_docs/RawFeature.md @@ -15,22 +15,22 @@ raw_feature表示连续值特征, 支持数值int、float、double等数值类 } ``` -| 字段名 | 含义 | -| --------------- | -------------------------------------------------------------------------------------- | -| feature_name | 必选项, 特征名 | +| 字段名 | 含义 | +| --------------- | ------------------------------------------------------------ | +| feature_name | 必选项, 特征名 | | expression | 必选项，expression描述该feature所依赖的字段来源, 来源必须是user、item、context中的一种 | -| value_dimension | 可选项，默认值为1，表示输出的字段的维度 | -| normalizer | 可选项，归一化方法，详见后文 | +| value_dimension | 可选项，默认值为1，表示输出的字段的维度 | +| normalizer | 可选项，归一化方法，详见后文 | ## 示例 ^\]表示多值分隔符，注意这是一个符号，其ASCII编码是"\\x1D"，而不是两个符号 -| 类型 | item:ctr的取值 | 输出的feature | -| ------- | -------------- | -------------------------------------------------------------- | -| int64_t | 100 | (ctr, 100) | -| double | 100.1 | (ctr, 100.1) | -| 多值int | 123^\]456 | (ctr, (123,456)) (注意，输入字段必须与配置的dimension维度一致) | +| 类型 | item:ctr的取值 | 输出的feature | +| ------- | ----------- | --------------------------------------------- | +| int64_t | 100 | (ctr, 100) | +| double | 100.1 | (ctr, 100.1) | +| 多值int | 123^\]456 | (ctr, (123,456)) (注意，输入字段必须与配置的dimension维度一致) | ## Normalizer diff --git a/docs/source/feature/odl_sample.md b/docs/source/feature/odl_sample.md index 02f623b07..493ad5081 100644 --- a/docs/source/feature/odl_sample.md +++ b/docs/source/feature/odl_sample.md @@ -299,7 +299,7 @@ ``` - ttl(miliseconds)的设置考虑两个因素: - odl_sample_with_lbl相对请求时间request_time的延迟 - - ttl < 相对延迟, 就会有样本丢失 + - ttl \< 相对延迟, 就会有样本丢失 - 统计相对延迟: - 将odl_sample_with_lbl / odl_callback_log落到MaxCompute - 按request_id join 计算ts的差异 diff --git a/docs/source/feature/rtp_fg.md b/docs/source/feature/rtp_fg.md index 41160cd91..ef3f66e11 100644 --- a/docs/source/feature/rtp_fg.md +++ b/docs/source/feature/rtp_fg.md @@ -44,7 +44,7 @@ - 如果设成false, 转换成EasyRec的config时会转成IdFeature, 可以减少字符串分割的开销 - - 多值分隔符使用chr(29)[ctrl+v ctrl+], 即"\\u001D". + - 多值分隔符使用chr(29)\[ctrl+v ctrl+\], 即"\\u001D". - [多值类型说明](./fg_docs/mutiValues.md) @@ -132,13 +132,13 @@ - 双层查找, 根据category和item_id查找value. - - match feature里面多值分隔符可以使用chr(29) (ctrl+v ctrl+\])或者逗号[,]， 如: + - match feature里面多值分隔符可以使用chr(29) (ctrl+v ctrl+\])或者逗号\[,\]， 如: ``` 50011740^107287172:0.2^]36806676:0.3^]122572685:0.5|50006842^16788816:0.1^]10122:0.2^]29889:0.3^]30068:19 ``` - - needWeighting: 生成特征权重，即kv格式, kv之间用[ctrl+v ctrl+e]分割, 转换成TagFeature. + - needWeighting: 生成特征权重，即kv格式, kv之间用\[ctrl+v ctrl+e\]分割, 转换成TagFeature. - [sequence_feature](./fg_docs/SequenceFeature.md) @@ -293,7 +293,7 @@ tunnel download taobao_fg_test_out taobao_fg_test_out.txt -fd=';'; | ----- | ------- | ------- | ------------------------------------------------------------------------------------------------------------- | | 0 | 336811 | 100002 | user_id_100002^Bcms_segid_5^Bcms_group_id_2^Bage_level_2^Bpvalue_level_1^Bshopping_level_3^Boccupation_1^B... | -#### 从配置文件[fg.json]生成EasyRec的config +#### 从配置文件\[fg.json\]生成EasyRec的config 从Git克隆EasyRec @@ -325,7 +325,7 @@ python -m easy_rec.python.tools.convert_rtp_fg --label is_product_detail is_pur - --separator: feature之间的分隔符, 默认是CTRL_B(u0002) -- --selected_cols: 指定输入列，包括label、[sample_weight]和features，其中label可以指定多列，表示要使用多个label(一般是多任务模型), 最后一列必须是features, 如: +- --selected_cols: 指定输入列，包括label、\[sample_weight\]和features，其中label可以指定多列，表示要使用多个label(一般是多任务模型), 最后一列必须是features, 如: ``` label0,label1,sample_weight,features @@ -392,7 +392,7 @@ pai -name easy_rec_ext - 参数说明: [请参考](../export.md#pai) - 注意事项: - - 请检查fg.config, 保证导出的模型是支持多个placeholder的输入[每个特征一个placeholder] + - 请检查fg.config, 保证导出的模型是支持多个placeholder的输入\[每个特征一个placeholder\] ``` export_config { @@ -404,7 +404,7 @@ pai -name easy_rec_ext - 如果有设置feature_config.features.max_partitions, 请加入下面的命令重置: - - -Dedit_config_json='{"feature_config.features[:].max_partitions":1}'进行修改, 可以获得更好的性能 + - -Dedit_config_json='{"feature_config.features\[:\].max_partitions":1}'进行修改, 可以获得更好的性能 #### 增加特征 @@ -499,7 +499,7 @@ eascmd -i -k -e update ali_rec_rn - tables: item特征存储在hologres表里面, 支持分多个表存储 - key: 必填, itemId列的名字; - value: 可选，需要加载的列名, 多个列名之间用逗号(,)分割; - - condition: 可选，where子语句支持筛选item, 如itemId < 10000; + - condition: 可选，where子语句支持筛选item, 如itemId \< 10000; - timekey: 可选，用于item的增量更新，支持的格式: timestamp和int - static: 可选, 表示是静态特征，不用周期性更新 - 支持多个item表, 如果多张表有重复的列, 后面的表覆盖前面的表 diff --git a/docs/source/feature/rtp_native.md b/docs/source/feature/rtp_native.md index e36815ab8..7d3730674 100644 --- a/docs/source/feature/rtp_native.md +++ b/docs/source/feature/rtp_native.md @@ -34,7 +34,7 @@ - is_multi: id_feature是否是多值属性 - 默认是false, 转换成EasyRec的config时会转成IdFeature - 如果设成true, 转换成EasyRec的config时会转成TagFeature. - - 多值分隔符使用chr(29)[ctrl+v ctrl+]. + - 多值分隔符使用chr(29)\[ctrl+v ctrl+\]. - num_buckets: 当输入是unsigned int类型的时候，并且输入有界的时候，可以指定num_bucket为输入的最大值. - hash_bucket_size: 对应EasyRec feature_config.features的hash_bucket_size。hash_bucket方式是目前RTP唯一支持的embedding方式 - embedding_dimension/embedding_dim: 对应EasyRec feature_config.features里面的embedding_dim. @@ -102,13 +102,13 @@ - [MatchFeature](http://easyrec.oss-cn-beijing.aliyuncs.com/fg_docs/MatchFeature.pdf) - 双层查找, 根据category和item_id查找value. - - match Feature里面多值分隔符可以使用chr(29) (ctrl+v ctrl+\])或者逗号[,]， 如: + - match Feature里面多值分隔符可以使用chr(29) (ctrl+v ctrl+\])或者逗号\[,\]， 如: ``` 50011740^107287172:0.2^]36806676:0.3^]122572685:0.5|50006842^16788816:0.1^]10122:0.2^]29889:0.3^]30068:19 ``` - - needWeighting: 生成特征权重，即kv格式, kv之间用[ctrl+v ctrl+e]分割, 转换成TagFeature. + - needWeighting: 生成特征权重，即kv格式, kv之间用\[ctrl+v ctrl+e\]分割, 转换成TagFeature. - [OverLapFeature](http://easyrec.oss-cn-beijing.aliyuncs.com/fg_docs/OverLapFeature.pdf) @@ -120,7 +120,7 @@ - combiner: 默认是mean, 也可以是sum. - 影响数据生成和 EasyRec feature_config 生成, 主要是多值Feature. - [多值类型说明](http://easyrec.oss-cn-beijing.aliyuncs.com/fg_docs/%E5%A4%9A%E5%80%BC%E7%B1%BB%E5%9E%8B.pdf) - - 多值feature使用chr(29)[ctrl+v ctrl+]\]作为分隔符. + - 多值feature使用chr(29)\[ctrl+v ctrl+\]\]作为分隔符. - 全局配置说明: diff --git a/docs/source/intro.md b/docs/source/intro.md index f735da430..1f1cc70a0 100644 --- a/docs/source/intro.md +++ b/docs/source/intro.md @@ -30,7 +30,7 @@ EasyRec implements state of the art machine learning models used in common recom ### Simple to config - Flexible feature config and simple model config -- Efficient and robust feature generation[used in taobao] +- Efficient and robust feature generation\[used in taobao\] - Nice web interface in development ### It is smart diff --git a/docs/source/models/cmbf.md b/docs/source/models/cmbf.md index 7761cf7ce..78cf5404a 100644 --- a/docs/source/models/cmbf.md +++ b/docs/source/models/cmbf.md @@ -105,7 +105,7 @@ model_config: { - 配置一个名为`general`的feature_group，包含需要做跨模态attention的常规特征，这些特征的`embedding_dim`必须相同。 - 配置一个名为`text`的feature_group，包含需要做跨模态attention的不定长文本序列特征，这些特征的`embedding_dim`必须相同。 - 注意：CMBF 模型要求所有文本侧（包括`text`和`general`两个特征组）输入特征的 embedding_dim 保持一致。 - - [可选] 配置一个名为`other`的feature_group，包含不需要做跨模态attention的其他特征，如各类统计特征。 + - \[可选\] 配置一个名为`other`的feature_group，包含不需要做跨模态attention的其他特征，如各类统计特征。 - cmbf/config: CMBF 模型相关的参数 @@ -115,7 +115,7 @@ model_config: { - text_multi_head_num: 文本单模态学习模块中的 head 数量，默认为1 - image_head_size: 单模态学习模块中的图像tower，multi-headed self-attention的每个head的size - text_head_size: 单模态学习模块中的文本tower，multi-headed self-attention的每个head的size - - image_feature_patch_num: [可选，默认值为1] 当只有一个image feature时生效，表示该图像特征是一个复合embedding，维度为`image_feature_patch_num * embedding_size`。 + - image_feature_patch_num: \[可选，默认值为1\] 当只有一个image feature时生效，表示该图像特征是一个复合embedding，维度为`image_feature_patch_num * embedding_size`。 - image_self_attention_layer_num: 单模态学习模块中的图像tower，multi-headed self-attention的层数；当只有一个图像特征时，可设置为0 - text_self_attention_layer_num: 单模态学习模块中的文本tower，multi-headed self-attention的层数 - cross_modal_layer_num: 跨模态融合模块的层数，建议设在1到5之间，默认为1 @@ -127,7 +127,7 @@ model_config: { - use_position_embeddings: bool, default is true；是否为文本序列添加位置编码 - max_position_embeddings: 文本序列的最大位置，当`use_position_embeddings`为true时，必须配置；并且必须大于或等于所有特征配置`max_seq_len`的最大值 - text_seq_emb_dropout_prob: 文本序列embedding的dropout概率 - - other_feature_dnn: [可选] 其他特征的MLP网络配置 + - other_feature_dnn: \[可选\] 其他特征的MLP网络配置 - cmbf/final_dnn: 输出模块的MLP网络配置 diff --git a/docs/source/models/dbmtl.md b/docs/source/models/dbmtl.md index 7c6483a2b..aa4015aa7 100644 --- a/docs/source/models/dbmtl.md +++ b/docs/source/models/dbmtl.md @@ -342,7 +342,7 @@ model_config: { - text_multi_head_num: 文本单模态学习模块中的 head 数量，默认为1 - image_head_size: 单模态学习模块中的图像tower，multi-headed self-attention的每个head的size - text_head_size: 单模态学习模块中的文本tower，multi-headed self-attention的每个head的size - - image_feature_patch_num: [可选，默认值为1] 当只有一个image feature时生效，表示该图像特征是一个复合embedding，维度为`image_feature_patch_num * embedding_size`。 + - image_feature_patch_num: \[可选，默认值为1\] 当只有一个image feature时生效，表示该图像特征是一个复合embedding，维度为`image_feature_patch_num * embedding_size`。 - image_self_attention_layer_num: 单模态学习模块中的图像tower，multi-headed self-attention的层数 - text_self_attention_layer_num: 单模态学习模块中的文本tower，multi-headed self-attention的层数 - cross_modal_layer_num: 跨模态融合模块的层数，建议设在1到5之间，默认为1 @@ -354,7 +354,7 @@ model_config: { - use_position_embeddings: bool, default is true；是否为文本序列添加位置编码 - max_position_embeddings: 文本序列的最大位置，当`use_position_embeddings`为true时，必须配置；并且必须大于或等于所有特征配置`max_seq_len`的最大值 - text_seq_emb_dropout_prob: 文本序列embedding的dropout概率 - - other_feature_dnn: [可选] 其他特征的MLP网络配置 + - other_feature_dnn: \[可选\] 其他特征的MLP网络配置 - 其余与dbmtl一致 #### DBMTL+UNITER @@ -461,7 +461,7 @@ model_config: { - hidden_dropout_prob: multi-headed attention模块中FC layer的dropout概率 - use_position_embeddings: bool, default is true；是否为文本序列添加位置编码 - max_position_embeddings: 文本序列的最大位置，当`use_position_embeddings`为true时，必须配置；并且必须大于或等于所有特征配置`max_seq_len`的最大值 - - other_feature_dnn: [可选] 其他特征的MLP网络配置 + - other_feature_dnn: \[可选\] 其他特征的MLP网络配置 - 其余与dbmtl一致 diff --git a/docs/source/models/dlrm.md b/docs/source/models/dlrm.md index 58f0617a5..66ad84e69 100644 --- a/docs/source/models/dlrm.md +++ b/docs/source/models/dlrm.md @@ -2,7 +2,7 @@ ### 简介 -DLRM(Deep Learning Recommendation Model for Personalization and Recommendation Systems[Facebook])是一种DNN模型, 支持使用连续值特征(price/age/...)和ID类特征(user_id/item_id/...), 并对特征之间的交互(interaction)进行了建模(基于内积的方式). +DLRM(Deep Learning Recommendation Model for Personalization and Recommendation Systems\[Facebook\])是一种DNN模型, 支持使用连续值特征(price/age/...)和ID类特征(user_id/item_id/...), 并对特征之间的交互(interaction)进行了建模(基于内积的方式). ``` output: diff --git a/docs/source/models/dssm.md b/docs/source/models/dssm.md index 1d7a9e206..243f57a92 100644 --- a/docs/source/models/dssm.md +++ b/docs/source/models/dssm.md @@ -55,7 +55,7 @@ model_config:{ - dnn: deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - id: 指定user_id/item_id列 -- simi_func: 向量相似度函数，包括[COSINE, INNER_PRODUCT, EUCLID]，默认COSINE，建议使用INNER_PRODUCT +- simi_func: 向量相似度函数，包括\[COSINE, INNER_PRODUCT, EUCLID\]，默认COSINE，建议使用INNER_PRODUCT - embedding_regularization: 对embedding部分加regularization，防止overfit 支持的metric_set包括: diff --git a/docs/source/models/dssm_neg_sampler.md b/docs/source/models/dssm_neg_sampler.md index bf8ecac70..14aa95288 100644 --- a/docs/source/models/dssm_neg_sampler.md +++ b/docs/source/models/dssm_neg_sampler.md @@ -99,7 +99,7 @@ model_config:{ - dnn: deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - id: 指定user_id/item_id列 -- simi_func: 向量相似度函数，包括[COSINE, INNER_PRODUCT, EUCLID]，默认COSINE，建议使用INNER_PRODUCT +- simi_func: 向量相似度函数，包括\[COSINE, INNER_PRODUCT, EUCLID\]，默认COSINE，建议使用INNER_PRODUCT - scale_simi: 是否自动缩放相似度便于loss计算，建议设置成false - loss_type: 目前只支持SOFTMAX_CROSS_ENTROPY - embedding_regularization: 对embedding部分加regularization，防止overfit diff --git a/docs/source/models/loss.md b/docs/source/models/loss.md index 8e5d4485b..f5c9cee32 100644 --- a/docs/source/models/loss.md +++ b/docs/source/models/loss.md @@ -4,28 +4,28 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 ### 使用单个损失函数 -| 损失函数 | 说明 | -| ------------------------------------------ | ---------------------------------------------------------------------------- | -| CLASSIFICATION | 分类Loss，二分类为sigmoid_cross_entropy；多分类为softmax_cross_entropy | -| L2_LOSS | 平方损失 | -| SIGMOID_L2_LOSS | 对sigmoid函数的结果计算平方损失 | -| CROSS_ENTROPY_LOSS | log loss 负对数损失 | -| BINARY_CROSS_ENTROPY_LOSS | 仅用在知识蒸馏中的BCE损失 | -| KL_DIVERGENCE_LOSS | 仅用在知识蒸馏中的KL散度损失 | -| CIRCLE_LOSS | CoMetricLearningI2I模型专用 | -| MULTI_SIMILARITY_LOSS | CoMetricLearningI2I模型专用 | -| SOFTMAX_CROSS_ENTROPY_WITH_NEGATIVE_MINING | 自动负采样版本的多分类softmax_cross_entropy，用在二分类任务中 | -| BINARY_FOCAL_LOSS | 支持困难样本挖掘和类别平衡的focal loss | -| PAIR_WISE_LOSS | 以优化全局AUC为目标的rank loss | -| PAIRWISE_FOCAL_LOSS | pair粒度的focal loss, 支持自定义pair分组 | -| PAIRWISE_LOGISTIC_LOSS | pair粒度的logistic loss, 支持自定义pair分组 | -| PAIRWISE_HINGE_LOSS | pair粒度的hinge loss, 支持自定义pair分组 | -| JRC_LOSS | 二分类 + listwise ranking loss | -| F1_REWEIGHTED_LOSS | 可以调整二分类召回率和准确率相对权重的损失函数，可有效对抗正负样本不平衡问题 | -| ORDER_CALIBRATE_LOSS | 使用目标依赖关系校正预测结果的辅助损失函数，详见[AITM](aitm.md)模型 | -| LISTWISE_RANK_LOSS | listwise的排序损失 | -| LISTWISE_DISTILL_LOSS | 用来蒸馏给定list排序的损失函数，与listwise rank loss 比较类似 | -| ZILN_LOSS | LTV预测任务的损失函数（num_class必须设置为3） | +| 损失函数 | 说明 | +| ------------------------------------------ | ---------------------------------------------------------- | +| CLASSIFICATION | 分类Loss，二分类为sigmoid_cross_entropy；多分类为softmax_cross_entropy | +| L2_LOSS | 平方损失 | +| SIGMOID_L2_LOSS | 对sigmoid函数的结果计算平方损失 | +| CROSS_ENTROPY_LOSS | log loss 负对数损失 | +| BINARY_CROSS_ENTROPY_LOSS | 仅用在知识蒸馏中的BCE损失 | +| KL_DIVERGENCE_LOSS | 仅用在知识蒸馏中的KL散度损失 | +| CIRCLE_LOSS | CoMetricLearningI2I模型专用 | +| MULTI_SIMILARITY_LOSS | CoMetricLearningI2I模型专用 | +| SOFTMAX_CROSS_ENTROPY_WITH_NEGATIVE_MINING | 自动负采样版本的多分类softmax_cross_entropy，用在二分类任务中 | +| BINARY_FOCAL_LOSS | 支持困难样本挖掘和类别平衡的focal loss | +| PAIR_WISE_LOSS | 以优化全局AUC为目标的rank loss | +| PAIRWISE_FOCAL_LOSS | pair粒度的focal loss, 支持自定义pair分组 | +| PAIRWISE_LOGISTIC_LOSS | pair粒度的logistic loss, 支持自定义pair分组 | +| PAIRWISE_HINGE_LOSS | pair粒度的hinge loss, 支持自定义pair分组 | +| JRC_LOSS | 二分类 + listwise ranking loss | +| F1_REWEIGHTED_LOSS | 可以调整二分类召回率和准确率相对权重的损失函数，可有效对抗正负样本不平衡问题 | +| ORDER_CALIBRATE_LOSS | 使用目标依赖关系校正预测结果的辅助损失函数，详见[AITM](aitm.md)模型 | +| LISTWISE_RANK_LOSS | listwise的排序损失 | +| LISTWISE_DISTILL_LOSS | 用来蒸馏给定list排序的损失函数，与listwise rank loss 比较类似 | +| ZILN_LOSS | LTV预测任务的损失函数（num_class必须设置为3） | - ZILN_LOSS：使用时模型有3个可选的输出（在多目标任务重，输出名有一个目标相关的后缀） - probs: 预估的转化概率 diff --git a/docs/source/models/mind.md b/docs/source/models/mind.md index 2ae30138a..3553ae3fd 100644 --- a/docs/source/models/mind.md +++ b/docs/source/models/mind.md @@ -137,7 +137,7 @@ model_config:{ - 使用增量训练，增量训练可以防止负采样的穿越。 -- 使用HPO对squash_pow[0.1 - 1.0]和simi_pow[10 - 100]进行搜索调优。 +- 使用HPO对squash_pow\[0.1 - 1.0\]和simi_pow\[10 - 100\]进行搜索调优。 - 要看的指标是召回率，准确率和兴趣损失，三个指标要一起看。 diff --git a/docs/source/models/multi_tower.md b/docs/source/models/multi_tower.md index f65675956..9e9d343e5 100644 --- a/docs/source/models/multi_tower.md +++ b/docs/source/models/multi_tower.md @@ -80,9 +80,9 @@ model_config: { - group_name: 可以根据实际情况取 - wide_deep: 必须是DEEP - losses: 可选，可以选择同时配置两个loss函数，并且为每个loss配置不同的权重 - - loss_type: CLASSIFICATION [默认值] 二分类的sigmoid cross entropy loss - - loss_type: PAIR_WISE_LOSS [可选] 以优化AUC为主要目标的 pairwise rank loss - - loss_type: F1_REWEIGHTED_LOSS [可选] 可以调节二分类模型recall/precision相对权重的loss; 注意不要与`loss_type: CLASSIFICATION`同时使用 + - loss_type: CLASSIFICATION \[默认值\] 二分类的sigmoid cross entropy loss + - loss_type: PAIR_WISE_LOSS \[可选\] 以优化AUC为主要目标的 pairwise rank loss + - loss_type: F1_REWEIGHTED_LOSS \[可选\] 可以调节二分类模型recall/precision相对权重的loss; 注意不要与`loss_type: CLASSIFICATION`同时使用 - f1_reweight_loss: 可以调节二分类模型`recall/precision`相对权重的损失函数 - f1_beta_square: 大于1的值会导致模型更关注`recall`，小于1的值会导致模型更关注`precision` - F1 分数，又称平衡F分数（balanced F Score），它被定义为精确率和召回率的调和平均数。 diff --git a/docs/source/models/uniter.md b/docs/source/models/uniter.md index 534f37851..5af92baba 100644 --- a/docs/source/models/uniter.md +++ b/docs/source/models/uniter.md @@ -100,7 +100,7 @@ model_config: { - 配置一个名为`general`的feature_group，包含需要做跨模态attention的常规特征，这些特征的`embedding_dim`必须相同。 - 配置一个名为`text`的feature_group，包含需要做跨模态attention的不定长文本序列特征，这些特征的`embedding_dim`必须相同。 - 注意：CMBF 模型要求所有文本侧（包括`text`和`general`两个特征组）输入特征的 embedding_dim 保持一致。 - - [可选] 配置一个名为`other`的feature_group，包含不需要做跨模态attention的其他特征，如各类统计特征。 + - \[可选\] 配置一个名为`other`的feature_group，包含不需要做跨模态attention的其他特征，如各类统计特征。 - uniter/config: UNITER 模型相关的参数 @@ -113,7 +113,7 @@ model_config: { - hidden_dropout_prob: multi-headed attention模块中FC layer的dropout概率 - use_position_embeddings: bool, default is true；是否为文本序列添加位置编码 - max_position_embeddings: 文本序列的最大位置，当`use_position_embeddings`为true时，必须配置；并且必须大于或等于所有特征配置`max_seq_len`的最大值 - - other_feature_dnn: [可选] 其他特征的MLP网络配置 + - other_feature_dnn: \[可选\] 其他特征的MLP网络配置 - uniter/final_dnn: 输出模块的MLP网络配置 diff --git a/docs/source/online_train.md b/docs/source/online_train.md index 4c4436c7b..981a31cef 100644 --- a/docs/source/online_train.md +++ b/docs/source/online_train.md @@ -325,5 +325,5 @@ train_config { ## A/B实验 -- 推荐引擎: 在推荐引擎[如PAI-REC]里面配置一个新的实验，更新[PAI-EAS服务配置](http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/pairec/docs/pairec/html/config/algo.html) +- 推荐引擎: 在推荐引擎\[如PAI-REC\]里面配置一个新的实验，更新[PAI-EAS服务配置](http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/pairec/docs/pairec/html/config/algo.html) - 报表: 天级报表, 小时级报表 diff --git "a/docs/source/predict/Local \347\246\273\347\272\277\351\242\204\346\265\213.md" "b/docs/source/predict/Local \347\246\273\347\272\277\351\242\204\346\265\213.md" index 1ae0f38d4..40046b86d 100644 --- "a/docs/source/predict/Local \347\246\273\347\272\277\351\242\204\346\265\213.md" +++ "b/docs/source/predict/Local \347\246\273\347\272\277\351\242\204\346\265\213.md" @@ -52,7 +52,7 @@ predictor = Predictor('model/export/') 1. list 格式 1. dict 格式 - 输出是list of dict，dict里面包含一个字段y，即score: 范围在[0, 1]之间 + 输出是list of dict，dict里面包含一个字段y，即score: 范围在\[0, 1\]之间 ``` # interface 1, input is a list of fields, the order of fields diff --git "a/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" "b/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" index cf7082baf..dd867a165 100644 --- "a/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" +++ "b/docs/source/predict/MaxCompute \347\246\273\347\272\277\351\242\204\346\265\213.md" @@ -54,12 +54,12 @@ pai -name easy_rec_ext - 二分类模型(要求num_class=1)，导出字段:logits、probs，对应: sigmoid之前的值/概率 - 回归模型，导出字段: y，对应: 预测值 - 多分类模型(num_class > 1)，导出字段: - - logits: string(json), softmax之前的vector, shape[num_class] - - probs: string(json), softmax之后的vector, shape[num_class] + - logits: string(json), softmax之前的vector, shape\[num_class\] + - probs: string(json), softmax之后的vector, shape\[num_class\] - 如果一个分类目标是is_click, 输出概率的变量名称是probs_is_click - 多目标模型中有一个回归目标是paytime，那么输出回归预测分的变量名称是：y_paytime - - logits_y: logits[y], float, 类别y对应的softmax之前的概率 - - probs_y: probs[y], float, 类别y对应的概率 + - logits_y: logits\[y\], float, 类别y对应的softmax之前的概率 + - probs_y: probs\[y\], float, 类别y对应的概率 - y: 类别id, = argmax(probs_y), int, 概率最大的类别 - 示例: ```sql diff --git a/docs/source/quick_start/designer_tutorial.md b/docs/source/quick_start/designer_tutorial.md index 168e18bf5..66a22d15c 100644 --- a/docs/source/quick_start/designer_tutorial.md +++ b/docs/source/quick_start/designer_tutorial.md @@ -24,33 +24,33 @@ PAI-Designer（Studio 2.0）是基于云原生架构Pipeline Service（PAIFlow ### 输入桩配置 -| 输入桩（从左到右） | 限制数据类型 | 对应PAI命令参数 | 是否必选 | -| ------------------ | ------------- | --------------------------------------------------------- | -------- | -| 训练表 | MaxCompute表 | `train_tables` | 是 | -| 评估表 | MaxCompute表 | `eval_tables` | 否 | -| checkpoint | OSS存储的模型 | `edit_config_json`中的`train_config.fine_tune_checkpoint` | 否 | -| 分箱表 | MaxCompute表 | `boundary_table` | 否 | +| 输入桩（从左到右） | 限制数据类型 | 对应PAI命令参数 | 是否必选 | +| ---------- | ----------- | ------------------------------------------------------- | ---- | +| 训练表 | MaxCompute表 | `train_tables` | 是 | +| 评估表 | MaxCompute表 | `eval_tables` | 否 | +| checkpoint | OSS存储的模型 | `edit_config_json`中的`train_config.fine_tune_checkpoint` | 否 | +| 分箱表 | MaxCompute表 | `boundary_table` | 否 | ### 右侧参数说明 -| 页签 | 参数 | 是否必选 | 描述 | 默认值 | -| -------- | ----------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | -| 参数设置 | 模型路径 | 否 | 对应PAI命令参数`model_dir` | 工作流自动设置的工作路径 | -| 参数设置 | EasyRec配置 | 是 | 在下方编辑框填写config配置，保存至指定的OSS路径下，对应PAI命令参数`config` | | -| 参数设置 | 指定算法版本 | 否 | 点开高级选项后，可以自定义EasyRec的执行版本。请先参考文档[EasyRec版本更新](../release.md)上传对应版本的tar包到OSS，在这个参数中选中上传的文件。对应参数`script` | 空 | -| 执行调优 | ps数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 2 | -| 执行调优 | ps CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | -| 执行调优 | ps Memory数量（MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | -| 执行调优 | Worker数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | -| 执行调优 | Worker CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | -| 执行调优 | Worker Memory用量（单位为MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | -| 执行调优 | Worker GPU卡数 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 0 | +| 页签 | 参数 | 是否必选 | 描述 | 默认值 | +| ---- | ---------------------- | ---- | ------------------------------------------------------------------------------------------------------ | ------------ | +| 参数设置 | 模型路径 | 否 | 对应PAI命令参数`model_dir` | 工作流自动设置的工作路径 | +| 参数设置 | EasyRec配置 | 是 | 在下方编辑框填写config配置，保存至指定的OSS路径下，对应PAI命令参数`config` | | +| 参数设置 | 指定算法版本 | 否 | 点开高级选项后，可以自定义EasyRec的执行版本。请先参考文档[EasyRec版本更新](../release.md)上传对应版本的tar包到OSS，在这个参数中选中上传的文件。对应参数`script` | 空 | +| 执行调优 | ps数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 2 | +| 执行调优 | ps CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | +| 执行调优 | ps Memory数量（MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | +| 执行调优 | Worker数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | +| 执行调优 | Worker CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | +| 执行调优 | Worker Memory用量（单位为MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | +| 执行调优 | Worker GPU卡数 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 0 | ### 输出桩配置 -| 输出桩（从左到右） | 数据类型 | 对应PAI命令参数 | -| ------------------ | ------------- | --------------- | -| 输出模型 | OSS存储的模型 | `model_dir ` | +| 输出桩（从左到右） | 数据类型 | 对应PAI命令参数 | +| --------- | -------- | ------------ | +| 输出模型 | OSS存储的模型 | `model_dir ` | ### 对应PAI命令 @@ -63,30 +63,30 @@ PAI-Designer（Studio 2.0）是基于云原生架构Pipeline Service（PAIFlow ### 输入桩配置 -| 输入桩（从左到右） | 限制数据类型 | 对应PAI命令参数 | 是否必选 | -| ------------------ | ------------- | ----------------- | -------- | -| 输入模型 | OSS存储的模型 | `saved_model_dir` | 是 | -| 输入表 | MaxCompute表 | `input_table` | 是 | +| 输入桩（从左到右） | 限制数据类型 | 对应PAI命令参数 | 是否必选 | +| --------- | ----------- | ----------------- | ---- | +| 输入模型 | OSS存储的模型 | `saved_model_dir` | 是 | +| 输入表 | MaxCompute表 | `input_table` | 是 | ### 右侧参数说明 -| 页签 | 参数 | 是否必选 | 描述 | 默认值 | -| -------- | ----------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------- | -| 参数设置 | 输入选择列 | 否 | 从输入表选择特征列给到预测模型，不能与排除列同时使用 | - | -| 参数设置 | 排除列 | 否 | 预测模型不需要使用的输入列，不能和输入选择列同时使用 | - | -| 参数设置 | 输出保留列 | 否 | 在预测结构表中原样输出的列 | - | -| 参数设置 | 预测详情输出列 | 否 | 选择预测模型的输出到MaxCompute表的映射，细节请参见[EasyRec离线预测文档](../predict/MaxCompute%20%E7%A6%BB%E7%BA%BF%E9%A2%84%E6%B5%8B.md) | 默认为"probs double" | -| 参数设置 | miniBatch的大小 | 否 | 对应参数`batch_size` | 1024 | -| 执行调优 | Worker数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | -| 执行调优 | Worker CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | -| 执行调优 | Worker Memory用量（单位为MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | -| 执行调优 | Worker GPU卡数 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 0 | +| 页签 | 参数 | 是否必选 | 描述 | 默认值 | +| ---- | ---------------------- | ---- | -------------------------------------------------------------------------------------------------------------- | ----------------- | +| 参数设置 | 输入选择列 | 否 | 从输入表选择特征列给到预测模型，不能与排除列同时使用 | - | +| 参数设置 | 排除列 | 否 | 预测模型不需要使用的输入列，不能和输入选择列同时使用 | - | +| 参数设置 | 输出保留列 | 否 | 在预测结构表中原样输出的列 | - | +| 参数设置 | 预测详情输出列 | 否 | 选择预测模型的输出到MaxCompute表的映射，细节请参见[EasyRec离线预测文档](../predict/MaxCompute%20%E7%A6%BB%E7%BA%BF%E9%A2%84%E6%B5%8B.md) | 默认为"probs double" | +| 参数设置 | miniBatch的大小 | 否 | 对应参数`batch_size` | 1024 | +| 执行调优 | Worker数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | +| 执行调优 | Worker CPU数量 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 6 | +| 执行调优 | Worker Memory用量（单位为MB） | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 30000 | +| 执行调优 | Worker GPU卡数 | 否 | 完整的执行调优参数会拼装成`cluster`参数 | 0 | ### 输出桩配置 -| 输出桩（从左到右） | 数据类型 | 对应PAI命令参数 | -| ------------------ | ------------ | --------------- | -| 输出表 | MaxCompute表 | `output_table ` | +| 输出桩（从左到右） | 数据类型 | 对应PAI命令参数 | +| --------- | ----------- | --------------- | +| 输出表 | MaxCompute表 | `output_table ` | ### 对应PAI命令 diff --git a/docs/source/quick_start/local_tutorial.md b/docs/source/quick_start/local_tutorial.md index 0fd6ada0d..4a00140b4 100644 --- a/docs/source/quick_start/local_tutorial.md +++ b/docs/source/quick_start/local_tutorial.md @@ -24,16 +24,16 @@ pip install tensorflow_probability==0.5.0 常见版本对应关系： | TensorFlow版本 | TensorFlowProbability版本 | -| -------------- | ------------------------- | -| 1.12 | 0.5.0 | -| 1.15 | 0.8.0 | -| 2.5.0 | 0.13.0 | -| 2.6.0 | 0.14.0 | -| 2.7.0 | 0.15.0 | -| 2.8.0 | 0.16.0 | -| 2.10 | 0.18.0 | -| 2.11 | 0.19.0 | -| 2.12 | 0.20.0 | +| ------------ | ----------------------- | +| 1.12 | 0.5.0 | +| 1.15 | 0.8.0 | +| 2.5.0 | 0.13.0 | +| 2.6.0 | 0.14.0 | +| 2.7.0 | 0.15.0 | +| 2.8.0 | 0.16.0 | +| 2.10 | 0.18.0 | +| 2.11 | 0.19.0 | +| 2.12 | 0.20.0 | 其他版本对应关系请查看链接：[Releases · tensorflow/probability](https://github.com/tensorflow/probability/releases)。 @@ -60,7 +60,7 @@ docker exec -it bash 可选镜像： -- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-paitf1.12-0.8.5 [只能跑在DLC环境] +- mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-paitf1.12-0.8.5 \[只能跑在DLC环境\] - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-paitf1.15-0.8.5 - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15.5-0.8.5 - mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15.5-gpu-0.8.5 diff --git a/docs/source/tf_on_yarn.md b/docs/source/tf_on_yarn.md index afbdc3da0..ba0f22972 100644 --- a/docs/source/tf_on_yarn.md +++ b/docs/source/tf_on_yarn.md @@ -18,7 +18,7 @@ Data Science版本的EMR集群支持GPU调度，所以在Core节点，推荐用 参数说明： -- -t APP_TYPE  提交的任务类型，支持三种类型的任务类型[tensorflow-ps, tensorflow-mpi, standalone]，三种类型要配合后面运行模式使用 +- -t APP_TYPE  提交的任务类型，支持三种类型的任务类型\[tensorflow-ps, tensorflow-mpi, standalone\]，三种类型要配合后面运行模式使用 > tensorflow-ps使用的是原生TensorFlow ps 类型 @@ -27,7 +27,7 @@ Data Science版本的EMR集群支持GPU调度，所以在Core节点，推荐用 > tensorflow-worker多worker模式，适用于MultiWorkerMirroredStrategy - -a APP_NAME 提交的任务名称，用户可以根据需要起名 -- -m MODE 提交的运行时环境，目前支持四种类型运行时环境[local, virtual-env,docker] +- -m MODE 提交的运行时环境，目前支持四种类型运行时环境\[local, virtual-env,docker\] > local 使用的是emr-worker上面的python运行环境，所以如果要使用一些第三方python包需要手动在所有机器上进行安装 diff --git a/docs/source/vector_retrieve.md b/docs/source/vector_retrieve.md index 578521179..8d3f7b909 100644 --- a/docs/source/vector_retrieve.md +++ b/docs/source/vector_retrieve.md @@ -21,19 +21,19 @@ pai -name easy_rec_ext -project algo_public_dev ## 参数说明 -| 参数名 | 默认值 | 参数说明 | -| --------------------- | ------------- | ---------------------------------------------------------------------------------- | -| query_table | 无 | 输入查询表, schema: (id bigint, vector string) | -| doc_table | 无 | 输入索引表, schema: (id bigint, vector string) | -| output_table | 无 | 输出表, schema: (query_id bigint, doc_id bigint, distance double) | +| 参数名 | 默认值 | 参数说明 | +| --------------------- | ------------- | ------------------------------------------------------------------------- | +| query_table | 无 | 输入查询表, schema: (id bigint, vector string) | +| doc_table | 无 | 输入索引表, schema: (id bigint, vector string) | +| output_table | 无 | 输出表, schema: (query_id bigint, doc_id bigint, distance double) | | knn_distance | inner_product | 计算距离的方法：l2、inner_product | -| knn_num_neighbours | 无 | top n, 每个query输出多少个近邻 | -| knn_feature_dims | 无 | 向量维度 | -| knn_feature_delimiter | , | 向量字符串分隔符 | -| knn_index_type | ivfflat | 向量索引类型：'flat', 'ivfflat', 'ivfpq', 'gpu_flat', 'gpu_ivfflat', 'gpu_ivfpg' | -| knn_nlist | 5 | 聚类的簇个数, number of split cluster on each worker | -| knn_nprobe | 2 | 检索时只考虑距离与输入向量最近的簇个数, number of probe part on each worker | -| knn_compress_dim | 8 | 当index_type为`ivfpq` and `gpu_ivfpq`时, 指定压缩的维度，必须为float属性个数的因子 | +| knn_num_neighbours | 无 | top n, 每个query输出多少个近邻 | +| knn_feature_dims | 无 | 向量维度 | +| knn_feature_delimiter | , | 向量字符串分隔符 | +| knn_index_type | ivfflat | 向量索引类型：'flat', 'ivfflat', 'ivfpq', 'gpu_flat', 'gpu_ivfflat', 'gpu_ivfpg' | +| knn_nlist | 5 | 聚类的簇个数, number of split cluster on each worker | +| knn_nprobe | 2 | 检索时只考虑距离与输入向量最近的簇个数, number of probe part on each worker | +| knn_compress_dim | 8 | 当index_type为`ivfpq` and `gpu_ivfpq`时, 指定压缩的维度，必须为float属性个数的因子 | ## 使用示例 diff --git a/easy_rec/__init__.py b/easy_rec/__init__.py index a35666f52..a81a50c0d 100644 --- a/easy_rec/__init__.py +++ b/easy_rec/__init__.py @@ -12,7 +12,8 @@ parent_dir = os.path.dirname(curr_dir) sys.path.insert(0, parent_dir) -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') # Avoid import tensorflow which conflicts with the version used in EasyRecProcessor if 'PROCESSOR_TEST' not in os.environ: @@ -49,8 +50,7 @@ def get_ops_dir(): ops_dir = None from easy_rec.python.inference.predictor import ( # isort:skip # noqa: E402 - Predictor, - ) + Predictor,) from easy_rec.python.main import evaluate # isort:skip # noqa: E402 from easy_rec.python.main import distribute_evaluate # isort:skip # noqa: E402 from easy_rec.python.main import export # isort:skip # noqa: E402 @@ -69,8 +69,7 @@ def get_ops_dir(): def help(): - print( - """ + print(""" 1 Train 1.1 Train 1gpu CUDA_VISIBLE_DEVICES=0 python -m easy_rec.python.train_eval @@ -115,5 +114,4 @@ def help(): for row in reader: inputs.append({ f : row[fid+1] for fid, f in enumerate(field_keys) }) output_res = self._predictor.predict(inputs, batch_size=32) -""" - ) +""") diff --git a/easy_rec/python/builders/hyperparams_builder.py b/easy_rec/python/builders/hyperparams_builder.py index 73af4b54e..e018d94c4 100644 --- a/easy_rec/python/builders/hyperparams_builder.py +++ b/easy_rec/python/builders/hyperparams_builder.py @@ -37,13 +37,15 @@ def build_regularizer(regularizer): """ regularizer_oneof = regularizer.WhichOneof('regularizer_oneof') if regularizer_oneof == 'l1_regularizer': - return regularizers.l1_regularizer(scale=float(regularizer.l1_regularizer.scale)) + return regularizers.l1_regularizer( + scale=float(regularizer.l1_regularizer.scale)) if regularizer_oneof == 'l2_regularizer': - return regularizers.l2_regularizer(scale=float(regularizer.l2_regularizer.scale)) + return regularizers.l2_regularizer( + scale=float(regularizer.l2_regularizer.scale)) if regularizer_oneof == 'l1_l2_regularizer': return regularizers.l1_l2_regularizer( - scale_l1=float(regularizer.l1_l2_regularizer.scale_l1), - scale_l2=float(regularizer.l1_l2_regularizer.scale_l2), + scale_l1=float(regularizer.l1_l2_regularizer.scale_l1), + scale_l2=float(regularizer.l1_l2_regularizer.scale_l2), ) raise ValueError('Unknown regularizer function: {}'.format(regularizer_oneof)) @@ -64,16 +66,17 @@ def build_initializer(initializer): initializer_oneof = initializer.WhichOneof('initializer_oneof') if initializer_oneof == 'truncated_normal_initializer': return tf.truncated_normal_initializer( - mean=initializer.truncated_normal_initializer.mean, - stddev=initializer.truncated_normal_initializer.stddev, + mean=initializer.truncated_normal_initializer.mean, + stddev=initializer.truncated_normal_initializer.stddev, ) if initializer_oneof == 'random_normal_initializer': return tf.random_normal_initializer( - mean=initializer.random_normal_initializer.mean, - stddev=initializer.random_normal_initializer.stddev, + mean=initializer.random_normal_initializer.mean, + stddev=initializer.random_normal_initializer.stddev, ) if initializer_oneof == 'glorot_normal_initializer': return tf.glorot_normal_initializer() if initializer_oneof == 'constant_initializer': - return tf.constant_initializer([x for x in initializer.constant_initializer.consts]) + return tf.constant_initializer( + [x for x in initializer.constant_initializer.consts]) raise ValueError('Unknown initializer function: {}'.format(initializer_oneof)) diff --git a/easy_rec/python/builders/loss_builder.py b/easy_rec/python/builders/loss_builder.py index 7a99a7de0..1914152ca 100644 --- a/easy_rec/python/builders/loss_builder.py +++ b/easy_rec/python/builders/loss_builder.py @@ -5,41 +5,45 @@ import numpy as np import tensorflow as tf -from easy_rec.python.loss.f1_reweight_loss import ( # NOQA - f1_reweight_sigmoid_cross_entropy, -) from easy_rec.python.loss.focal_loss import sigmoid_focal_loss_with_logits from easy_rec.python.loss.jrc_loss import jrc_loss +from easy_rec.python.protos.loss_pb2 import LossType + +from easy_rec.python.loss.f1_reweight_loss import ( # NOQA + f1_reweight_sigmoid_cross_entropy,) from easy_rec.python.loss.listwise_loss import ( # NOQA - listwise_distill_loss, - listwise_rank_loss, + listwise_distill_loss, listwise_rank_loss, ) from easy_rec.python.loss.pairwise_loss import ( # NOQA - pairwise_focal_loss, - pairwise_hinge_loss, - pairwise_logistic_loss, - pairwise_loss, + pairwise_focal_loss, pairwise_hinge_loss, pairwise_logistic_loss, + pairwise_loss, ) from easy_rec.python.loss.zero_inflated_lognormal import ( # NOQA - zero_inflated_lognormal_loss, -) -from easy_rec.python.protos.loss_pb2 import LossType + zero_inflated_lognormal_loss,) if tf.__version__ >= '2.0': tf = tf.compat.v1 -def build(loss_type, label, pred, loss_weight=1.0, num_class=1, loss_param=None, **kwargs): +def build(loss_type, + label, + pred, + loss_weight=1.0, + num_class=1, + loss_param=None, + **kwargs): loss_name = kwargs.pop('loss_name') if 'loss_name' in kwargs else 'unknown' if loss_type == LossType.CLASSIFICATION: if num_class == 1: - return tf.losses.sigmoid_cross_entropy(label, logits=pred, weights=loss_weight, **kwargs) + return tf.losses.sigmoid_cross_entropy( + label, logits=pred, weights=loss_weight, **kwargs) else: assert label.dtype in [ - tf.int32, - tf.int64, + tf.int32, + tf.int64, ], 'label.dtype must in [tf.int32, tf.int64] when use sparse_softmax_cross_entropy.' - return tf.losses.sparse_softmax_cross_entropy(labels=label, logits=pred, weights=loss_weight, **kwargs) + return tf.losses.sparse_softmax_cross_entropy( + labels=label, logits=pred, weights=loss_weight, **kwargs) elif loss_type == LossType.CROSS_ENTROPY_LOSS: return tf.losses.log_loss(label, pred, weights=loss_weight, **kwargs) elif loss_type == LossType.BINARY_CROSS_ENTROPY_LOSS: @@ -47,7 +51,8 @@ def build(loss_type, label, pred, loss_weight=1.0, num_class=1, loss_param=None, return tf.reduce_mean(losses) elif loss_type in [LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS]: logging.info('%s is used' % LossType.Name(loss_type)) - return tf.losses.mean_squared_error(labels=label, predictions=pred, weights=loss_weight, **kwargs) + return tf.losses.mean_squared_error( + labels=label, predictions=pred, weights=loss_weight, **kwargs) elif loss_type == LossType.ZILN_LOSS: if loss_param is None: loss = zero_inflated_lognormal_loss(label, pred) @@ -58,13 +63,13 @@ def build(loss_type, label, pred, loss_weight=1.0, num_class=1, loss_param=None, class_weight = loss_param.classification_weight reg_weight = loss_param.regression_weight loss = zero_inflated_lognormal_loss( - label, - pred, - max_sigma=max_sigma, - mu_reg=mu_reg, - sigma_reg=sigma_reg, - class_weight=class_weight, - reg_weight=reg_weight, + label, + pred, + max_sigma=max_sigma, + mu_reg=mu_reg, + sigma_reg=sigma_reg, + class_weight=class_weight, + reg_weight=reg_weight, ) if np.isscalar(loss_weight) and loss_weight != 1.0: return loss * loss_weight @@ -74,27 +79,27 @@ def build(loss_type, label, pred, loss_weight=1.0, num_class=1, loss_param=None, if loss_param is None: return jrc_loss(label, pred, session, name=loss_name) return jrc_loss( - label, - pred, - session, - loss_param.alpha, - loss_weight_strategy=loss_param.loss_weight_strategy, - sample_weights=loss_weight, - same_label_loss=loss_param.same_label_loss, - name=loss_name, + label, + pred, + session, + loss_param.alpha, + loss_weight_strategy=loss_param.loss_weight_strategy, + sample_weights=loss_weight, + same_label_loss=loss_param.same_label_loss, + name=loss_name, ) elif loss_type == LossType.PAIR_WISE_LOSS: session = kwargs.get('session_ids', None) margin = 0 if loss_param is None else loss_param.margin temp = 1.0 if loss_param is None else loss_param.temperature return pairwise_loss( - label, - pred, - session_ids=session, - margin=margin, - temperature=temp, - weights=loss_weight, - name=loss_name, + label, + pred, + session_ids=session, + margin=margin, + temperature=temp, + weights=loss_weight, + name=loss_name, ) elif loss_type == LossType.PAIRWISE_LOGISTIC_LOSS: session = kwargs.get('session_ids', None) @@ -105,15 +110,15 @@ def build(loss_type, label, pred, loss_weight=1.0, num_class=1, loss_param=None, hinge_margin = loss_param.hinge_margin lbl_margin = False if loss_param is None else loss_param.use_label_margin return pairwise_logistic_loss( - label, - pred, - session_ids=session, - temperature=temp, - hinge_margin=hinge_margin, - ohem_ratio=ohem_ratio, - weights=loss_weight, - use_label_margin=lbl_margin, - name=loss_name, + label, + pred, + session_ids=session, + temperature=temp, + hinge_margin=hinge_margin, + ohem_ratio=ohem_ratio, + weights=loss_weight, + use_label_margin=lbl_margin, + name=loss_name, ) elif loss_type == LossType.PAIRWISE_HINGE_LOSS: session = kwargs.get('session_ids', None) @@ -127,36 +132,37 @@ def build(loss_type, label, pred, loss_weight=1.0, num_class=1, loss_param=None, use_label_margin = loss_param.use_label_margin use_exponent = loss_param.use_exponent return pairwise_hinge_loss( - label, - pred, - session_ids=session, - temperature=temp, - margin=margin, - ohem_ratio=ohem_ratio, - weights=loss_weight, - label_is_logits=label_is_logits, - use_label_margin=use_label_margin, - use_exponent=use_exponent, - name=loss_name, + label, + pred, + session_ids=session, + temperature=temp, + margin=margin, + ohem_ratio=ohem_ratio, + weights=loss_weight, + label_is_logits=label_is_logits, + use_label_margin=use_label_margin, + use_exponent=use_exponent, + name=loss_name, ) elif loss_type == LossType.PAIRWISE_FOCAL_LOSS: session = kwargs.get('session_ids', None) if loss_param is None: - return pairwise_focal_loss(label, pred, session_ids=session, weights=loss_weight, name=loss_name) + return pairwise_focal_loss( + label, pred, session_ids=session, weights=loss_weight, name=loss_name) hinge_margin = None if loss_param.HasField('hinge_margin'): hinge_margin = loss_param.hinge_margin return pairwise_focal_loss( - label, - pred, - session_ids=session, - gamma=loss_param.gamma, - alpha=loss_param.alpha if loss_param.HasField('alpha') else None, - hinge_margin=hinge_margin, - ohem_ratio=loss_param.ohem_ratio, - temperature=loss_param.temperature, - weights=loss_weight, - name=loss_name, + label, + pred, + session_ids=session, + gamma=loss_param.gamma, + alpha=loss_param.alpha if loss_param.HasField('alpha') else None, + hinge_margin=hinge_margin, + ohem_ratio=loss_param.ohem_ratio, + temperature=loss_param.temperature, + weights=loss_weight, + name=loss_name, ) elif loss_type == LossType.LISTWISE_RANK_LOSS: session = kwargs.get('session_ids', None) @@ -168,14 +174,14 @@ def build(loss_type, label, pred, loss_weight=1.0, num_class=1, loss_param=None, if loss_param.HasField('transform_fn'): trans_fn = loss_param.transform_fn return listwise_rank_loss( - label, - pred, - session, - temperature=temp, - label_is_logits=label_is_logits, - transform_fn=trans_fn, - scale_logits=scale, - weights=loss_weight, + label, + pred, + session, + temperature=temp, + label_is_logits=label_is_logits, + transform_fn=trans_fn, + scale_logits=scale, + weights=loss_weight, ) elif loss_type == LossType.LISTWISE_DISTILL_LOSS: session = kwargs.get('session_ids', None) @@ -187,41 +193,42 @@ def build(loss_type, label, pred, loss_weight=1.0, num_class=1, loss_param=None, if loss_param.HasField('transform_fn'): trans_fn = loss_param.transform_fn return listwise_distill_loss( - label, - pred, - session, - temperature=temp, - label_clip_max_value=label_clip_max_value, - transform_fn=trans_fn, - scale_logits=scale, - weights=loss_weight, + label, + pred, + session, + temperature=temp, + label_clip_max_value=label_clip_max_value, + transform_fn=trans_fn, + scale_logits=scale, + weights=loss_weight, ) elif loss_type == LossType.F1_REWEIGHTED_LOSS: f1_beta_square = 1.0 if loss_param is None else loss_param.f1_beta_square label_smoothing = 0 if loss_param is None else loss_param.label_smoothing return f1_reweight_sigmoid_cross_entropy( - label, - pred, - f1_beta_square, - weights=loss_weight, - label_smoothing=label_smoothing, + label, + pred, + f1_beta_square, + weights=loss_weight, + label_smoothing=label_smoothing, ) elif loss_type == LossType.BINARY_FOCAL_LOSS: if loss_param is None: - return sigmoid_focal_loss_with_logits(label, pred, sample_weights=loss_weight, name=loss_name) + return sigmoid_focal_loss_with_logits( + label, pred, sample_weights=loss_weight, name=loss_name) gamma = loss_param.gamma alpha = None if loss_param.HasField('alpha'): alpha = loss_param.alpha return sigmoid_focal_loss_with_logits( - label, - pred, - gamma=gamma, - alpha=alpha, - ohem_ratio=loss_param.ohem_ratio, - sample_weights=loss_weight, - label_smoothing=loss_param.label_smoothing, - name=loss_name, + label, + pred, + gamma=gamma, + alpha=alpha, + ohem_ratio=loss_param.ohem_ratio, + sample_weights=loss_weight, + label_smoothing=loss_param.label_smoothing, + name=loss_name, ) else: raise ValueError('unsupported loss type: %s' % LossType.Name(loss_type)) @@ -242,8 +249,8 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): loss_dict = {} for kd in kds: assert kd.pred_name in prediction_dict, 'invalid predict_name: %s available ones: %s' % ( - kd.pred_name, - ','.join(prediction_dict.keys()), + kd.pred_name, + ','.join(prediction_dict.keys()), ) loss_name = kd.loss_name @@ -252,16 +259,16 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): loss_name += '_' + kd.soft_label_name.replace('/', '_') loss_weight = kd.loss_weight - if kd.HasField('task_space_indicator_name') and kd.HasField('task_space_indicator_value'): + if kd.HasField('task_space_indicator_name') and kd.HasField( + 'task_space_indicator_value'): in_task_space = tf.to_float( - tf.equal( - feature_dict[kd.task_space_indicator_name], - kd.task_space_indicator_value, - ) - ) + tf.equal( + feature_dict[kd.task_space_indicator_name], + kd.task_space_indicator_value, + )) loss_weight = loss_weight * ( - kd.in_task_space_weight * in_task_space + kd.out_task_space_weight * (1 - in_task_space) - ) + kd.in_task_space_weight * in_task_space + kd.out_task_space_weight * + (1 - in_task_space)) label = label_dict[kd.soft_label_name] pred = prediction_dict[kd.pred_name] @@ -328,14 +335,19 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): labels = label preds = pred losses = tf.keras.losses.KLD(labels, preds) - loss_dict[loss_name] = tf.reduce_mean(losses, name=loss_name) * loss_weight + loss_dict[loss_name] = tf.reduce_mean( + losses, name=loss_name) * loss_weight elif kd.loss_type == LossType.BINARY_CROSS_ENTROPY_LOSS: - losses = tf.keras.backend.binary_crossentropy(label, pred, from_logits=True) - loss_dict[loss_name] = tf.reduce_mean(losses, name=loss_name) * loss_weight + losses = tf.keras.backend.binary_crossentropy( + label, pred, from_logits=True) + loss_dict[loss_name] = tf.reduce_mean( + losses, name=loss_name) * loss_weight elif kd.loss_type == LossType.CROSS_ENTROPY_LOSS: - loss_dict[loss_name] = tf.losses.log_loss(label, pred, weights=loss_weight) + loss_dict[loss_name] = tf.losses.log_loss( + label, pred, weights=loss_weight) elif kd.loss_type == LossType.L2_LOSS: - loss_dict[loss_name] = tf.losses.mean_squared_error(labels=label, predictions=pred, weights=loss_weight) + loss_dict[loss_name] = tf.losses.mean_squared_error( + labels=label, predictions=pred, weights=loss_weight) else: loss_param = kd.WhichOneof('loss_param') kwargs = {} @@ -344,11 +356,11 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): if hasattr(loss_param, 'session_name'): kwargs['session_ids'] = feature_dict[loss_param.session_name] loss_dict[loss_name] = build( - kd.loss_type, - label, - pred, - loss_weight=loss_weight, - loss_param=loss_param, - **kwargs, + kd.loss_type, + label, + pred, + loss_weight=loss_weight, + loss_param=loss_param, + **kwargs, ) return loss_dict diff --git a/easy_rec/python/builders/optimizer_builder.py b/easy_rec/python/builders/optimizer_builder.py index 6b1cd57b4..2075e0811 100644 --- a/easy_rec/python/builders/optimizer_builder.py +++ b/easy_rec/python/builders/optimizer_builder.py @@ -47,23 +47,25 @@ def build(optimizer_config): learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) optimizer = tf.train.RMSPropOptimizer( - learning_rate, - decay=config.decay, - momentum=config.momentum_optimizer_value, - epsilon=config.epsilon, + learning_rate, + decay=config.decay, + momentum=config.momentum_optimizer_value, + epsilon=config.epsilon, ) if optimizer_type == 'momentum_optimizer': config = optimizer_config.momentum_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=config.momentum_optimizer_value) + optimizer = tf.train.MomentumOptimizer( + learning_rate, momentum=config.momentum_optimizer_value) if optimizer_type == 'adam_optimizer': config = optimizer_config.adam_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - optimizer = tf.train.AdamOptimizer(learning_rate, beta1=config.beta1, beta2=config.beta2) + optimizer = tf.train.AdamOptimizer( + learning_rate, beta1=config.beta1, beta2=config.beta2) if optimizer_type == 'adamw_optimizer': config = optimizer_config.adamw_optimizer @@ -71,22 +73,23 @@ def build(optimizer_config): summary_vars.append(learning_rate) logging.info('adamw_optimizer weight_decay = %.8f' % config.weight_decay) optimizer = weight_decay_optimizers.AdamWOptimizer( - weight_decay=config.weight_decay, - learning_rate=learning_rate, - beta1=config.beta1, - beta2=config.beta2, + weight_decay=config.weight_decay, + learning_rate=learning_rate, + beta1=config.beta1, + beta2=config.beta2, ) if optimizer_type == 'adam_asyncw_optimizer': config = optimizer_config.adam_asyncw_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - logging.info('adam_asyncw_optimizer weight_decay = %.8f' % config.weight_decay) + logging.info('adam_asyncw_optimizer weight_decay = %.8f' % + config.weight_decay) optimizer = weight_decay_optimizers.AdamAsyncWOptimizer( - weight_decay=config.weight_decay, - learning_rate=learning_rate, - beta1=config.beta1, - beta2=config.beta2, + weight_decay=config.weight_decay, + learning_rate=learning_rate, + beta1=config.beta1, + beta2=config.beta2, ) if optimizer_type == 'lazy_adam_optimizer': @@ -95,49 +98,55 @@ def build(optimizer_config): summary_vars.append(learning_rate) from easy_rec.python.compat.adam_s import AdamOptimizerS - optimizer = AdamOptimizerS(learning_rate=learning_rate, beta1=config.beta1, beta2=config.beta2) + optimizer = AdamOptimizerS( + learning_rate=learning_rate, beta1=config.beta1, beta2=config.beta2) if optimizer_type == 'momentumw_optimizer': config = optimizer_config.momentumw_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - logging.info('momentumw_optimizer weight_decay = %.8f' % config.weight_decay) + logging.info('momentumw_optimizer weight_decay = %.8f' % + config.weight_decay) optimizer = weight_decay_optimizers.MomentumWOptimizer( - weight_decay=config.weight_decay, - learning_rate=learning_rate, - momentum=config.momentum_optimizer_value, + weight_decay=config.weight_decay, + learning_rate=learning_rate, + momentum=config.momentum_optimizer_value, ) if optimizer_type == 'adagrad_optimizer': config = optimizer_config.adagrad_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - optimizer = tf.train.AdagradOptimizer(learning_rate, initial_accumulator_value=config.initial_accumulator_value) + optimizer = tf.train.AdagradOptimizer( + learning_rate, + initial_accumulator_value=config.initial_accumulator_value) if optimizer_type == 'adam_async_optimizer': config = optimizer_config.adam_async_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) - optimizer = tf.train.AdamAsyncOptimizer(learning_rate, beta1=config.beta1, beta2=config.beta2) + optimizer = tf.train.AdamAsyncOptimizer( + learning_rate, beta1=config.beta1, beta2=config.beta2) if optimizer_type == 'ftrl_optimizer': config = optimizer_config.ftrl_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) optimizer = tf.train.FtrlOptimizer( - learning_rate=learning_rate, - learning_rate_power=config.learning_rate_power, - initial_accumulator_value=config.initial_accumulator_value, - l1_regularization_strength=config.l1_reg, - l2_regularization_strength=config.l2_reg, - l2_shrinkage_regularization_strength=config.l2_shrinkage_reg, + learning_rate=learning_rate, + learning_rate_power=config.learning_rate_power, + initial_accumulator_value=config.initial_accumulator_value, + l1_regularization_strength=config.l1_reg, + l2_regularization_strength=config.l2_reg, + l2_shrinkage_regularization_strength=config.l2_shrinkage_reg, ) if optimizer is None: raise ValueError('Optimizer %s not supported.' % optimizer_type) if optimizer_config.use_moving_average: - optimizer = tf.contrib.opt.MovingAverageOptimizer(optimizer, average_decay=optimizer_config.moving_average_decay) + optimizer = tf.contrib.opt.MovingAverageOptimizer( + optimizer, average_decay=optimizer_config.moving_average_decay) return optimizer, summary_vars @@ -158,19 +167,20 @@ def _create_learning_rate(learning_rate_config): learning_rate_type = learning_rate_config.WhichOneof('learning_rate') if learning_rate_type == 'constant_learning_rate': config = learning_rate_config.constant_learning_rate - learning_rate = tf.constant(config.learning_rate, dtype=tf.float32, name='learning_rate') + learning_rate = tf.constant( + config.learning_rate, dtype=tf.float32, name='learning_rate') if learning_rate_type == 'exponential_decay_learning_rate': config = learning_rate_config.exponential_decay_learning_rate learning_rate = learning_schedules.exponential_decay_with_burnin( - tf.train.get_or_create_global_step(), - config.initial_learning_rate, - config.decay_steps, - config.decay_factor, - burnin_learning_rate=config.burnin_learning_rate, - burnin_steps=config.burnin_steps, - min_learning_rate=config.min_learning_rate, - staircase=config.staircase, + tf.train.get_or_create_global_step(), + config.initial_learning_rate, + config.decay_steps, + config.decay_factor, + burnin_learning_rate=config.burnin_learning_rate, + burnin_steps=config.burnin_steps, + min_learning_rate=config.min_learning_rate, + staircase=config.staircase, ) if learning_rate_type == 'manual_step_learning_rate': @@ -181,41 +191,41 @@ def _create_learning_rate(learning_rate_config): learning_rate_sequence = [config.initial_learning_rate] learning_rate_sequence += [x.learning_rate for x in config.schedule] learning_rate = learning_schedules.manual_stepping( - tf.train.get_or_create_global_step(), - learning_rate_step_boundaries, - learning_rate_sequence, - config.warmup, + tf.train.get_or_create_global_step(), + learning_rate_step_boundaries, + learning_rate_sequence, + config.warmup, ) if learning_rate_type == 'cosine_decay_learning_rate': config = learning_rate_config.cosine_decay_learning_rate learning_rate = learning_schedules.cosine_decay_with_warmup( - tf.train.get_or_create_global_step(), - config.learning_rate_base, - config.total_steps, - config.warmup_learning_rate, - config.warmup_steps, - config.hold_base_rate_steps, + tf.train.get_or_create_global_step(), + config.learning_rate_base, + config.total_steps, + config.warmup_learning_rate, + config.warmup_steps, + config.hold_base_rate_steps, ) if learning_rate_type == 'poly_decay_learning_rate': config = learning_rate_config.poly_decay_learning_rate learning_rate = tf.train.polynomial_decay( - config.learning_rate_base, - tf.train.get_or_create_global_step(), - config.total_steps, - config.end_learning_rate, - config.power, + config.learning_rate_base, + tf.train.get_or_create_global_step(), + config.total_steps, + config.end_learning_rate, + config.power, ) if learning_rate_type == 'transformer_learning_rate': config = learning_rate_config.transformer_learning_rate learning_rate = learning_schedules.transformer_policy( - tf.train.get_or_create_global_step(), - config.learning_rate_base, - config.hidden_size, - config.warmup_steps, - config.step_scaling_rate, + tf.train.get_or_create_global_step(), + config.learning_rate_base, + config.hidden_size, + config.warmup_steps, + config.step_scaling_rate, ) if learning_rate is None: diff --git a/easy_rec/python/builders/strategy_builder.py b/easy_rec/python/builders/strategy_builder.py index 2524aa46a..ef106423d 100644 --- a/easy_rec/python/builders/strategy_builder.py +++ b/easy_rec/python/builders/strategy_builder.py @@ -2,7 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.protos.train_pb2 import DistributionStrategy, TrainConfig +from easy_rec.python.protos.train_pb2 import DistributionStrategy +from easy_rec.python.protos.train_pb2 import TrainConfig def build(train_config): @@ -25,18 +26,17 @@ def build(train_config): import pai distribution = pai.distribute.ExascaleStrategy( - max_splits=10, - issorted=True, - optimize_clip_by_global_norm=False, - enable_sparse_allreduce=False, - enable_hierarchical_allreduce=True, + max_splits=10, + issorted=True, + optimize_clip_by_global_norm=False, + enable_sparse_allreduce=False, + enable_hierarchical_allreduce=True, ) # the older version of MultiWorkerMirroredStrategy # works under tf1.12 to tf1.15 elif train_config.train_distribute == DistributionStrategy.CollectiveAllReduceStrategy: distribution = tf.contrib.distribute.CollectiveAllReduceStrategy( - num_gpus_per_worker=train_config.num_gpus_per_worker - ) + num_gpus_per_worker=train_config.num_gpus_per_worker) # works under tf1.15 and tf2.x elif train_config.train_distribute == DistributionStrategy.PSStrategy: if tf.__version__ <= '1.15': diff --git a/easy_rec/python/compat/adam_s.py b/easy_rec/python/compat/adam_s.py index 6afeb59bd..909ea1bdd 100644 --- a/easy_rec/python/compat/adam_s.py +++ b/easy_rec/python/compat/adam_s.py @@ -14,18 +14,18 @@ # ============================================================================== """Adam for TensorFlow.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function from tensorflow.python.eager import context from tensorflow.python.framework import ops +from tensorflow.python.training import optimizer +from tensorflow.python.training import training_ops + from tensorflow.python.ops import ( # NOQA - array_ops, - control_flow_ops, - math_ops, - resource_variable_ops, - state_ops, + array_ops, control_flow_ops, math_ops, resource_variable_ops, state_ops, ) -from tensorflow.python.training import optimizer, training_ops class AdamOptimizerS(optimizer.Optimizer): @@ -38,13 +38,13 @@ class AdamOptimizerS(optimizer.Optimizer): """ def __init__( - self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - use_locking=False, - name='Adam', + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + use_locking=False, + name='Adam', ): r"""Construct a new Adam optimizer. @@ -121,8 +121,8 @@ def _get_beta_accumulators(self): else: graph = ops.get_default_graph() return ( - self._get_non_slot_variable('beta1_power', graph=graph), - self._get_non_slot_variable('beta2_power', graph=graph), + self._get_non_slot_variable('beta1_power', graph=graph), + self._get_non_slot_variable('beta2_power', graph=graph), ) def _create_slots(self, var_list): @@ -131,8 +131,10 @@ def _create_slots(self, var_list): # workers (these need to go on the same PS, otherwise some updates are # silently ignored). first_var = min(var_list, key=lambda x: x.name) - self._create_non_slot_variable(initial_value=self._beta1, name='beta1_power', colocate_with=first_var) - self._create_non_slot_variable(initial_value=self._beta2, name='beta2_power', colocate_with=first_var) + self._create_non_slot_variable( + initial_value=self._beta1, name='beta1_power', colocate_with=first_var) + self._create_non_slot_variable( + initial_value=self._beta2, name='beta2_power', colocate_with=first_var) # Create slots for the first and second moments. for v in var_list: @@ -155,17 +157,17 @@ def _apply_dense(self, grad, var): v = self.get_slot(var, 'v') beta1_power, beta2_power = self._get_beta_accumulators() return training_ops.apply_adam( - var, - m, - v, - math_ops.cast(beta1_power, var.dtype.base_dtype), - math_ops.cast(beta2_power, var.dtype.base_dtype), - math_ops.cast(self._lr_t, var.dtype.base_dtype), - math_ops.cast(self._beta1_t, var.dtype.base_dtype), - math_ops.cast(self._beta2_t, var.dtype.base_dtype), - math_ops.cast(self._epsilon_t, var.dtype.base_dtype), - grad, - use_locking=self._use_locking, + var, + m, + v, + math_ops.cast(beta1_power, var.dtype.base_dtype), + math_ops.cast(beta2_power, var.dtype.base_dtype), + math_ops.cast(self._lr_t, var.dtype.base_dtype), + math_ops.cast(self._beta1_t, var.dtype.base_dtype), + math_ops.cast(self._beta2_t, var.dtype.base_dtype), + math_ops.cast(self._epsilon_t, var.dtype.base_dtype), + grad, + use_locking=self._use_locking, ).op def _resource_apply_dense(self, grad, var): @@ -173,17 +175,17 @@ def _resource_apply_dense(self, grad, var): v = self.get_slot(var, 'v') beta1_power, beta2_power = self._get_beta_accumulators() return training_ops.resource_apply_adam( - var.handle, - m.handle, - v.handle, - math_ops.cast(beta1_power, grad.dtype.base_dtype), - math_ops.cast(beta2_power, grad.dtype.base_dtype), - math_ops.cast(self._lr_t, grad.dtype.base_dtype), - math_ops.cast(self._beta1_t, grad.dtype.base_dtype), - math_ops.cast(self._beta2_t, grad.dtype.base_dtype), - math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), - grad, - use_locking=self._use_locking, + var.handle, + m.handle, + v.handle, + math_ops.cast(beta1_power, grad.dtype.base_dtype), + math_ops.cast(beta2_power, grad.dtype.base_dtype), + math_ops.cast(self._lr_t, grad.dtype.base_dtype), + math_ops.cast(self._beta1_t, grad.dtype.base_dtype), + math_ops.cast(self._beta2_t, grad.dtype.base_dtype), + math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), + grad, + use_locking=self._use_locking, ) def _apply_sparse_shared(self, grad, var, indices, scatter_add): @@ -212,31 +214,39 @@ def _apply_sparse_shared(self, grad, var, indices, scatter_add): # var_update = state_ops.assign_sub( # var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking) v_part_sqrt = math_ops.sqrt(v_part_n) - var_update = scatter_add(var, indices, -lr * m_part_n / (v_part_sqrt + epsilon_t)) + var_update = scatter_add(var, indices, + -lr * m_part_n / (v_part_sqrt + epsilon_t)) return control_flow_ops.group(*[var_update, m_t, v_t]) def _apply_sparse(self, grad, var): return self._apply_sparse_shared( - grad.values, - var, - grad.indices, - lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda - x, i, v, use_locking=self._use_locking - ), + grad.values, + var, + grad.indices, + lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda + x, + i, + v, + use_locking=self._use_locking), ) def _resource_scatter_add(self, x, i, v): - with ops.control_dependencies([resource_variable_ops.resource_scatter_add(x.handle, i, v)]): + with ops.control_dependencies( + [resource_variable_ops.resource_scatter_add(x.handle, i, v)]): return x.value() def _resource_apply_sparse(self, grad, var, indices): - return self._apply_sparse_shared(grad, var, indices, self._resource_scatter_add) + return self._apply_sparse_shared(grad, var, indices, + self._resource_scatter_add) def _finish(self, update_ops, name_scope): # Update the power accumulators. with ops.control_dependencies(update_ops): beta1_power, beta2_power = self._get_beta_accumulators() with ops.colocate_with(beta1_power): - update_beta1 = beta1_power.assign(beta1_power * self._beta1_t, use_locking=self._use_locking) - update_beta2 = beta2_power.assign(beta2_power * self._beta2_t, use_locking=self._use_locking) - return control_flow_ops.group(*update_ops + [update_beta1, update_beta2], name=name_scope) + update_beta1 = beta1_power.assign( + beta1_power * self._beta1_t, use_locking=self._use_locking) + update_beta2 = beta2_power.assign( + beta2_power * self._beta2_t, use_locking=self._use_locking) + return control_flow_ops.group( + *update_ops + [update_beta1, update_beta2], name=name_scope) diff --git a/easy_rec/python/compat/array_ops.py b/easy_rec/python/compat/array_ops.py index 4b847ba05..ea8bf5102 100644 --- a/easy_rec/python/compat/array_ops.py +++ b/easy_rec/python/compat/array_ops.py @@ -1,6 +1,8 @@ import numpy as np import tensorflow as tf -from tensorflow.python.framework import constant_op, ops, sparse_tensor +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import ops +from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import gen_math_ops @@ -10,7 +12,8 @@ def convert_to_int_tensor(tensor, name, dtype=tf.int32): if tensor.dtype.is_integer: tensor = gen_math_ops.cast(tensor, dtype) else: - raise TypeError('%s must be an integer tensor; dtype=%s' % (name, tensor.dtype)) + raise TypeError('%s must be an integer tensor; dtype=%s' % + (name, tensor.dtype)) return tensor @@ -55,7 +58,8 @@ def get_positive_axis(axis, ndims): elif -ndims <= axis < 0: return axis + ndims else: - raise ValueError('axis=%s out of bounds: expected %s<=axis<%s' % (axis, -ndims, ndims)) + raise ValueError('axis=%s out of bounds: expected %s<=axis<%s' % + (axis, -ndims, ndims)) elif axis < 0: raise ValueError('axis may only be negative if ndims is statically known.') return axis @@ -69,7 +73,8 @@ def tile_one_dimension(data, axis, multiple): multiples[axis] = multiple else: ones_value = tf.ones(tf.rank(data), tf.int32) - multiples = tf.concat([ones_value[:axis], [multiple], ones_value[axis + 1 :]], axis=0) + multiples = tf.concat( + [ones_value[:axis], [multiple], ones_value[axis + 1:]], axis=0) return tf.tile(data, multiples) @@ -78,7 +83,8 @@ def _all_dimensions(x): # Fast path: avoid creating Rank and Range ops if ndims is known. if isinstance(x, ops.Tensor) and x.get_shape().ndims is not None: return constant_op.constant(np.arange(x.get_shape().ndims), dtype=tf.int32) - if isinstance(x, sparse_tensor.SparseTensor) and x.dense_shape.get_shape().is_fully_defined(): + if isinstance(x, sparse_tensor.SparseTensor) and x.dense_shape.get_shape( + ).is_fully_defined(): r = x.dense_shape.get_shape().dims[0].value # sparse.dense_shape is 1-D. return constant_op.constant(np.arange(r), dtype=tf.int32) @@ -141,21 +147,24 @@ def repeat_with_axis(data, repeats, axis, name=None): if repeats.shape.ndims == 0: expanded = tf.expand_dims(data, axis + 1) tiled = tile_one_dimension(expanded, axis + 1, repeats) - result_shape = tf.concat([data_shape[:axis], [-1], data_shape[axis + 1 :]], axis=0) + result_shape = tf.concat([data_shape[:axis], [-1], data_shape[axis + 1:]], + axis=0) return tf.reshape(tiled, result_shape) # Broadcast the `repeats` tensor so rank(repeats) == axis + 1. if repeats.shape.ndims != axis + 1: repeats_shape = tf.shape(repeats) repeats_ndims = tf.rank(repeats) - broadcast_shape = tf.concat([data_shape[: axis + 1 - repeats_ndims], repeats_shape], axis=0) + broadcast_shape = tf.concat( + [data_shape[:axis + 1 - repeats_ndims], repeats_shape], axis=0) repeats = tf.broadcast_to(repeats, broadcast_shape) repeats.set_shape([None] * (axis + 1)) # Create a "sequence mask" based on `repeats`, where slices across `axis` # contain one `True` value for each repetition. E.g., if # `repeats = [3, 1, 2]`, then `mask = [[1, 1, 1], [1, 0, 0], [1, 1, 0]]`. - max_repeat = gen_math_ops.maximum(0, gen_math_ops._max(repeats, _all_dimensions(repeats))) + max_repeat = gen_math_ops.maximum( + 0, gen_math_ops._max(repeats, _all_dimensions(repeats))) mask = tf.sequence_mask(repeats, max_repeat) # Add a new dimension around each value that needs to be repeated, and @@ -171,13 +180,15 @@ def repeat_with_axis(data, repeats, axis, name=None): if axis == 0: result = masked else: - result_shape = tf.concat([data_shape[:axis], [-1], data_shape[axis + 1 :]], axis=0) + result_shape = tf.concat([data_shape[:axis], [-1], data_shape[axis + 1:]], + axis=0) result = tf.reshape(masked, result_shape) # Preserve shape information. if data.shape.ndims is not None: new_axis_size = 0 if repeats.shape[0] == 0 else None - result.set_shape(data.shape[:axis].concatenate([new_axis_size]).concatenate(data.shape[axis + 1 :])) + result.set_shape(data.shape[:axis].concatenate( + [new_axis_size]).concatenate(data.shape[axis + 1:])) return result diff --git a/easy_rec/python/compat/dynamic_variable.py b/easy_rec/python/compat/dynamic_variable.py index 04fea08bb..393ed7f52 100644 --- a/easy_rec/python/compat/dynamic_variable.py +++ b/easy_rec/python/compat/dynamic_variable.py @@ -21,12 +21,11 @@ from sparse_operation_kit.experiment.communication import num_gpus from tensorflow.python.eager import context from tensorflow.python.framework import ops - # from tensorflow.python.ops import array_ops from tensorflow.python.ops import resource_variable_ops + from tensorflow.python.ops.resource_variable_ops import ( # NOQA - ResourceVariable, - variable_accessed, + ResourceVariable, variable_accessed, ) # from tensorflow.python.util import object_identity @@ -88,27 +87,29 @@ class DynamicVariable(ResourceVariable): """ def __init__( - self, - dimension, - initializer=None, - var_type=None, - name=None, - constraint=None, - trainable=True, - key_type=None, - dtype=None, - mode=None, - variable_def=None, - import_scope=None, - **kwargs, + self, + dimension, + initializer=None, + var_type=None, + name=None, + constraint=None, + trainable=True, + key_type=None, + dtype=None, + mode=None, + variable_def=None, + import_scope=None, + **kwargs, ): self._indices = None if variable_def is not None: - super(DynamicVariable, self)._init_from_proto(variable_def, import_scope=import_scope, validate_shape=False) + super(DynamicVariable, self)._init_from_proto( + variable_def, import_scope=import_scope, validate_shape=False) g = ops.get_default_graph() handle = g.as_graph_element( - ops.prepend_name_scope(variable_def.variable_name, import_scope=import_scope), - allow_operation=False, + ops.prepend_name_scope( + variable_def.variable_name, import_scope=import_scope), + allow_operation=False, ) self._dimension = handle.op.get_attr('shape').dim[-1].size self._key_type = handle.op.get_attr('key_type') @@ -137,7 +138,8 @@ def __init__( self._config = json.dumps(kwargs) self._config_dict = kwargs if var_type == 'hybrid' and self._key_type != tf.int64: - raise NotImplementedError('only key_type tf.int64 is supported in HKV backend') + raise NotImplementedError( + 'only key_type tf.int64 is supported in HKV backend') if name is None: global dynamic_variable_count name = 'sok_dynamic_Variable_' + str(dynamic_variable_count) @@ -146,15 +148,15 @@ def __init__( self._var_type = var_type self._base = super(DynamicVariable, self) self._base.__init__( - initial_value=[[0.0] * dimension], - trainable=trainable, - name=name + '/proxy', - dtype=self._handle_dtype, - constraint=constraint, - distribute_strategy=None, - synchronization=None, - aggregation=None, - shape=[None, dimension], + initial_value=[[0.0] * dimension], + trainable=trainable, + name=name + '/proxy', + dtype=self._handle_dtype, + constraint=constraint, + distribute_strategy=None, + synchronization=None, + aggregation=None, + shape=[None, dimension], ) with ops.init_scope(): @@ -168,33 +170,33 @@ def __init__( initializer = '' if initializer is None else initializer self._initializer = initializer handle = dynamic_variable_ops.dummy_var_handle( - container='DummyVariableContainer', - shared_name=self._dummy_name, - key_type=self._key_type, - dtype=self._handle_dtype, - shape=shape, + container='DummyVariableContainer', + shared_name=self._dummy_name, + key_type=self._key_type, + dtype=self._handle_dtype, + shape=shape, ) if type(initializer) is str: init_op = dynamic_variable_ops.dummy_var_initialize( - handle, - initializer=initializer, - var_type=var_type, - unique_name=self._dummy_name, - key_type=self._key_type, - dtype=self._handle_dtype, - config=self._config, + handle, + initializer=initializer, + var_type=var_type, + unique_name=self._dummy_name, + key_type=self._key_type, + dtype=self._handle_dtype, + config=self._config, ) else: with tf.control_dependencies([initializer._initializer_op]): initial_val = initializer.read_value() init_op = dynamic_variable_ops.dummy_var_initialize( - handle, - initializer=initial_val, - var_type=var_type, - unique_name=self._dummy_name, - key_type=self._key_type, - dtype=self._handle_dtype, - config=self._config, + handle, + initializer=initial_val, + var_type=var_type, + unique_name=self._dummy_name, + key_type=self._key_type, + dtype=self._handle_dtype, + config=self._config, ) # TODO: Add is_initialized_op # is_initialized_op = ops.convert_to_tensor(True) @@ -206,17 +208,17 @@ def __init__( self._initializer_op = tf.group([self._initializer_op, init_op]) # self._is_initialized_op = tf.group([self._is_initialized_op, is_initialized_op]) - handle_data = resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData() + handle_data = resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData( + ) handle_data.is_set = True handle_data.shape_and_type.append( - resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult.HandleShapeAndType( - shape=self.shape.as_proto(), dtype=self.dtype.as_datatype_enum - ) - ) + resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult + .HandleShapeAndType( + shape=self.shape.as_proto(), dtype=self.dtype.as_datatype_enum)) resource_variable_ops._set_handle_shapes_and_types( - self._handle, - handle_data, - graph_mode=False if context.executing_eagerly() else True, + self._handle, + handle_data, + graph_mode=False if context.executing_eagerly() else True, ) def is_static(self): @@ -249,14 +251,15 @@ def __repr__(self): if self.is_static(): return self._base.__repr__() return "" % ( - self._dummy_name, - self.shape, - self.dtype.name, + self._dummy_name, + self.shape, + self.dtype.name, ) @property def size(self): - return dynamic_variable_ops.dummy_var_shape(self._dummy_handle, key_type=self._key_type, dtype=self._handle_dtype) + return dynamic_variable_ops.dummy_var_shape( + self._dummy_handle, key_type=self._key_type, dtype=self._handle_dtype) @property def indices(self): @@ -312,10 +315,10 @@ def sparse_read(self, indices, name=None, lookup_only=False): if indices.dtype == tf.int32: indices = tf.cast(indices, tf.int64) return dynamic_variable_ops.dummy_var_sparse_read( - self._dummy_handle, - indices, - dtype=self._handle_dtype, - lookup_only=lookup_only, + self._dummy_handle, + indices, + dtype=self._handle_dtype, + lookup_only=lookup_only, ) def scatter_sub(self, sparse_delta, use_locking=False, name=None): @@ -324,9 +327,9 @@ def scatter_sub(self, sparse_delta, use_locking=False, name=None): if not isinstance(sparse_delta, ops.IndexedSlices): raise TypeError('sparse_delta is not IndexedSlices: %s' % sparse_delta) return dynamic_variable_ops.dummy_var_scatter_add( - self._dummy_handle, - sparse_delta.indices, - ops.convert_to_tensor(-sparse_delta.values, self.dtype), + self._dummy_handle, + sparse_delta.indices, + ops.convert_to_tensor(-sparse_delta.values, self.dtype), ) def scatter_add(self, sparse_delta, use_locking=False, name=None): @@ -335,9 +338,9 @@ def scatter_add(self, sparse_delta, use_locking=False, name=None): if not isinstance(sparse_delta, ops.IndexedSlices): raise TypeError('sparse_delta is not IndexedSlices: %s' % sparse_delta) return dynamic_variable_ops.dummy_var_scatter_add( - self._dummy_handle, - sparse_delta.indices, - ops.convert_to_tensor(sparse_delta.values, self.dtype), + self._dummy_handle, + sparse_delta.indices, + ops.convert_to_tensor(sparse_delta.values, self.dtype), ) def scatter_update(self, sparse_delta, use_locking=False, name=None): @@ -346,9 +349,9 @@ def scatter_update(self, sparse_delta, use_locking=False, name=None): if not isinstance(sparse_delta, ops.IndexedSlices): raise TypeError('sparse_delta is not IndexedSlices: %s' % sparse_delta) return dynamic_variable_ops.dummy_var_scatter_update( - self._dummy_handle, - sparse_delta.indices, - ops.convert_to_tensor(sparse_delta.values, self.dtype), + self._dummy_handle, + sparse_delta.indices, + ops.convert_to_tensor(sparse_delta.values, self.dtype), ) # ------------------------------------------------------------------------- @@ -368,7 +371,8 @@ def to_proto(self, *args, **kwargs): @staticmethod def from_proto(variable_def, import_scope=None): if '/DummyVarHandle' in variable_def.variable_name: - return DynamicVariable(dimension=0, variable_def=variable_def, import_scope=import_scope) + return DynamicVariable( + dimension=0, variable_def=variable_def, import_scope=import_scope) else: return _resource_var_from_proto(variable_def, import_scope) # raise NotImplementedError("from_proto() is not supported.") @@ -384,12 +388,14 @@ def is_initialized(self, name): return True if self.is_static(): return self._base.is_initialized(name) - raise NotImplementedError('is_initialized() is not supported in dynamic mode.') + raise NotImplementedError( + 'is_initialized() is not supported in dynamic mode.') def _read_variable_op(self): if self.is_static(): return self._base._read_variable_op() - raise NotImplementedError('_read_variable_op() is not supported in dynamic mode.') + raise NotImplementedError( + '_read_variable_op() is not supported in dynamic mode.') def value(self): if self.is_static(): @@ -399,12 +405,14 @@ def value(self): def _dense_var_to_tensor(self, *args, **kwargs): if self.is_static(): return self._base._dense_var_to_tensor(*args, **kwargs) - raise NotImplementedError('_dense_var_to_tensor() is not supported in dynamic mode.') + raise NotImplementedError( + '_dense_var_to_tensor() is not supported in dynamic mode.') def _gather_saveables_for_checkpoint(self): if self.is_static(): return self._base._gather_saveables_for_checkpoint() - raise NotImplementedError('_gather_saveables_for_checkpoint() is not supported in dynamic mode.') + raise NotImplementedError( + '_gather_saveables_for_checkpoint() is not supported in dynamic mode.') def gather_nd(self, *args, **kwargs): if self.is_static(): @@ -444,22 +452,26 @@ def scatter_dim(self, *args, **kwargs): def batch_scatter_update(self, *args, **kwargs): if self.is_static(): return self._base.batch_scatter_update(*args, **kwargs) - raise NotImplementedError('batch_scatter_update() is not supported in dynamic mode.') + raise NotImplementedError( + 'batch_scatter_update() is not supported in dynamic mode.') def scatter_nd_sub(self, *args, **kwargs): if self.is_static(): return self._base.scatter_nd_sub(*args, **kwargs) - raise NotImplementedError('scatter_nd_sub() is not supported in dynamic mode.') + raise NotImplementedError( + 'scatter_nd_sub() is not supported in dynamic mode.') def scatter_nd_update(self, *args, **kwargs): if self.is_static(): return self._base.scatter_nd_update(*args, **kwargs) - raise NotImplementedError('scatter_nd_update() is not supported in dynamic mode.') + raise NotImplementedError( + 'scatter_nd_update() is not supported in dynamic mode.') def _strided_slice_assign(self, *args, **kwargs): if self.is_static(): return self._base._strided_slice_assign(*args, **kwargs) - raise NotImplementedError('_strided_slice_assign() is not supported in dynamic mode.') + raise NotImplementedError( + '_strided_slice_assign() is not supported in dynamic mode.') def __int__(self, *args, **kwargs): if self.is_static(): @@ -503,7 +515,8 @@ def export(var): the values of the given variable. """ if isinstance(var, DynamicVariable): - indices, values = dynamic_variable_ops.dummy_var_export(var.handle, key_type=var.key_type, dtype=var.handle_dtype) + indices, values = dynamic_variable_ops.dummy_var_export( + var.handle, key_type=var.key_type, dtype=var.handle_dtype) with tf.device('CPU'): indices = tf.identity(indices) values = tf.identity(values) diff --git a/easy_rec/python/compat/early_stopping.py b/easy_rec/python/compat/early_stopping.py index a977e1843..573a898c6 100644 --- a/easy_rec/python/compat/early_stopping.py +++ b/easy_rec/python/compat/early_stopping.py @@ -21,26 +21,28 @@ import os import threading import time -from distutils.version import LooseVersion import tensorflow as tf -from tensorflow.python.framework import dtypes, ops -from tensorflow.python.ops import init_ops, state_ops, variable_scope -from tensorflow.python.platform import gfile, tf_logging +from distutils.version import LooseVersion +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import state_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.platform import gfile +from tensorflow.python.platform import tf_logging from tensorflow.python.summary import summary_iterator -from tensorflow.python.training import ( # NOQA - basic_session_run_hooks, - session_run_hook, - training_util, -) from easy_rec.python.utils.config_util import parse_time from easy_rec.python.utils.load_class import load_by_path +from tensorflow.python.training import ( # NOQA + basic_session_run_hooks, session_run_hook, training_util, +) + if LooseVersion(tf.__version__) >= LooseVersion('2.12.0'): from tensorflow_estimator.python.estimator.estimator_export import ( # NOQA - estimator_export, - ) + estimator_export,) else: from tensorflow.python.util.tf_export import estimator_export @@ -59,7 +61,10 @@ def find_early_stop_var(var_list): @estimator_export('estimator.experimental.make_early_stopping_hook') -def make_early_stopping_hook(estimator, should_stop_fn, run_every_secs=60, run_every_steps=None): +def make_early_stopping_hook(estimator, + should_stop_fn, + run_every_secs=60, + run_every_steps=None): """Creates early-stopping hook. Returns a `SessionRunHook` that stops training when `should_stop_fn` returns `True`. Usage example: @@ -98,7 +103,8 @@ def make_early_stopping_hook(estimator, should_stop_fn, run_every_secs=60, run_e ValueError: If both `run_every_secs` and `run_every_steps` are set. """ if run_every_secs is not None and run_every_steps is not None: - raise ValueError('Only one of `run_every_secs` and `run_every_steps` must ' 'be set.') + raise ValueError('Only one of `run_every_secs` and `run_every_steps` must ' + 'be set.') if estimator.config.is_chief: return _StopOnPredicateHook(should_stop_fn, run_every_secs, run_every_steps) @@ -108,13 +114,13 @@ def make_early_stopping_hook(estimator, should_stop_fn, run_every_secs=60, run_e @estimator_export('estimator.experimental.stop_if_higher_hook') def stop_if_higher_hook( - estimator, - metric_name, - threshold, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None, + estimator, + metric_name, + threshold, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None, ): """Creates hook to stop if the given metric is higher than the threshold. @@ -155,26 +161,26 @@ def stop_if_higher_hook( early stopping if true. """ return _stop_if_threshold_crossed_hook( - estimator=estimator, - metric_name=metric_name, - threshold=threshold, - higher_is_better=True, - eval_dir=eval_dir, - min_steps=min_steps, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps, + estimator=estimator, + metric_name=metric_name, + threshold=threshold, + higher_is_better=True, + eval_dir=eval_dir, + min_steps=min_steps, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps, ) @estimator_export('estimator.experimental.stop_if_lower_hook') def stop_if_lower_hook( - estimator, - metric_name, - threshold, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None, + estimator, + metric_name, + threshold, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None, ): """Creates hook to stop if the given metric is lower than the threshold. @@ -215,26 +221,26 @@ def stop_if_lower_hook( early stopping if true. """ return _stop_if_threshold_crossed_hook( - estimator=estimator, - metric_name=metric_name, - threshold=threshold, - higher_is_better=False, - eval_dir=eval_dir, - min_steps=min_steps, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps, + estimator=estimator, + metric_name=metric_name, + threshold=threshold, + higher_is_better=False, + eval_dir=eval_dir, + min_steps=min_steps, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps, ) @estimator_export('estimator.experimental.stop_if_no_increase_hook') def stop_if_no_increase_hook( - estimator, - metric_name, - max_steps_without_increase, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None, + estimator, + metric_name, + max_steps_without_increase, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None, ): """Creates hook to stop if metric does not increase within given max steps. @@ -276,24 +282,24 @@ def stop_if_no_increase_hook( training steps, and initiates early stopping if true. """ return _stop_if_no_metric_improvement_hook( - estimator=estimator, - metric_name=metric_name, - max_steps_without_improvement=max_steps_without_increase, - higher_is_better=True, - eval_dir=eval_dir, - min_steps=min_steps, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps, + estimator=estimator, + metric_name=metric_name, + max_steps_without_improvement=max_steps_without_increase, + higher_is_better=True, + eval_dir=eval_dir, + min_steps=min_steps, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps, ) def custom_early_stop_hook( - estimator, - eval_dir, - custom_stop_func, - custom_stop_func_params, - run_every_secs=60, - run_every_steps=None, + estimator, + eval_dir, + custom_stop_func, + custom_stop_func_params, + run_every_secs=60, + run_every_steps=None, ): """Custom early stop hook for complex early stop conditions. @@ -317,7 +323,8 @@ def custom_early_stop_hook( if eval_dir is None: eval_dir = estimator.eval_dir() - if isinstance(custom_stop_func, str) or isinstance(custom_stop_func, type('')): + if isinstance(custom_stop_func, str) or isinstance(custom_stop_func, + type('')): custom_stop_func = load_by_path(custom_stop_func) def _custom_stop_fn(): @@ -325,22 +332,22 @@ def _custom_stop_fn(): return custom_stop_func(eval_results, custom_stop_func_params) return make_early_stopping_hook( - estimator=estimator, - should_stop_fn=_custom_stop_fn, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps, + estimator=estimator, + should_stop_fn=_custom_stop_fn, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps, ) @estimator_export('estimator.experimental.stop_if_no_decrease_hook') def stop_if_no_decrease_hook( - estimator, - metric_name, - max_steps_without_decrease, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None, + estimator, + metric_name, + max_steps_without_decrease, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None, ): """Creates hook to stop if metric does not decrease within given max steps. @@ -382,14 +389,14 @@ def stop_if_no_decrease_hook( training steps, and initiates early stopping if true. """ return _stop_if_no_metric_improvement_hook( - estimator=estimator, - metric_name=metric_name, - max_steps_without_improvement=max_steps_without_decrease, - higher_is_better=False, - eval_dir=eval_dir, - min_steps=min_steps, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps, + estimator=estimator, + metric_name=metric_name, + max_steps_without_improvement=max_steps_without_decrease, + higher_is_better=False, + eval_dir=eval_dir, + min_steps=min_steps, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps, ) @@ -412,18 +419,19 @@ def read_eval_metrics(eval_dir): metrics[value.tag] = value.simple_value if metrics: eval_metrics_dict[event.step].update(metrics) - return collections.OrderedDict(sorted(eval_metrics_dict.items(), key=lambda t: t[0])) + return collections.OrderedDict( + sorted(eval_metrics_dict.items(), key=lambda t: t[0])) def _stop_if_threshold_crossed_hook( - estimator, - metric_name, - threshold, - higher_is_better, - eval_dir, - min_steps, - run_every_secs, - run_every_steps, + estimator, + metric_name, + threshold, + higher_is_better, + eval_dir, + min_steps, + run_every_secs, + run_every_steps, ): """Creates early-stopping hook to stop training if threshold is crossed.""" if eval_dir is None: @@ -442,33 +450,34 @@ def stop_if_threshold_crossed_fn(): val = metrics[metric_name] if is_lhs_better(val, threshold): tf_logging.info( - 'At step %s, metric "%s" has value %s which is %s the configured ' 'threshold (%s) for early stopping.', - step, - metric_name, - val, - greater_or_lesser, - threshold, + 'At step %s, metric "%s" has value %s which is %s the configured ' + 'threshold (%s) for early stopping.', + step, + metric_name, + val, + greater_or_lesser, + threshold, ) return True return False return make_early_stopping_hook( - estimator=estimator, - should_stop_fn=stop_if_threshold_crossed_fn, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps, + estimator=estimator, + should_stop_fn=stop_if_threshold_crossed_fn, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps, ) def _stop_if_no_metric_improvement_hook( - estimator, - metric_name, - max_steps_without_improvement, - higher_is_better, - eval_dir, - min_steps, - run_every_secs, - run_every_steps, + estimator, + metric_name, + max_steps_without_improvement, + higher_is_better, + eval_dir, + min_steps, + run_every_secs, + run_every_steps, ): """Returns hook to stop training if given metric shows no improvement.""" if eval_dir is None: @@ -492,21 +501,21 @@ def stop_if_no_metric_improvement_fn(): best_val_step = step if step - best_val_step >= max_steps_without_improvement: tf_logging.info( - 'No %s in metric "%s" for %s steps, which is greater than or equal ' - 'to max steps (%s) configured for early stopping.', - increase_or_decrease, - metric_name, - step - best_val_step, - max_steps_without_improvement, + 'No %s in metric "%s" for %s steps, which is greater than or equal ' + 'to max steps (%s) configured for early stopping.', + increase_or_decrease, + metric_name, + step - best_val_step, + max_steps_without_improvement, ) return True return False return make_early_stopping_hook( - estimator=estimator, - should_stop_fn=stop_if_no_metric_improvement_fn, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps, + estimator=estimator, + should_stop_fn=stop_if_no_metric_improvement_fn, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps, ) @@ -520,20 +529,24 @@ def _summaries(eval_dir): `tensorflow.Event` object read from the event files. """ if gfile.Exists(eval_dir): - for event_file in gfile.Glob(os.path.join(eval_dir, _EVENT_FILE_GLOB_PATTERN)): + for event_file in gfile.Glob( + os.path.join(eval_dir, _EVENT_FILE_GLOB_PATTERN)): for event in summary_iterator.summary_iterator(event_file): yield event def _get_or_create_stop_var(): - with variable_scope.variable_scope(name_or_scope=EARLY_STOP_SIG_SCOPE, values=[], reuse=variable_scope.AUTO_REUSE): + with variable_scope.variable_scope( + name_or_scope=EARLY_STOP_SIG_SCOPE, + values=[], + reuse=variable_scope.AUTO_REUSE): return variable_scope.get_variable( - name=EARLY_STOP_SIG_NAME, - shape=[], - dtype=dtypes.bool, - initializer=init_ops.constant_initializer(False), - collections=[ops.GraphKeys.GLOBAL_VARIABLES], - trainable=False, + name=EARLY_STOP_SIG_NAME, + shape=[], + dtype=dtypes.bool, + initializer=init_ops.constant_initializer(False), + collections=[ops.GraphKeys.GLOBAL_VARIABLES], + trainable=False, ) @@ -545,7 +558,8 @@ def __init__(self, should_stop_fn, run_every_secs=60, run_every_steps=None): raise TypeError('`should_stop_fn` must be callable.') self._should_stop_fn = should_stop_fn - self._timer = basic_session_run_hooks.SecondOrStepTimer(every_secs=run_every_secs, every_steps=run_every_steps) + self._timer = basic_session_run_hooks.SecondOrStepTimer( + every_secs=run_every_secs, every_steps=run_every_steps) self._global_step_tensor = None self._stop_var = _get_or_create_stop_var() self._stop_op = None @@ -563,7 +577,8 @@ def after_run(self, run_context, run_values): if self._timer.should_trigger_for_step(global_step): self._timer.update_last_triggered_step(global_step) if self._should_stop_fn(): - tf_logging.info('Requesting early stopping at global step %d', global_step) + tf_logging.info('Requesting early stopping at global step %d', + global_step) run_context.session.run(self._stop_op) run_context.request_stop() @@ -589,11 +604,13 @@ def after_run(self, run_context, run_values): class OssStopSignalHook(session_run_hook.SessionRunHook): + def __init__(self, model_dir, run_every_secs=10, run_every_steps=None): self._stop_sig_file = os.path.join(model_dir, 'OSS_STOP_SIGNAL') self._stop = False self._check_run = True - self._timer = basic_session_run_hooks.SecondOrStepTimer(every_secs=run_every_secs, every_steps=run_every_steps) + self._timer = basic_session_run_hooks.SecondOrStepTimer( + every_secs=run_every_secs, every_steps=run_every_steps) sleep_time = run_every_secs if run_every_secs is not None else 1 self._curr_step = 0 @@ -603,7 +620,8 @@ def _check_stop(): self._timer.update_last_triggered_step(self._curr_step) if gfile.Exists(self._stop_sig_file): self._stop = True - logging.info('OssStopSignalHook: stop on signal %s' % self._stop_sig_file) + logging.info('OssStopSignalHook: stop on signal %s' % + self._stop_sig_file) break else: time.sleep(sleep_time) @@ -640,15 +658,16 @@ def oss_stop_hook(estimator, run_every_secs=10, run_every_steps=None): """ if estimator.config.is_chief: return OssStopSignalHook( - estimator.model_dir, - run_every_secs=run_every_secs, - run_every_steps=run_every_steps, + estimator.model_dir, + run_every_secs=run_every_secs, + run_every_steps=run_every_steps, ) else: return _CheckForStoppingHook() class DeadlineStopHook(session_run_hook.SessionRunHook): + def __init__(self, deadline_ts): self._deadline_ts = deadline_ts self._stop_var = _get_or_create_stop_var() diff --git a/easy_rec/python/compat/embedding_ops.py b/easy_rec/python/compat/embedding_ops.py index ef66595f2..b1ffd8080 100644 --- a/easy_rec/python/compat/embedding_ops.py +++ b/easy_rec/python/compat/embedding_ops.py @@ -10,7 +10,9 @@ def _prune_invalid_ids(sparse_ids, sparse_weights): """Prune invalid IDs (< 0) from the input ids and weights.""" is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) if sparse_weights is not None: - is_id_valid = math_ops.logical_and(is_id_valid, array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) + is_id_valid = math_ops.logical_and( + is_id_valid, + array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) if sparse_weights is not None: sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) @@ -27,14 +29,14 @@ def _prune_invalid_weights(sparse_ids, sparse_weights): def safe_embedding_lookup_sparse( - embedding_weights, - sparse_ids, - sparse_weights=None, - combiner='mean', - default_id=None, - name=None, - partition_strategy='div', - max_norm=None, + embedding_weights, + sparse_ids, + sparse_weights=None, + combiner='mean', + default_id=None, + name=None, + partition_strategy='div', + max_norm=None, ): """Lookup embedding results, accounting for invalid IDs and empty features. @@ -85,28 +87,36 @@ def safe_embedding_lookup_sparse( raise ValueError('Missing embedding_weights %s.' % embedding_weights) embed_tensors = [ops.convert_to_tensor(embedding_weights)] - with ops.name_scope(name, 'embedding_lookup', embed_tensors + [sparse_ids, sparse_weights]) as scope: + with ops.name_scope(name, 'embedding_lookup', + embed_tensors + [sparse_ids, sparse_weights]) as scope: # Reshape higher-rank sparse ids and weights to linear segment ids. original_shape = sparse_ids.dense_shape original_rank_dim = sparse_ids.dense_shape.get_shape()[0] - original_rank = array_ops.size(original_shape) if original_rank_dim.value is None else original_rank_dim.value + original_rank = array_ops.size( + original_shape + ) if original_rank_dim.value is None else original_rank_dim.value sparse_ids = sparse_ops.sparse_reshape( - sparse_ids, - [ - math_ops.reduce_prod(array_ops.slice(original_shape, [0], [original_rank - 1])), - array_ops.gather(original_shape, original_rank - 1), - ], + sparse_ids, + [ + math_ops.reduce_prod( + array_ops.slice(original_shape, [0], [original_rank - 1])), + array_ops.gather(original_shape, original_rank - 1), + ], ) if sparse_weights is not None: - sparse_weights = sparse_tensor.SparseTensor(sparse_ids.indices, sparse_weights.values, sparse_ids.dense_shape) + sparse_weights = sparse_tensor.SparseTensor(sparse_ids.indices, + sparse_weights.values, + sparse_ids.dense_shape) # Prune invalid ids and weights. sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) if combiner != 'sum': - sparse_ids, sparse_weights = _prune_invalid_weights(sparse_ids, sparse_weights) + sparse_ids, sparse_weights = _prune_invalid_weights( + sparse_ids, sparse_weights) # Fill in dummy values for empty features, if necessary. - sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, default_id or 0) + sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows( + sparse_ids, default_id or 0) if sparse_weights is not None: sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) @@ -114,44 +124,46 @@ def safe_embedding_lookup_sparse( values = sparse_ids.values if values.dtype != dtypes.int64: values = math_ops.to_int64(values) - sparse_ids = sparse_tensor.SparseTensor(indices=indices, values=values, dense_shape=sparse_ids.dense_shape) + sparse_ids = sparse_tensor.SparseTensor( + indices=indices, values=values, dense_shape=sparse_ids.dense_shape) result = embedding_ops.embedding_lookup_sparse( - embedding_weights, - sparse_ids, - sparse_weights, - combiner=combiner, - partition_strategy=partition_strategy, - name=None if default_id is None else scope, - max_norm=max_norm, + embedding_weights, + sparse_ids, + sparse_weights, + combiner=combiner, + partition_strategy=partition_strategy, + name=None if default_id is None else scope, + max_norm=max_norm, ) if default_id is None: # Broadcast is_row_empty to the same shape as embedding_lookup_result, # for use in Select. is_row_empty = array_ops.tile( - array_ops.reshape(is_row_empty, [-1, 1]), - array_ops.stack([1, array_ops.shape(result)[1]]), + array_ops.reshape(is_row_empty, [-1, 1]), + array_ops.stack([1, array_ops.shape(result)[1]]), ) - result = array_ops.where(is_row_empty, array_ops.zeros_like(result), result, name=scope) + result = array_ops.where( + is_row_empty, array_ops.zeros_like(result), result, name=scope) # Reshape back from linear ids back into higher-dimensional dense result. final_result = array_ops.reshape( - result, - array_ops.concat( - [ - array_ops.slice( - math_ops.cast(original_shape, dtypes.int32), - [0], - [original_rank - 1], - ), - array_ops.slice(array_ops.shape(result), [1], [-1]), - ], - 0, - ), + result, + array_ops.concat( + [ + array_ops.slice( + math_ops.cast(original_shape, dtypes.int32), + [0], + [original_rank - 1], + ), + array_ops.slice(array_ops.shape(result), [1], [-1]), + ], + 0, + ), ) final_result.set_shape( - tensor_shape.unknown_shape((original_rank_dim - 1).value).concatenate(result.get_shape()[1:]) - ) + tensor_shape.unknown_shape( + (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) return final_result diff --git a/easy_rec/python/compat/embedding_parallel_saver.py b/easy_rec/python/compat/embedding_parallel_saver.py index f540bb42d..66a9610b0 100644 --- a/easy_rec/python/compat/embedding_parallel_saver.py +++ b/easy_rec/python/compat/embedding_parallel_saver.py @@ -5,20 +5,18 @@ import numpy as np from tensorflow.core.protobuf import saver_pb2 -from tensorflow.python.framework import dtypes, ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.platform import gfile +from tensorflow.python.training import saver + +from easy_rec.python.utils import constant # from tensorflow.python.ops import math_ops # from tensorflow.python.ops import logging_ops from tensorflow.python.ops import ( # NOQA - array_ops, - control_flow_ops, - script_ops, - state_ops, + array_ops, control_flow_ops, script_ops, state_ops, ) -from tensorflow.python.platform import gfile -from tensorflow.python.training import saver - -from easy_rec.python.utils import constant try: import horovod.tensorflow as hvd @@ -49,52 +47,55 @@ def _get_embed_part_id(embed_file): class EmbeddingParallelSaver(saver.Saver): + def __init__( - self, - var_list=None, - reshape=False, - sharded=False, - max_to_keep=5, - keep_checkpoint_every_n_hours=10000.0, - name=None, - restore_sequentially=False, - saver_def=None, - builder=None, - defer_build=False, - allow_empty=False, - write_version=saver_pb2.SaverDef.V2, - pad_step_number=False, - save_relative_paths=False, - filename=None, + self, + var_list=None, + reshape=False, + sharded=False, + max_to_keep=5, + keep_checkpoint_every_n_hours=10000.0, + name=None, + restore_sequentially=False, + saver_def=None, + builder=None, + defer_build=False, + allow_empty=False, + write_version=saver_pb2.SaverDef.V2, + pad_step_number=False, + save_relative_paths=False, + filename=None, ): self._kv_vars = [] self._embed_vars = [] tf_vars = [] embed_para_vars = ops.get_collection(constant.EmbeddingParallel) for var in var_list: - if dynamic_variable is not None and isinstance(var, dynamic_variable.DynamicVariable): + if dynamic_variable is not None and isinstance( + var, dynamic_variable.DynamicVariable): self._kv_vars.append(var) elif var.name in embed_para_vars: - logging.info('save shard embedding %s part_id=%d part_shape=%s' % (var.name, hvd.rank(), var.get_shape())) + logging.info('save shard embedding %s part_id=%d part_shape=%s' % + (var.name, hvd.rank(), var.get_shape())) self._embed_vars.append(var) else: tf_vars.append(var) super(EmbeddingParallelSaver, self).__init__( - tf_vars, - reshape=reshape, - sharded=sharded, - max_to_keep=max_to_keep, - keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours, - name=name, - restore_sequentially=restore_sequentially, - saver_def=saver_def, - builder=builder, - defer_build=defer_build, - allow_empty=allow_empty, - write_version=write_version, - pad_step_number=pad_step_number, - save_relative_paths=save_relative_paths, - filename=filename, + tf_vars, + reshape=reshape, + sharded=sharded, + max_to_keep=max_to_keep, + keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours, + name=name, + restore_sequentially=restore_sequentially, + saver_def=saver_def, + builder=builder, + defer_build=defer_build, + allow_empty=allow_empty, + write_version=write_version, + pad_step_number=pad_step_number, + save_relative_paths=save_relative_paths, + filename=filename, ) self._is_build = False @@ -109,7 +110,8 @@ def _save_embed(embed, filename, var_name): filename = filename.decode('utf-8') var_name = var_name.decode('utf-8').replace('/', '__') embed_dir = filename + '-embedding/' - logging.info('task[%d] save_dense_embed: %s to %s' % (task_id, var_name, embed_dir)) + logging.info('task[%d] save_dense_embed: %s to %s' % + (task_id, var_name, embed_dir)) if not gfile.Exists(embed_dir): gfile.MakeDirs(embed_dir) embed_file = filename + '-embedding/embed-' + var_name + '-part-%d.bin' % task_id @@ -126,16 +128,21 @@ def _save_embed(embed, filename, var_name): gfile.DeleteRecursively(embed_file) return np.asarray([embed_file], order='C', dtype=np.object) - file_name = ops.get_default_graph().get_tensor_by_name(self.saver_def.filename_tensor_name) - save_paths = script_ops.py_func(_save_embed, [embed_var, file_name, embed_var.name], dtypes.string) + file_name = ops.get_default_graph().get_tensor_by_name( + self.saver_def.filename_tensor_name) + save_paths = script_ops.py_func(_save_embed, + [embed_var, file_name, embed_var.name], + dtypes.string) return save_paths def _load_dense_embedding(self, embed_var): - file_name = ops.get_default_graph().get_tensor_by_name(self.saver_def.filename_tensor_name) + file_name = ops.get_default_graph().get_tensor_by_name( + self.saver_def.filename_tensor_name) embed_dim = embed_var.get_shape()[-1] embed_part_size = embed_var.get_shape()[0] - def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, filename, var_name): + def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, + filename, var_name): filename = filename.decode('utf-8') var_name = var_name.decode('utf-8').replace('/', '__') embed_pattern = filename + '-embedding/embed-' + var_name + '-part-*.bin' @@ -143,10 +150,8 @@ def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, filename, embed_files.sort(key=_get_embed_part_id) - logging.info( - 'task[%d] embed_files=%s embed_dim=%d embed_part_size=%d' - % (part_id, ','.join(embed_files), embed_dim, embed_part_size) - ) + logging.info('task[%d] embed_files=%s embed_dim=%d embed_part_size=%d' % + (part_id, ','.join(embed_files), embed_dim, embed_part_size)) part_embed_vals = np.zeros([embed_part_size, embed_dim], dtype=np.float32) part_update_cnt = 0 @@ -158,11 +163,10 @@ def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, filename, embed_ids_o = np.arange(len(embed_val)) embed_ids_o = part_id_o + embed_ids_o * len(embed_files) sel_ids = np.where( - np.logical_and( - (embed_ids_o % part_num) == part_id, - embed_ids_o < embed_part_size * part_num, - ) - )[0] + np.logical_and( + (embed_ids_o % part_num) == part_id, + embed_ids_o < embed_part_size * part_num, + ))[0] part_update_cnt += len(sel_ids) embed_ids = embed_ids_o[sel_ids] embed_ids_n = np.array(embed_ids / part_num, dtype=np.int64) @@ -173,35 +177,35 @@ def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, filename, with ops.control_dependencies([embed_var._initializer_op]): if load_embed_lib is not None: embed_val = load_embed_lib.load_embed( - task_index=hvd.rank(), - task_num=hvd.size(), - embed_dim=embed_dim, - embed_part_size=embed_part_size, - var_name='embed-' + embed_var.name.replace('/', '__'), - ckpt_path=file_name, + task_index=hvd.rank(), + task_num=hvd.size(), + embed_dim=embed_dim, + embed_part_size=embed_part_size, + var_name='embed-' + embed_var.name.replace('/', '__'), + ckpt_path=file_name, ) else: embed_val = script_ops.py_func( - _load_embed, - [ - embed_var, - embed_dim, - embed_part_size, - hvd.rank(), - hvd.size(), - file_name, - embed_var.name, - ], - dtypes.float32, + _load_embed, + [ + embed_var, + embed_dim, + embed_part_size, + hvd.rank(), + hvd.size(), + file_name, + embed_var.name, + ], + dtypes.float32, ) embed_val.set_shape(embed_var.get_shape()) return state_ops.assign(embed_var, embed_val) def _save_kv_embedding(self, sok_var): indices, values = dynamic_variable_ops.dummy_var_export( - sok_var.handle, key_type=sok_var.key_type, dtype=sok_var.handle_dtype - ) - file_name = ops.get_default_graph().get_tensor_by_name(self.saver_def.filename_tensor_name) + sok_var.handle, key_type=sok_var.key_type, dtype=sok_var.handle_dtype) + file_name = ops.get_default_graph().get_tensor_by_name( + self.saver_def.filename_tensor_name) def _save_key_vals(indices, values, filename, var_name): var_name = var_name.decode('utf-8').replace('/', '__') @@ -230,19 +234,22 @@ def _save_key_vals(indices, values, filename, var_name): return np.asarray([key_file, val_file], order='C', dtype=np.object) - save_paths = script_ops.py_func(_save_key_vals, [indices, values, file_name, sok_var.name], dtypes.string) + save_paths = script_ops.py_func(_save_key_vals, + [indices, values, file_name, sok_var.name], + dtypes.string) return save_paths def _load_kv_embedding(self, sok_var): + def _load_key_vals(filename, var_name): var_name = var_name.decode('utf-8').replace('/', '__') filename = filename.decode('utf-8') key_file_pattern = filename + '-embedding/embed-' + var_name + '-part-*.key' - logging.info( - 'key_file_pattern=%s filename=%s var_name=%s var=%s' % (key_file_pattern, filename, var_name, str(sok_var)) - ) + logging.info('key_file_pattern=%s filename=%s var_name=%s var=%s' % + (key_file_pattern, filename, var_name, str(sok_var))) key_files = gfile.Glob(key_file_pattern) - logging.info('key_file_pattern=%s file_num=%d' % (key_file_pattern, len(key_files))) + logging.info('key_file_pattern=%s file_num=%d' % + (key_file_pattern, len(key_files))) all_keys = [] all_vals = [] for key_file in key_files: @@ -253,27 +260,22 @@ def _load_key_vals(filename, var_name): if len(tmp_ids) == 0: break all_keys.append(tmp_keys.take(tmp_ids, axis=0)) - logging.info( - 'part_keys.shape=%s %s %s' - % ( + logging.info('part_keys.shape=%s %s %s' % ( str(tmp_keys.shape), str(tmp_ids.shape), str(all_keys[-1].shape), - ) - ) + )) val_file = key_file[:-4] + 'vals' with gfile.GFile(val_file, 'rb') as fin: - tmp_vals = np.frombuffer(fin.read(), dtype=np.float32).reshape([-1, sok_var._dimension]) + tmp_vals = np.frombuffer( + fin.read(), dtype=np.float32).reshape([-1, sok_var._dimension]) all_vals.append(tmp_vals.take(tmp_ids, axis=0)) - logging.info( - 'part_vals.shape=%s %s %s' - % ( + logging.info('part_vals.shape=%s %s %s' % ( str(tmp_vals.shape), str(tmp_ids.shape), str(all_vals[-1].shape), - ) - ) + )) all_keys = np.concatenate(all_keys, axis=0) all_vals = np.concatenate(all_vals, axis=0) @@ -284,21 +286,22 @@ def _load_key_vals(filename, var_name): all_vals = all_vals.take(shuffle_ids, axis=0) return all_keys, all_vals - file_name = ops.get_default_graph().get_tensor_by_name(self.saver_def.filename_tensor_name) + file_name = ops.get_default_graph().get_tensor_by_name( + self.saver_def.filename_tensor_name) if load_embed_lib is not None: keys, vals = load_embed_lib.load_kv_embed( - task_index=hvd.rank(), - task_num=hvd.size(), - embed_dim=sok_var._dimension, - var_name='embed-' + sok_var.name.replace('/', '__'), - ckpt_path=file_name, + task_index=hvd.rank(), + task_num=hvd.size(), + embed_dim=sok_var._dimension, + var_name='embed-' + sok_var.name.replace('/', '__'), + ckpt_path=file_name, ) else: logging.warning('libload_embed.so not loaded, will use python script_ops') keys, vals = script_ops.py_func( - _load_key_vals, - [file_name, sok_var.name], - (dtypes.int64, dtypes.float32), + _load_key_vals, + [file_name, sok_var.name], + (dtypes.int64, dtypes.float32), ) with ops.control_dependencies([sok_var._initializer_op]): return dynamic_variable_ops.dummy_var_assign(sok_var.handle, keys, vals) @@ -314,13 +317,15 @@ def build(self): restore_ops.append(self._load_kv_embedding(sok_var)) for embed_var in self._embed_vars: restore_ops.append(self._load_dense_embedding(embed_var)) - old_restore_op = ops.get_default_graph().get_operation_by_name(self.saver_def.restore_op_name) + old_restore_op = ops.get_default_graph().get_operation_by_name( + self.saver_def.restore_op_name) restore_ops.append(old_restore_op) restore_op_n = control_flow_ops.group(restore_ops) self.saver_def.restore_op_name = restore_op_n.name if self.saver_def.save_tensor_name and self._has_embed_vars(): - file_name = ops.get_default_graph().get_tensor_by_name(self.saver_def.filename_tensor_name) + file_name = ops.get_default_graph().get_tensor_by_name( + self.saver_def.filename_tensor_name) save_part_ops = [] for sok_var in self._kv_vars: save_part_op = self._save_kv_embedding(sok_var) @@ -328,7 +333,8 @@ def build(self): for embed_var in self._embed_vars: save_part_op = self._save_dense_embedding(embed_var) save_part_ops.append(save_part_op) - old_save_op = ops.get_default_graph().get_tensor_by_name(self.saver_def.save_tensor_name) + old_save_op = ops.get_default_graph().get_tensor_by_name( + self.saver_def.save_tensor_name) # only the first worker needs to save non embedding variables if hvd.rank() == 0: save_part_ops.append(old_save_op) diff --git a/easy_rec/python/compat/estimator_train.py b/easy_rec/python/compat/estimator_train.py index 543f75d16..dbd853759 100644 --- a/easy_rec/python/compat/estimator_train.py +++ b/easy_rec/python/compat/estimator_train.py @@ -4,18 +4,17 @@ import os import tensorflow as tf -from tensorflow.python.distribute import estimator_training as distribute_coordinator_training # NOQA from tensorflow.python.estimator import run_config as run_config_lib -from tensorflow.python.estimator.training import ( # NOQA - _assert_eval_spec, - _ContinuousEvalListener, - _TrainingExecutor, -) from tensorflow.python.util import compat from easy_rec.python.compat.exporter import FinalExporter from easy_rec.python.utils import estimator_utils +from tensorflow.python.distribute import estimator_training as distribute_coordinator_training # NOQA +from tensorflow.python.estimator.training import ( # NOQA + _assert_eval_spec, _ContinuousEvalListener, _TrainingExecutor, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 gfile = tf.gfile @@ -26,7 +25,8 @@ class TrainDoneListener(_ContinuousEvalListener): def __init__(self, estimator): self._model_dir = estimator.model_dir - self._train_done_file = os.path.join(self._model_dir, 'ESTIMATOR_TRAIN_DONE') + self._train_done_file = os.path.join(self._model_dir, + 'ESTIMATOR_TRAIN_DONE') @property def train_done_file(self): @@ -47,8 +47,8 @@ def after_eval(self, eval_result): latest_ckpt_path = estimator_utils.latest_checkpoint(model_dir) if latest_ckpt_path != last_ckpt_path: logging.info( - 'TrainDoneListener: latest_ckpt_path[%s] != last_ckpt_path[%s]' % (latest_ckpt_path, last_ckpt_path) - ) + 'TrainDoneListener: latest_ckpt_path[%s] != last_ckpt_path[%s]' % + (latest_ckpt_path, last_ckpt_path)) # there are more checkpoints wait to be evaluated return True return not gfile.Exists(self._train_done_file) @@ -60,10 +60,10 @@ def train_and_evaluate(estimator, train_spec, eval_spec): train_done_listener = TrainDoneListener(estimator) executor = _TrainingExecutor( - estimator=estimator, - train_spec=train_spec, - eval_spec=eval_spec, - continuous_eval_listener=train_done_listener, + estimator=estimator, + train_spec=train_spec, + eval_spec=eval_spec, + continuous_eval_listener=train_done_listener, ) config = estimator.config @@ -71,34 +71,38 @@ def train_and_evaluate(estimator, train_spec, eval_spec): # environment, we run `train_and_evaluate` via distribute coordinator. if distribute_coordinator_training.should_run_distribute_coordinator(config): logging.info('Running `train_and_evaluate` with Distribute Coordinator.') - distribute_coordinator_training.train_and_evaluate(estimator, train_spec, eval_spec, _TrainingExecutor) + distribute_coordinator_training.train_and_evaluate(estimator, train_spec, + eval_spec, + _TrainingExecutor) return if config.task_type == run_config_lib.TaskType.EVALUATOR and config.task_id > 0: raise ValueError( - 'For distributed training, there can only be one `evaluator` task ' '(with task id 0). Given task id {}'.format( - config.task_id - ) - ) + 'For distributed training, there can only be one `evaluator` task ' + '(with task id 0). Given task id {}'.format(config.task_id)) result = executor.run() # fix for the bug evaluator fails to export in case num_epoch is reached # before num_steps is reached or num_steps is set to infinite if estimator_utils.is_evaluator(): - export_dir_base = os.path.join(compat.as_str_any(estimator.model_dir), compat.as_str_any('export')) + export_dir_base = os.path.join( + compat.as_str_any(estimator.model_dir), compat.as_str_any('export')) for exporter in eval_spec.exporters: if isinstance(exporter, FinalExporter): - export_path = os.path.join(compat.as_str_any(export_dir_base), compat.as_str_any(exporter.name)) + export_path = os.path.join( + compat.as_str_any(export_dir_base), + compat.as_str_any(exporter.name)) # avoid duplicate export if gfile.IsDirectory(export_path + '/'): continue exporter.export( - estimator=estimator, - export_path=export_path, - checkpoint_path=estimator_utils.latest_checkpoint(estimator.model_dir), - eval_result=None, - is_the_final_export=True, + estimator=estimator, + export_path=export_path, + checkpoint_path=estimator_utils.latest_checkpoint( + estimator.model_dir), + eval_result=None, + is_the_final_export=True, ) if estimator_utils.is_chief(): diff --git a/easy_rec/python/compat/exporter.py b/easy_rec/python/compat/exporter.py index 2dd24694f..5fa43e431 100644 --- a/easy_rec/python/compat/exporter.py +++ b/easy_rec/python/compat/exporter.py @@ -15,15 +15,20 @@ # ============================================================================== """`Exporter` class represents different flavors of model export.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import os -from tensorflow.python.estimator import gc, util +from tensorflow.python.estimator import gc +from tensorflow.python.estimator import util from tensorflow.python.estimator.canned import metric_keys -from tensorflow.python.estimator.exporter import Exporter, _SavedModelExporter +from tensorflow.python.estimator.exporter import Exporter +from tensorflow.python.estimator.exporter import _SavedModelExporter from tensorflow.python.framework import errors_impl -from tensorflow.python.platform import gfile, tf_logging +from tensorflow.python.platform import gfile +from tensorflow.python.platform import tf_logging from tensorflow.python.summary import summary_iterator from easy_rec.python.utils import io_util @@ -47,10 +52,12 @@ def _loss_smaller(best_eval_result, current_eval_result): """ default_key = metric_keys.MetricKeys.LOSS if not best_eval_result or default_key not in best_eval_result: - raise ValueError('best_eval_result cannot be empty or no loss is found in it.') + raise ValueError( + 'best_eval_result cannot be empty or no loss is found in it.') if not current_eval_result or default_key not in current_eval_result: - raise ValueError('current_eval_result cannot be empty or no loss is found in it.') + raise ValueError( + 'current_eval_result cannot be empty or no loss is found in it.') return best_eval_result[default_key] > current_eval_result[default_key] @@ -59,12 +66,16 @@ def _verify_compare_fn_args(compare_fn): """Verifies compare_fn arguments.""" args = set(util.fn_args(compare_fn)) if 'best_eval_result' not in args: - raise ValueError('compare_fn (%s) must include best_eval_result argument.' % compare_fn) + raise ValueError('compare_fn (%s) must include best_eval_result argument.' % + compare_fn) if 'current_eval_result' not in args: - raise ValueError('compare_fn (%s) must include current_eval_result argument.' % compare_fn) + raise ValueError( + 'compare_fn (%s) must include current_eval_result argument.' % + compare_fn) non_valid_args = list(args - set(['best_eval_result', 'current_eval_result'])) if non_valid_args: - raise ValueError('compare_fn (%s) has following not expected args: %s' % (compare_fn, non_valid_args)) + raise ValueError('compare_fn (%s) has following not expected args: %s' % + (compare_fn, non_valid_args)) def _get_ckpt_version(path): @@ -82,14 +93,14 @@ class BestExporter(Exporter): """ def __init__( - self, - name='best_exporter', - serving_input_receiver_fn=None, - event_file_pattern='eval_val/*.tfevents.*', - compare_fn=_loss_smaller, - assets_extra=None, - as_text=False, - exports_to_keep=5, + self, + name='best_exporter', + serving_input_receiver_fn=None, + event_file_pattern='eval_val/*.tfevents.*', + compare_fn=_loss_smaller, + assets_extra=None, + as_text=False, + exports_to_keep=5, ): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. @@ -171,7 +182,9 @@ def make_train_and_eval_fn(): raise ValueError('`compare_fn` must not be None.') _verify_compare_fn_args(self._compare_fn) - self._saved_model_exporter = _SavedModelExporter(name, serving_input_receiver_fn, assets_extra, as_text) + self._saved_model_exporter = _SavedModelExporter(name, + serving_input_receiver_fn, + assets_extra, as_text) self._event_file_pattern = event_file_pattern self._model_dir = None @@ -179,13 +192,16 @@ def make_train_and_eval_fn(): self._exports_to_keep = exports_to_keep if exports_to_keep is not None and exports_to_keep <= 0: - raise ValueError('`exports_to_keep`, if provided, must be a positive number. Got %s' % exports_to_keep) + raise ValueError( + '`exports_to_keep`, if provided, must be a positive number. Got %s' % + exports_to_keep) @property def name(self): return self._saved_model_exporter.name - def export(self, estimator, export_path, checkpoint_path, eval_result, is_the_final_export): + def export(self, estimator, export_path, checkpoint_path, eval_result, + is_the_final_export): export_result = None if self._model_dir != estimator.model_dir and self._event_file_pattern: @@ -193,27 +209,30 @@ def export(self, estimator, export_path, checkpoint_path, eval_result, is_the_fi tf_logging.info('Loading best metric from event files.') self._model_dir = estimator.model_dir - full_event_file_pattern = os.path.join(self._model_dir, self._event_file_pattern) - self._best_eval_result = self._get_best_eval_result(full_event_file_pattern, eval_result) + full_event_file_pattern = os.path.join(self._model_dir, + self._event_file_pattern) + self._best_eval_result = self._get_best_eval_result( + full_event_file_pattern, eval_result) if self._best_eval_result is None or self._compare_fn( - best_eval_result=self._best_eval_result, current_eval_result=eval_result - ): + best_eval_result=self._best_eval_result, + current_eval_result=eval_result): tf_logging.info('Performing best model export.') self._best_eval_result = eval_result export_result = self._saved_model_exporter.export( - estimator, - export_path, - checkpoint_path, - eval_result, - is_the_final_export, + estimator, + export_path, + checkpoint_path, + eval_result, + is_the_final_export, ) self._garbage_collect_exports(export_path) # cp best checkpoints to best folder model_dir, _ = os.path.split(checkpoint_path) # add / is to be compatiable with oss best_dir = os.path.join(model_dir, 'best_ckpt/') - tf_logging.info('Copy best checkpoint %s to %s' % (checkpoint_path, best_dir)) + tf_logging.info('Copy best checkpoint %s to %s' % + (checkpoint_path, best_dir)) if not gfile.Exists(best_dir): gfile.MakeDirs(best_dir) for tmp_file in gfile.Glob(checkpoint_path + '.*'): @@ -226,7 +245,8 @@ def export(self, estimator, export_path, checkpoint_path, eval_result, is_the_fi try: gfile.Copy(tmp_file, dst_path) except Exception as ex: - tf_logging.warn('Copy file %s to %s failed: %s' % (tmp_file, dst_path, str(ex))) + tf_logging.warn('Copy file %s to %s failed: %s' % + (tmp_file, dst_path, str(ex))) self._garbage_collect_ckpts(best_dir) return export_result @@ -249,9 +269,8 @@ def _garbage_collect_ckpts(self, best_dir): tmp_steps = sorted(tmp_steps) drop_num = len(tmp_steps) - self._exports_to_keep tf_logging.info( - 'garbage_collect_ckpts: steps: %s export_to_keep: %d drop num: %d' - % (str(tmp_steps), self._exports_to_keep, drop_num) - ) + 'garbage_collect_ckpts: steps: %s export_to_keep: %d drop num: %d' % + (str(tmp_steps), self._exports_to_keep, drop_num)) for ver in tmp_steps[:drop_num]: tmp_prefix = os.path.join(best_dir, 'model.ckpt-%d.*' % ver) for tmp_file in gfile.Glob(tmp_prefix): @@ -281,7 +300,8 @@ def _export_version_parser(path): # pylint: disable=protected-access keep_filter = gc._largest_export_versions(self._exports_to_keep) delete_filter = gc._negation(keep_filter) - for p in delete_filter(gc._get_paths(export_dir_base, parser=_export_version_parser)): + for p in delete_filter( + gc._get_paths(export_dir_base, parser=_export_version_parser)): try: gfile.DeleteRecursively(io_util.fix_oss_dir(p.path)) except errors_impl.NotFoundError as e: @@ -312,7 +332,8 @@ def _get_best_eval_result(self, event_files, curr_eval_result): if value.HasField('simple_value'): event_eval_result[value.tag] = value.simple_value if len(event_eval_result) >= 2: - if best_eval_result is None or self._compare_fn(best_eval_result, event_eval_result): + if best_eval_result is None or self._compare_fn( + best_eval_result, event_eval_result): best_eval_result = event_eval_result return best_eval_result @@ -323,7 +344,11 @@ class FinalExporter(Exporter): This class performs a single export at the end of training. """ - def __init__(self, name, serving_input_receiver_fn, assets_extra=None, as_text=False): + def __init__(self, + name, + serving_input_receiver_fn, + assets_extra=None, + as_text=False): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. Args: @@ -344,19 +369,24 @@ def __init__(self, name, serving_input_receiver_fn, assets_extra=None, as_text=F Raises: ValueError: if any arguments is invalid. """ - self._saved_model_exporter = _SavedModelExporter(name, serving_input_receiver_fn, assets_extra, as_text) + self._saved_model_exporter = _SavedModelExporter(name, + serving_input_receiver_fn, + assets_extra, as_text) @property def name(self): return self._saved_model_exporter.name - def export(self, estimator, export_path, checkpoint_path, eval_result, is_the_final_export): + def export(self, estimator, export_path, checkpoint_path, eval_result, + is_the_final_export): if not is_the_final_export: return None tf_logging.info('Performing the final export in the end of training.') - return self._saved_model_exporter.export(estimator, export_path, checkpoint_path, eval_result, is_the_final_export) + return self._saved_model_exporter.export(estimator, export_path, + checkpoint_path, eval_result, + is_the_final_export) class LatestExporter(Exporter): @@ -366,12 +396,12 @@ class LatestExporter(Exporter): """ def __init__( - self, - name, - serving_input_receiver_fn, - assets_extra=None, - as_text=False, - exports_to_keep=5, + self, + name, + serving_input_receiver_fn, + assets_extra=None, + as_text=False, + exports_to_keep=5, ): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. @@ -396,19 +426,24 @@ def __init__( Raises: ValueError: if any arguments is invalid. """ - self._saved_model_exporter = _SavedModelExporter(name, serving_input_receiver_fn, assets_extra, as_text) + self._saved_model_exporter = _SavedModelExporter(name, + serving_input_receiver_fn, + assets_extra, as_text) self._exports_to_keep = exports_to_keep if exports_to_keep is not None and exports_to_keep <= 0: - raise ValueError('`exports_to_keep`, if provided, must be positive number') + raise ValueError( + '`exports_to_keep`, if provided, must be positive number') @property def name(self): return self._saved_model_exporter.name - def export(self, estimator, export_path, checkpoint_path, eval_result, is_the_final_export): - export_result = self._saved_model_exporter.export( - estimator, export_path, checkpoint_path, eval_result, is_the_final_export - ) + def export(self, estimator, export_path, checkpoint_path, eval_result, + is_the_final_export): + export_result = self._saved_model_exporter.export(estimator, export_path, + checkpoint_path, + eval_result, + is_the_final_export) self._garbage_collect_exports(export_path) return export_result @@ -436,7 +471,8 @@ def _export_version_parser(path): # pylint: disable=protected-access keep_filter = gc._largest_export_versions(self._exports_to_keep) delete_filter = gc._negation(keep_filter) - for p in delete_filter(gc._get_paths(export_dir_base, parser=_export_version_parser)): + for p in delete_filter( + gc._get_paths(export_dir_base, parser=_export_version_parser)): try: gfile.DeleteRecursively(io_util.fix_oss_dir(p.path)) except errors_impl.NotFoundError as e: diff --git a/easy_rec/python/compat/feature_column/feature_column.py b/easy_rec/python/compat/feature_column/feature_column.py index ccac194ac..308f6f581 100644 --- a/easy_rec/python/compat/feature_column/feature_column.py +++ b/easy_rec/python/compat/feature_column/feature_column.py @@ -128,7 +128,9 @@ in both places. """ -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import abc import collections @@ -138,32 +140,12 @@ import numpy as np import six from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes, ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib from tensorflow.python.framework import tensor_shape from tensorflow.python.keras.engine import training from tensorflow.python.layers import base - -# from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import ( # NOQA - array_ops, - check_ops, - control_flow_ops, - data_flow_ops, - embedding_ops, - init_ops, - lookup_ops, - math_ops, - nn_ops, - parsing_ops, - resource_variable_ops, - sparse_ops, - string_ops, - template, - variable_scope, - variables, -) - # from tensorflow.python.ops.ragged import ragged_tensor # from tensorflow.python.ops.ragged import ragged_util from tensorflow.python.platform import gfile @@ -172,7 +154,16 @@ from tensorflow.python.util import nest from easy_rec.python.compat.feature_column import utils as fc_utils -from easy_rec.python.utils import conditional, constant, embedding_utils +from easy_rec.python.utils import conditional +from easy_rec.python.utils import constant +from easy_rec.python.utils import embedding_utils + +# from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import ( # NOQA + array_ops, check_ops, control_flow_ops, data_flow_ops, embedding_ops, + init_ops, lookup_ops, math_ops, nn_ops, parsing_ops, resource_variable_ops, + sparse_ops, string_ops, template, variable_scope, variables, +) try: from easy_rec.python.compat import dynamic_variable @@ -185,13 +176,19 @@ hvd = None -def embedding_lookup_ragged(embedding_weights, ragged_ids, ragged_weights, combiner, max_norm=None, name=None): +def embedding_lookup_ragged(embedding_weights, + ragged_ids, + ragged_weights, + combiner, + max_norm=None, + name=None): segment_ids = ragged_ids.value_rowids() if segment_ids.dtype != dtypes.int32: segment_ids = math_ops.cast(segment_ids, dtypes.int32) ids = ragged_ids.flat_values ids, idx = array_ops.unique(ids) - embeddings = embedding_ops.embedding_lookup(embedding_weights, ids, partition_strategy='mod', max_norm=max_norm) + embeddings = embedding_ops.embedding_lookup( + embedding_weights, ids, partition_strategy='mod', max_norm=max_norm) if ragged_weights is not None: weights = ragged_weights.flat_values embeddings = array_ops.gather(embeddings, idx) @@ -224,11 +221,14 @@ def embedding_lookup_ragged(embedding_weights, ragged_ids, ragged_weights, combi else: assert idx is not None if combiner == 'sum': - embeddings = math_ops.sparse_segment_sum(embeddings, idx, segment_ids, name=name) + embeddings = math_ops.sparse_segment_sum( + embeddings, idx, segment_ids, name=name) elif combiner == 'mean': - embeddings = math_ops.sparse_segment_mean(embeddings, idx, segment_ids, name=name) + embeddings = math_ops.sparse_segment_mean( + embeddings, idx, segment_ids, name=name) elif combiner == 'sqrtn': - embeddings = math_ops.sparse_segment_sqrt_n(embeddings, idx, segment_ids, name=name) + embeddings = math_ops.sparse_segment_sqrt_n( + embeddings, idx, segment_ids, name=name) else: assert False, 'Unrecognized combiner' return embeddings @@ -236,12 +236,12 @@ def embedding_lookup_ragged(embedding_weights, ragged_ids, ragged_weights, combi # model parallel embedding lookup def embedding_parallel_lookup( - embedding, - lookup_indices, - output_ids, - is_training, - output_tensors=None, - batch_size=None, + embedding, + lookup_indices, + output_ids, + is_training, + output_tensors=None, + batch_size=None, ): N = len(output_ids) if batch_size is None: @@ -254,31 +254,34 @@ def embedding_parallel_lookup( all_ids, segment_lens = lookup_indices['sparse_fea'] all_uniq_ids, uniq_idx = array_ops.unique(all_ids) cumsum_lens = math_ops.cumsum(segment_lens) - segment_ids = array_ops.searchsorted(cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') - elif ( - isinstance(lookup_indices, dict) - and 'ragged_ids' in lookup_indices.keys() - and 'ragged_lens' in lookup_indices.keys() - ): + segment_ids = array_ops.searchsorted( + cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') + elif (isinstance(lookup_indices, dict) and + 'ragged_ids' in lookup_indices.keys() and + 'ragged_lens' in lookup_indices.keys()): all_ids, segment_lens = ( - lookup_indices['ragged_ids'], - lookup_indices['ragged_lens'], + lookup_indices['ragged_ids'], + lookup_indices['ragged_lens'], ) all_uniq_ids, uniq_idx = array_ops.unique(all_ids) cumsum_lens = math_ops.cumsum(segment_lens) - segment_ids = array_ops.searchsorted(cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') + segment_ids = array_ops.searchsorted( + cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') elif isinstance(lookup_indices[0], sparse_tensor_lib.SparseTensor): with ops.device('/cpu:0'): all_ids = array_ops.concat([x.values for x in lookup_indices], axis=0) - segment_ids = array_ops.concat([x.indices[:, 0] for x in lookup_indices], axis=0) + segment_ids = array_ops.concat([x.indices[:, 0] for x in lookup_indices], + axis=0) all_uniq_ids, uniq_idx = array_ops.unique(all_ids) elif 'RaggedTensor' in str(type(lookup_indices[0])): with ops.device('/cpu:0'): all_ids = array_ops.concat([x.values for x in lookup_indices], axis=0) - segment_lens = array_ops.concat([x.row_lengths() for x in lookup_indices], axis=0) + segment_lens = array_ops.concat([x.row_lengths() for x in lookup_indices], + axis=0) all_uniq_ids, uniq_idx = array_ops.unique(all_ids) cumsum_lens = math_ops.cumsum(segment_lens) - segment_ids = array_ops.searchsorted(cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') + segment_ids = array_ops.searchsorted( + cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') else: assert False, 'invalid indices type: %s' % str(type(lookup_indices[0])) @@ -286,17 +289,22 @@ def embedding_parallel_lookup( if num_parts > 1: # dynamic partition p_assignments = math_ops.cast(all_uniq_ids % num_parts, dtypes.int32) - gather_ids = data_flow_ops.dynamic_partition(all_uniq_ids, p_assignments, num_parts) + gather_ids = data_flow_ops.dynamic_partition(all_uniq_ids, p_assignments, + num_parts) original_ids = math_ops.range(array_ops.size(all_uniq_ids)) - original_part_ids = data_flow_ops.dynamic_partition(original_ids, p_assignments, num_parts) + original_part_ids = data_flow_ops.dynamic_partition(original_ids, + p_assignments, + num_parts) # all2all - split_sizes = array_ops.concat([array_ops.shape(x) for x in gather_ids], axis=0) + split_sizes = array_ops.concat([array_ops.shape(x) for x in gather_ids], + axis=0) send_ids = array_ops.concat(gather_ids, axis=0) recv_ids, recv_lens = hvd.alltoall(send_ids, split_sizes) # read embedding from dynamic variable if isinstance(embedding, dynamic_variable.DynamicVariable): - send_embed = embedding.sparse_read(recv_ids, lookup_only=(not is_training)) + send_embed = embedding.sparse_read( + recv_ids, lookup_only=(not is_training)) else: # find in subarray position # 0 2 4 6 8 10 ... @@ -306,28 +314,29 @@ def embedding_parallel_lookup( # all2all recv_embeddings, _ = hvd.alltoall(send_embed, recv_lens) - recv_embeddings = array_ops.split(recv_embeddings, num_or_size_splits=split_sizes) + recv_embeddings = array_ops.split( + recv_embeddings, num_or_size_splits=split_sizes) recv_embeddings = data_flow_ops.parallel_dynamic_stitch( - original_part_ids, recv_embeddings, name='parallel_dynamic_stitch' - ) + original_part_ids, recv_embeddings, name='parallel_dynamic_stitch') embeddings = math_ops.sparse_segment_sum( - recv_embeddings, - uniq_idx, - segment_ids, - num_segments=num_segments, - name='sparse_segment_sum', + recv_embeddings, + uniq_idx, + segment_ids, + num_segments=num_segments, + name='sparse_segment_sum', ) else: if isinstance(embedding, dynamic_variable.DynamicVariable): - recv_embeddings = embedding.sparse_read(all_uniq_ids, lookup_only=(not is_training)) + recv_embeddings = embedding.sparse_read( + all_uniq_ids, lookup_only=(not is_training)) else: recv_embeddings = array_ops.gather(embedding, all_uniq_ids) embeddings = math_ops.sparse_segment_sum( - recv_embeddings, - uniq_idx, - segment_ids, - num_segments=num_segments, - name='sparse_segment_sum', + recv_embeddings, + uniq_idx, + segment_ids, + num_segments=num_segments, + name='sparse_segment_sum', ) embed_dim = embedding.get_shape()[-1] @@ -340,30 +349,31 @@ def embedding_parallel_lookup( if batch_size is None: batch_size = -1 - return array_ops.reshape(array_ops.transpose(output_tensor, perm=[1, 0, 2]), [batch_size, N * embed_dim]) + return array_ops.reshape( + array_ops.transpose(output_tensor, perm=[1, 0, 2]), + [batch_size, N * embed_dim]) def _internal_input_layer( - features, - feature_columns, - weight_collections=None, - trainable=True, - cols_to_vars=None, - scope=None, - cols_to_output_tensors=None, - from_template=False, - feature_name_to_output_tensors=None, - is_training=True, + features, + feature_columns, + weight_collections=None, + trainable=True, + cols_to_vars=None, + scope=None, + cols_to_output_tensors=None, + from_template=False, + feature_name_to_output_tensors=None, + is_training=True, ): """See input_layer, `scope` is a name or variable scope to use.""" feature_columns = _normalize_feature_columns(feature_columns) for column in feature_columns: if not isinstance(column, _DenseColumn): raise ValueError( - 'Items of feature_columns must be a _DenseColumn. ' - 'You can wrap a categorical column with an ' - 'embedding_column or indicator_column. Given: {}'.format(column) - ) + 'Items of feature_columns must be a _DenseColumn. ' + 'You can wrap a categorical column with an ' + 'embedding_column or indicator_column. Given: {}'.format(column)) weight_collections = list(weight_collections or []) if ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections: weight_collections.append(ops.GraphKeys.GLOBAL_VARIABLES) @@ -379,18 +389,21 @@ def _get_logits(): logging.info('will sort columns[len=%d] by name' % len(tmp_cols)) tmp_cols = sorted(tmp_cols, key=lambda x: x.name) for column in tmp_cols: - with variable_scope.variable_scope(None, default_name=column._var_scope_name): - tensor = column._get_dense_tensor(builder, weight_collections=weight_collections, trainable=trainable) + with variable_scope.variable_scope( + None, default_name=column._var_scope_name): + tensor = column._get_dense_tensor( + builder, weight_collections=weight_collections, trainable=trainable) num_elements = column._variable_shape.num_elements() batch_size = array_ops.shape(tensor)[0] - output_tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements)) + output_tensor = array_ops.reshape( + tensor, shape=(batch_size, num_elements)) output_tensors.append(output_tensor) if cols_to_vars is not None: # Retrieve any variables created (some _DenseColumn's don't create # variables, in which case an empty list is returned). cols_to_vars[column] = ops.get_collection( - ops.GraphKeys.GLOBAL_VARIABLES, - scope=variable_scope.get_variable_scope().name, + ops.GraphKeys.GLOBAL_VARIABLES, + scope=variable_scope.get_variable_scope().name, ) if cols_to_output_tensors is not None: cols_to_output_tensors[column] = output_tensor @@ -432,7 +445,8 @@ def _get_var_type(column): batch_sizes = [] for column in feature_columns: ordered_columns.append(column) - with variable_scope.variable_scope(None, default_name=column._var_scope_name): + with variable_scope.variable_scope( + None, default_name=column._var_scope_name): # for features which does not require embedding if 'Embedding' not in str(type(column)): dense_cols.append(column) @@ -454,6 +468,31 @@ def _get_var_type(column): if column.ev_params is not None: assert dynamic_variable is not None, 'sok is not installed' embedding_weights = dynamic_variable.DynamicVariable( + name='embedding_weights', + dimension=column.dimension, + initializer='random {"stddev":0.0025}', # column.initializer, + var_type=_get_var_type(column), + trainable=column.trainable and trainable, + dtype=dtypes.float32, + init_capacity=column.ev_params.init_capacity, + max_capacity=column.ev_params.max_capacity, + ) + else: + embedding_weights = variable_scope.get_variable( + name='embedding_weights', + shape=embedding_shape, + dtype=dtypes.float32, + initializer=column.initializer, + trainable=column.trainable and trainable, + partitioner=None, + collections=weight_collections, + ) + shared_weights[shared_name] = embedding_weights + else: + with ops.device(embedding_device): + if column.ev_params is not None: + assert dynamic_variable is not None, 'sok is not installed' + embedding_weights = dynamic_variable.DynamicVariable( name='embedding_weights', dimension=column.dimension, initializer='random {"stddev":0.0025}', # column.initializer, @@ -462,9 +501,9 @@ def _get_var_type(column): dtype=dtypes.float32, init_capacity=column.ev_params.init_capacity, max_capacity=column.ev_params.max_capacity, - ) - else: - embedding_weights = variable_scope.get_variable( + ) + else: + embedding_weights = variable_scope.get_variable( name='embedding_weights', shape=embedding_shape, dtype=dtypes.float32, @@ -472,31 +511,6 @@ def _get_var_type(column): trainable=column.trainable and trainable, partitioner=None, collections=weight_collections, - ) - shared_weights[shared_name] = embedding_weights - else: - with ops.device(embedding_device): - if column.ev_params is not None: - assert dynamic_variable is not None, 'sok is not installed' - embedding_weights = dynamic_variable.DynamicVariable( - name='embedding_weights', - dimension=column.dimension, - initializer='random {"stddev":0.0025}', # column.initializer, - var_type=_get_var_type(column), - trainable=column.trainable and trainable, - dtype=dtypes.float32, - init_capacity=column.ev_params.init_capacity, - max_capacity=column.ev_params.max_capacity, - ) - else: - embedding_weights = variable_scope.get_variable( - name='embedding_weights', - shape=embedding_shape, - dtype=dtypes.float32, - initializer=column.initializer, - trainable=column.trainable and trainable, - partitioner=None, - collections=weight_collections, ) lookup_embeddings.append(embedding_weights) output_id = len(output_tensors) @@ -514,8 +528,8 @@ def _get_var_type(column): elif 'ragged_ids' in features.keys(): if lookup_indices is None: lookup_indices = { - 'ragged_ids': features['ragged_ids'], - 'ragged_lens': features['ragged_lens'], + 'ragged_ids': features['ragged_ids'], + 'ragged_lens': features['ragged_lens'], } if 'ragged_wgts' in features: lookup_indices['ragged_wgts'] = features['ragged_wgts'] @@ -524,17 +538,17 @@ def _get_var_type(column): lookup_indices = [] with ops.device('/cpu:0'): sparse_tensors = column.categorical_column._get_sparse_tensors( - builder, - weight_collections=weight_collections, - trainable=trainable, + builder, + weight_collections=weight_collections, + trainable=trainable, ) lookup_indices.append(sparse_tensors.id_tensor) if sparse_tensors.weight_tensor is not None: lookup_wgts.append(sparse_tensors.weight_tensor) if cols_to_vars is not None: cols_to_vars[column] = ops.get_collection( - ops.GraphKeys.GLOBAL_VARIABLES, - scope=variable_scope.get_variable_scope().name, + ops.GraphKeys.GLOBAL_VARIABLES, + scope=variable_scope.get_variable_scope().name, ) if dense_cnt > 0: @@ -542,7 +556,8 @@ def _get_var_type(column): fea_dim_s = 0 for dense_output_id, dense_col in zip(dense_output_ids, dense_cols): fea_dim_e = fea_dim_s + dense_col.shape[0] - output_tensors[dense_output_id] = features['dense_fea'][:, fea_dim_s:fea_dim_e] + output_tensors[dense_output_id] = features[ + 'dense_fea'][:, fea_dim_s:fea_dim_e] fea_dim_s = fea_dim_e batch_sizes.append(array_ops.shape(features['dense_fea'])[0]) else: @@ -564,12 +579,12 @@ def _get_var_type(column): uniq_embed_cnt = len(set(lookup_embeddings)) assert uniq_embed_cnt == 1, 'only one uniq embed is support for packed inputs' outputs = embedding_parallel_lookup( - lookup_embeddings[0], - lookup_indices, - lookup_output_ids, - is_training, - output_tensors, - batch_size, + lookup_embeddings[0], + lookup_indices, + lookup_output_ids, + is_training, + output_tensors, + batch_size, ) else: if batch_size is None: @@ -580,11 +595,13 @@ def _get_var_type(column): batch_size = math_ops.reduce_max(all_indices) + 1 # group lookup_embeddings grouped_inputs = {} - for embedding, lookup_indice, output_id in zip(lookup_embeddings, lookup_indices, lookup_output_ids): + for embedding, lookup_indice, output_id in zip(lookup_embeddings, + lookup_indices, + lookup_output_ids): if embedding not in grouped_inputs: grouped_inputs[embedding] = { - 'lookup_indice': [lookup_indice], - 'output_id': [output_id], + 'lookup_indice': [lookup_indice], + 'output_id': [output_id], } else: grouped_inputs[embedding]['lookup_indice'].append(lookup_indice) @@ -594,12 +611,12 @@ def _get_var_type(column): lookup_indices = grouped_inputs[embedding]['lookup_indice'] output_ids = grouped_inputs[embedding]['output_id'] outputs = embedding_parallel_lookup( - embedding, - lookup_indices, - output_ids, - is_training, - output_tensors, - batch_size, + embedding, + lookup_indices, + output_ids, + is_training, + output_tensors, + batch_size, ) for output_tensor, col in zip(output_tensors, feature_columns): @@ -626,23 +643,25 @@ def _get_var_type(column): if from_template: return _get_logits() else: - with variable_scope.variable_scope(scope, default_name='input_layer', values=features.values()): + with variable_scope.variable_scope( + scope, default_name='input_layer', values=features.values()): if embedding_utils.is_embedding_parallel(): return _get_logits_embedding_parallel() else: - with conditional(embedding_utils.embedding_on_cpu(), ops.device('/cpu:0')): + with conditional(embedding_utils.embedding_on_cpu(), + ops.device('/cpu:0')): return _get_logits() def input_layer( - features, - feature_columns, - weight_collections=None, - trainable=True, - cols_to_vars=None, - cols_to_output_tensors=None, - feature_name_to_output_tensors=None, - is_training=True, + features, + feature_columns, + weight_collections=None, + trainable=True, + cols_to_vars=None, + cols_to_output_tensors=None, + feature_name_to_output_tensors=None, + is_training=True, ): """Returns a dense `Tensor` as input layer based on given `feature_columns`. @@ -701,14 +720,14 @@ def input_layer( ValueError: if an item in `feature_columns` is not a `_DenseColumn`. """ return _internal_input_layer( - features, - feature_columns, - weight_collections=weight_collections, - trainable=trainable, - cols_to_vars=cols_to_vars, - cols_to_output_tensors=cols_to_output_tensors, - feature_name_to_output_tensors=feature_name_to_output_tensors, - is_training=is_training, + features, + feature_columns, + weight_collections=weight_collections, + trainable=trainable, + cols_to_vars=cols_to_vars, + cols_to_output_tensors=cols_to_output_tensors, + feature_name_to_output_tensors=feature_name_to_output_tensors, + is_training=is_training, ) @@ -720,13 +739,13 @@ class InputLayer(object): """An object-oriented version of `input_layer` that reuses variables.""" def __init__( - self, - feature_columns, - weight_collections=None, - trainable=True, - cols_to_vars=None, - name='feature_column_input_layer', - create_scope_now=True, + self, + feature_columns, + weight_collections=None, + trainable=True, + cols_to_vars=None, + name='feature_column_input_layer', + create_scope_now=True, ): """See `input_layer`.""" self._feature_columns = feature_columns @@ -735,18 +754,17 @@ def __init__( self._cols_to_vars = cols_to_vars self._name = name self._input_layer_template = template.make_template( - self._name, _internal_input_layer, create_scope_now_=create_scope_now - ) + self._name, _internal_input_layer, create_scope_now_=create_scope_now) self._scope = self._input_layer_template.variable_scope def __call__(self, features): return self._input_layer_template( - features=features, - feature_columns=self._feature_columns, - weight_collections=self._weight_collections, - trainable=self._trainable, - cols_to_vars=None, - from_template=True, + features=features, + feature_columns=self._feature_columns, + weight_collections=self._weight_collections, + trainable=self._trainable, + cols_to_vars=None, + from_template=True, ) @property @@ -779,13 +797,13 @@ def weights(self): def linear_model( - features, - feature_columns, - units=1, - sparse_combiner='sum', - weight_collections=None, - trainable=True, - cols_to_vars=None, + features, + feature_columns, + units=1, + sparse_combiner='sum', + weight_collections=None, + trainable=True, + cols_to_vars=None, ): """Returns a linear prediction `Tensor` based on given `feature_columns`. @@ -905,12 +923,12 @@ def linear_model( with variable_scope.variable_scope(None, 'linear_model') as vs: model_name = _strip_leading_slashes(vs.name) linear_model_layer = _LinearModel( - feature_columns=feature_columns, - units=units, - sparse_combiner=sparse_combiner, - weight_collections=weight_collections, - trainable=trainable, - name=model_name, + feature_columns=feature_columns, + units=units, + sparse_combiner=sparse_combiner, + weight_collections=weight_collections, + trainable=trainable, + name=model_name, ) retval = linear_model_layer(features) if cols_to_vars is not None: @@ -947,16 +965,17 @@ class _FCLinearWrapper(base.Layer): """ def __init__( - self, - feature_column, - units=1, - sparse_combiner='sum', - weight_collections=None, - trainable=True, - name=None, - **kwargs, + self, + feature_column, + units=1, + sparse_combiner='sum', + weight_collections=None, + trainable=True, + name=None, + **kwargs, ): - super(_FCLinearWrapper, self).__init__(trainable=trainable, name=name, **kwargs) + super(_FCLinearWrapper, self).__init__( + trainable=trainable, name=name, **kwargs) self._feature_column = feature_column self._units = units self._sparse_combiner = sparse_combiner @@ -965,18 +984,18 @@ def __init__( def build(self, _): if isinstance(self._feature_column, _CategoricalColumn): weight = self.add_variable( - name='weights', - shape=(self._feature_column._num_buckets, self._units), - initializer=init_ops.zeros_initializer(), - trainable=self.trainable, + name='weights', + shape=(self._feature_column._num_buckets, self._units), + initializer=init_ops.zeros_initializer(), + trainable=self.trainable, ) else: num_elements = self._feature_column._variable_shape.num_elements() weight = self.add_variable( - name='weights', - shape=[num_elements, self._units], - initializer=init_ops.zeros_initializer(), - trainable=self.trainable, + name='weights', + shape=[num_elements, self._units], + initializer=init_ops.zeros_initializer(), + trainable=self.trainable, ) _add_to_collections(weight, self._weight_collections) self._weight_var = weight @@ -984,13 +1003,13 @@ def build(self, _): def call(self, builder): weighted_sum = _create_weighted_sum( - column=self._feature_column, - builder=builder, - units=self._units, - sparse_combiner=self._sparse_combiner, - weight_collections=self._weight_collections, - trainable=self.trainable, - weight_var=self._weight_var, + column=self._feature_column, + builder=builder, + units=self._units, + sparse_combiner=self._sparse_combiner, + weight_collections=self._weight_collections, + trainable=self.trainable, + weight_var=self._weight_var, ) return weighted_sum @@ -998,17 +1017,22 @@ def call(self, builder): class _BiasLayer(base.Layer): """A layer for the bias term.""" - def __init__(self, units=1, trainable=True, weight_collections=None, name=None, **kwargs): + def __init__(self, + units=1, + trainable=True, + weight_collections=None, + name=None, + **kwargs): super(_BiasLayer, self).__init__(trainable=trainable, name=name, **kwargs) self._units = units self._weight_collections = weight_collections def build(self, _): self._bias_variable = self.add_variable( - 'bias_weights', - shape=[self._units], - initializer=init_ops.zeros_initializer(), - trainable=self.trainable, + 'bias_weights', + shape=[self._units], + initializer=init_ops.zeros_initializer(), + trainable=self.trainable, ) _add_to_collections(self._bias_variable, self._weight_collections) self.built = True @@ -1018,7 +1042,8 @@ def call(self, _): def _get_expanded_variable_list(variable): - if isinstance(variable, variables.Variable) or resource_variable_ops.is_resource_variable(variable): + if isinstance(variable, variables.Variable + ) or resource_variable_ops.is_resource_variable(variable): return [variable] # Single variable case. else: # Must be a PartitionedVariable, so convert into a list. return list(variable) @@ -1035,14 +1060,14 @@ class _LinearModel(training.Model): """ def __init__( - self, - feature_columns, - units=1, - sparse_combiner='sum', - weight_collections=None, - trainable=True, - name=None, - **kwargs, + self, + feature_columns, + units=1, + sparse_combiner='sum', + weight_collections=None, + trainable=True, + name=None, + **kwargs, ): super(_LinearModel, self).__init__(name=name, **kwargs) self._feature_columns = _normalize_feature_columns(feature_columns) @@ -1054,28 +1079,29 @@ def __init__( column_layers = {} for column in sorted(self._feature_columns, key=lambda x: x.name): - with variable_scope.variable_scope(None, default_name=column._var_scope_name) as vs: + with variable_scope.variable_scope( + None, default_name=column._var_scope_name) as vs: # Having the fully expressed variable scope name ends up doubly # expressing the outer scope (scope with which this method was called) # in the name of the variable that would get created. column_name = _strip_leading_slashes(vs.name) column_layer = _FCLinearWrapper( - column, - units, - sparse_combiner, - self._weight_collections, - trainable, - column_name, - **kwargs, + column, + units, + sparse_combiner, + self._weight_collections, + trainable, + column_name, + **kwargs, ) column_layers[column_name] = column_layer self._column_layers = self._add_layers(column_layers) self._bias_layer = _BiasLayer( - units=units, - trainable=trainable, - weight_collections=self._weight_collections, - name='bias_layer', - **kwargs, + units=units, + trainable=trainable, + weight_collections=self._weight_collections, + name='bias_layer', + **kwargs, ) self._cols_to_vars = {} @@ -1092,8 +1118,8 @@ def call(self, features): for column in self._feature_columns: if not isinstance(column, (_DenseColumn, _CategoricalColumn)): raise ValueError( - 'Items of feature_columns must be either a ' '_DenseColumn or _CategoricalColumn. Given: {}'.format(column) - ) + 'Items of feature_columns must be either a ' + '_DenseColumn or _CategoricalColumn. Given: {}'.format(column)) weighted_sums = [] ordered_columns = [] builder = _LazyBuilder(features) @@ -1102,14 +1128,16 @@ def call(self, features): ordered_columns.append(column) weighted_sum = layer(builder) weighted_sums.append(weighted_sum) - self._cols_to_vars[column] = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, scope=layer.scope_name) + self._cols_to_vars[column] = ops.get_collection( + ops.GraphKeys.GLOBAL_VARIABLES, scope=layer.scope_name) _verify_static_batch_size_equality(weighted_sums, ordered_columns) - predictions_no_bias = math_ops.add_n(weighted_sums, name='weighted_sum_no_bias') + predictions_no_bias = math_ops.add_n( + weighted_sums, name='weighted_sum_no_bias') predictions = nn_ops.bias_add( - predictions_no_bias, - self._bias_layer(builder, scope=variable_scope.get_variable_scope()), - name='weighted_sum', + predictions_no_bias, + self._bias_layer(builder, scope=variable_scope.get_variable_scope()), + name='weighted_sum', ) bias = self._bias_layer.variables[0] self._cols_to_vars['bias'] = _get_expanded_variable_list(bias) @@ -1159,7 +1187,8 @@ def _transform_features(features, feature_columns): """ feature_columns = _normalize_feature_columns(feature_columns) outputs = {} - with ops.name_scope(None, default_name='transform_features', values=features.values()): + with ops.name_scope( + None, default_name='transform_features', values=features.values()): builder = _LazyBuilder(features) for column in sorted(feature_columns, key=lambda x: x.name): with ops.name_scope(None, default_name=column.name): @@ -1215,26 +1244,26 @@ def make_parse_example_spec(feature_columns): result = {} for column in feature_columns: if not isinstance(column, _FeatureColumn): - raise ValueError('All feature_columns must be _FeatureColumn instances. ' 'Given: {}'.format(column)) + raise ValueError('All feature_columns must be _FeatureColumn instances. ' + 'Given: {}'.format(column)) config = column._parse_example_spec for key, value in six.iteritems(config): if key in result and value != result[key]: - raise ValueError( - 'feature_columns contain different parse_spec for key ' '{}. Given {} and {}'.format(key, value, result[key]) - ) + raise ValueError('feature_columns contain different parse_spec for key ' + '{}. Given {} and {}'.format(key, value, result[key])) result.update(config) return result def _embedding_column( - categorical_column, - dimension, - combiner='mean', - initializer=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True, + categorical_column, + dimension, + combiner='mean', + initializer=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True, ): """`_DenseColumn` that converts from sparse, categorical input. @@ -1310,40 +1339,46 @@ def model_fn(features, ...): if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError('Must specify both `ckpt_to_load_from` and ' '`tensor_name_in_ckpt` or none of them.') + raise ValueError('Must specify both `ckpt_to_load_from` and ' + '`tensor_name_in_ckpt` or none of them.') if (initializer is not None) and (not callable(initializer)): - raise ValueError( - 'initializer must be callable if specified. ' 'Embedding of column_name: {}'.format(categorical_column.name) - ) + raise ValueError('initializer must be callable if specified. ' + 'Embedding of column_name: {}'.format( + categorical_column.name)) if initializer is None: - initializer = init_ops.truncated_normal_initializer(mean=0.0, stddev=0.01 / math.sqrt(dimension)) + initializer = init_ops.truncated_normal_initializer( + mean=0.0, stddev=0.01 / math.sqrt(dimension)) embedding_shape = categorical_column._num_buckets, dimension def _creator(weight_collections, scope): embedding_column_layer = _EmbeddingColumnLayer( - embedding_shape=embedding_shape, - initializer=initializer, - weight_collections=weight_collections, - trainable=trainable, - name='embedding_column_layer', + embedding_shape=embedding_shape, + initializer=initializer, + weight_collections=weight_collections, + trainable=trainable, + name='embedding_column_layer', ) return embedding_column_layer(None, scope=scope) return _EmbeddingColumn( - categorical_column=categorical_column, - dimension=dimension, - combiner=combiner, - layer_creator=_creator, - ckpt_to_load_from=ckpt_to_load_from, - tensor_name_in_ckpt=tensor_name_in_ckpt, - max_norm=max_norm, - trainable=trainable, + categorical_column=categorical_column, + dimension=dimension, + combiner=combiner, + layer_creator=_creator, + ckpt_to_load_from=ckpt_to_load_from, + tensor_name_in_ckpt=tensor_name_in_ckpt, + max_norm=max_norm, + trainable=trainable, ) -def _numeric_column(key, shape=(1,), default_value=None, dtype=dtypes.float32, normalizer_fn=None): +def _numeric_column(key, + shape=(1,), + default_value=None, + dtype=dtypes.float32, + normalizer_fn=None): """Represents real valued or numerical features. Example: @@ -1397,19 +1432,21 @@ def _numeric_column(key, shape=(1,), default_value=None, dtype=dtypes.float32, n """ shape = _check_shape(shape, key) if not (dtype.is_integer or dtype.is_floating): - raise ValueError('dtype must be convertible to float. ' 'dtype: {}, key: {}'.format(dtype, key)) + raise ValueError('dtype must be convertible to float. ' + 'dtype: {}, key: {}'.format(dtype, key)) default_value = fc_utils.check_default_value(shape, default_value, dtype, key) if normalizer_fn is not None and not callable(normalizer_fn): - raise TypeError('normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) + raise TypeError( + 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) fc_utils.assert_key_is_string(key) return _NumericColumn( - key, - shape=shape, - default_value=default_value, - dtype=dtype, - normalizer_fn=normalizer_fn, + key, + shape=shape, + default_value=default_value, + dtype=dtype, + normalizer_fn=normalizer_fn, ) @@ -1481,11 +1518,13 @@ def _bucketized_column(source_column, boundaries): """ if not isinstance(source_column, _NumericColumn): raise ValueError( - 'source_column must be a column generated with numeric_column(). ' 'Given: {}'.format(source_column) - ) + 'source_column must be a column generated with numeric_column(). ' + 'Given: {}'.format(source_column)) if len(source_column.shape) > 1: - raise ValueError('source_column must be one-dimensional column. ' 'Given: {}'.format(source_column)) - if not boundaries or not (isinstance(boundaries, list) or isinstance(boundaries, tuple)): + raise ValueError('source_column must be one-dimensional column. ' + 'Given: {}'.format(source_column)) + if not boundaries or not (isinstance(boundaries, list) or + isinstance(boundaries, tuple)): raise ValueError('boundaries must be a sorted list.') for i in range(len(boundaries) - 1): if boundaries[i] >= boundaries[i + 1]: @@ -1493,7 +1532,9 @@ def _bucketized_column(source_column, boundaries): return _BucketizedColumn(source_column, tuple(boundaries)) -def _categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dtypes.string): +def _categorical_column_with_hash_bucket(key, + hash_bucket_size, + dtype=dtypes.string): """Represents sparse feature where ids are set by hashing. Use this when your sparse features are in string or integer format, and you @@ -1539,9 +1580,9 @@ def _categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dtypes.str raise ValueError('hash_bucket_size must be set. ' 'key: {}'.format(key)) if hash_bucket_size < 1: - raise ValueError( - 'hash_bucket_size must be at least 1. ' 'hash_bucket_size: {}, key: {}'.format(hash_bucket_size, key) - ) + raise ValueError('hash_bucket_size must be at least 1. ' + 'hash_bucket_size: {}, key: {}'.format( + hash_bucket_size, key)) fc_utils.assert_key_is_string(key) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) @@ -1550,12 +1591,12 @@ def _categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dtypes.str def _categorical_column_with_vocabulary_file( - key, - vocabulary_file, - vocabulary_size=None, - num_oov_buckets=0, - default_value=None, - dtype=dtypes.string, + key, + vocabulary_file, + vocabulary_size=None, + num_oov_buckets=0, + default_value=None, + dtype=dtypes.string, ): """A `_CategoricalColumn` with a vocabulary file. @@ -1645,10 +1686,11 @@ def _categorical_column_with_vocabulary_file( with gfile.GFile(vocabulary_file) as f: vocabulary_size = sum(1 for _ in f) logging.info( - 'vocabulary_size = %d in %s is inferred from the number of elements ' 'in the vocabulary_file %s.', - vocabulary_size, - key, - vocabulary_file, + 'vocabulary_size = %d in %s is inferred from the number of elements ' + 'in the vocabulary_file %s.', + vocabulary_size, + key, + vocabulary_file, ) # `vocabulary_size` isn't required for lookup, but it is for `_num_buckets`. @@ -1656,22 +1698,29 @@ def _categorical_column_with_vocabulary_file( raise ValueError('Invalid vocabulary_size in {}.'.format(key)) if num_oov_buckets: if default_value is not None: - raise ValueError("Can't specify both num_oov_buckets and default_value in {}.".format(key)) + raise ValueError( + "Can't specify both num_oov_buckets and default_value in {}.".format( + key)) if num_oov_buckets < 0: - raise ValueError('Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key)) + raise ValueError('Invalid num_oov_buckets {} in {}.'.format( + num_oov_buckets, key)) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) return _VocabularyFileCategoricalColumn( - key=key, - vocabulary_file=vocabulary_file, - vocabulary_size=vocabulary_size, - num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, - default_value=-1 if default_value is None else default_value, - dtype=dtype, + key=key, + vocabulary_file=vocabulary_file, + vocabulary_size=vocabulary_size, + num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, + default_value=-1 if default_value is None else default_value, + dtype=dtype, ) -def _categorical_column_with_vocabulary_list(key, vocabulary_list, dtype=None, default_value=-1, num_oov_buckets=0): +def _categorical_column_with_vocabulary_list(key, + vocabulary_list, + dtype=None, + default_value=-1, + num_oov_buckets=0): """A `_CategoricalColumn` with in-memory vocabulary. Use this when your inputs are in string or integer format, and you have an @@ -1748,31 +1797,39 @@ def _categorical_column_with_vocabulary_list(key, vocabulary_list, dtype=None, d ValueError: if `dtype` is not integer or string. """ if (vocabulary_list is None) or (len(vocabulary_list) < 1): - raise ValueError('vocabulary_list {} must be non-empty, column_name: {}'.format(vocabulary_list, key)) + raise ValueError( + 'vocabulary_list {} must be non-empty, column_name: {}'.format( + vocabulary_list, key)) if len(set(vocabulary_list)) != len(vocabulary_list): - raise ValueError('Duplicate keys in vocabulary_list {}, column_name: {}'.format(vocabulary_list, key)) + raise ValueError( + 'Duplicate keys in vocabulary_list {}, column_name: {}'.format( + vocabulary_list, key)) vocabulary_dtype = dtypes.as_dtype(np.array(vocabulary_list).dtype) if num_oov_buckets: if default_value != -1: - raise ValueError("Can't specify both num_oov_buckets and default_value in {}.".format(key)) + raise ValueError( + "Can't specify both num_oov_buckets and default_value in {}.".format( + key)) if num_oov_buckets < 0: - raise ValueError('Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key)) - fc_utils.assert_string_or_int(vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key)) + raise ValueError('Invalid num_oov_buckets {} in {}.'.format( + num_oov_buckets, key)) + fc_utils.assert_string_or_int( + vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key)) if dtype is None: dtype = vocabulary_dtype elif dtype.is_integer != vocabulary_dtype.is_integer: raise ValueError( - 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format(dtype, vocabulary_dtype, key) - ) + 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format( + dtype, vocabulary_dtype, key)) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) return _VocabularyListCategoricalColumn( - key=key, - vocabulary_list=tuple(vocabulary_list), - dtype=dtype, - default_value=default_value, - num_oov_buckets=num_oov_buckets, + key=key, + vocabulary_list=tuple(vocabulary_list), + dtype=dtype, + default_value=default_value, + num_oov_buckets=num_oov_buckets, ) @@ -1831,11 +1888,16 @@ def _categorical_column_with_identity(key, num_buckets, default_value=None): ValueError: if `default_value` is not in range `[0, num_buckets)`. """ if num_buckets < 1: - raise ValueError('num_buckets {} < 1, column_name {}'.format(num_buckets, key)) - if (default_value is not None) and ((default_value < 0) or (default_value >= num_buckets)): - raise ValueError('default_value {} not in range [0, {}), column_name {}'.format(default_value, num_buckets, key)) + raise ValueError('num_buckets {} < 1, column_name {}'.format( + num_buckets, key)) + if (default_value is not None) and ((default_value < 0) or + (default_value >= num_buckets)): + raise ValueError( + 'default_value {} not in range [0, {}), column_name {}'.format( + default_value, num_buckets, key)) fc_utils.assert_key_is_string(key) - return _IdentityCategoricalColumn(key=key, num_buckets=num_buckets, default_value=default_value) + return _IdentityCategoricalColumn( + key=key, num_buckets=num_buckets, default_value=default_value) def _indicator_column(categorical_column): @@ -1872,7 +1934,9 @@ def _indicator_column(categorical_column): return _IndicatorColumn(categorical_column) -def _weighted_categorical_column(categorical_column, weight_feature_key, dtype=dtypes.float32): +def _weighted_categorical_column(categorical_column, + weight_feature_key, + dtype=dtypes.float32): """Applies weight values to a `_CategoricalColumn`. Use this when each of your sparse inputs has both an ID and a value. For @@ -1940,9 +2004,9 @@ def _weighted_categorical_column(categorical_column, weight_feature_key, dtype=d if (dtype is None) or not (dtype.is_integer or dtype.is_floating): raise ValueError('dtype {} is not convertible to float.'.format(dtype)) return _WeightedCategoricalColumn( - categorical_column=categorical_column, - weight_feature_key=weight_feature_key, - dtype=dtype, + categorical_column=categorical_column, + weight_feature_key=weight_feature_key, + dtype=dtype, ) @@ -2051,23 +2115,25 @@ def _crossed_column(keys, hash_bucket_size, hash_key=None): ValueError: If `hash_bucket_size < 1`. """ if not hash_bucket_size or hash_bucket_size < 1: - raise ValueError('hash_bucket_size must be > 1. ' 'hash_bucket_size: {}'.format(hash_bucket_size)) + raise ValueError('hash_bucket_size must be > 1. ' + 'hash_bucket_size: {}'.format(hash_bucket_size)) if not keys or len(keys) < 2: - raise ValueError('keys must be a list with length > 1. Given: {}'.format(keys)) + raise ValueError( + 'keys must be a list with length > 1. Given: {}'.format(keys)) for key in keys: - if not isinstance(key, six.string_types) and not isinstance(key, _CategoricalColumn): + if not isinstance(key, six.string_types) and not isinstance( + key, _CategoricalColumn): raise ValueError( - 'Unsupported key type. All keys must be either string, or ' - 'categorical column except _HashedCategoricalColumn. ' - 'Given: {}'.format(key) - ) + 'Unsupported key type. All keys must be either string, or ' + 'categorical column except _HashedCategoricalColumn. ' + 'Given: {}'.format(key)) if isinstance(key, _HashedCategoricalColumn): raise ValueError( - 'categorical_column_with_hash_bucket is not supported for crossing. ' - 'Hashing before crossing will increase probability of collision. ' - 'Instead, use the feature name as a string. Given: {}'.format(key) - ) - return _CrossedColumn(keys=tuple(keys), hash_bucket_size=hash_bucket_size, hash_key=hash_key) + 'categorical_column_with_hash_bucket is not supported for crossing. ' + 'Hashing before crossing will increase probability of collision. ' + 'Instead, use the feature name as a string. Given: {}'.format(key)) + return _CrossedColumn( + keys=tuple(keys), hash_bucket_size=hash_bucket_size, hash_key=hash_key) # TODO(rohanj): Clearly define semantics of this layer. @@ -2075,13 +2141,13 @@ class _EmbeddingColumnLayer(base.Layer): """A layer that stores all the state required for a embedding column.""" def __init__( - self, - embedding_shape, - initializer, - weight_collections=None, - trainable=True, - name=None, - **kwargs, + self, + embedding_shape, + initializer, + weight_collections=None, + trainable=True, + name=None, + **kwargs, ): """Constructor. @@ -2097,7 +2163,8 @@ def __init__( name: Name of the layer **kwargs: keyword named properties. """ - super(_EmbeddingColumnLayer, self).__init__(trainable=trainable, name=name, **kwargs) + super(_EmbeddingColumnLayer, self).__init__( + trainable=trainable, name=name, **kwargs) self._embedding_shape = embedding_shape self._initializer = initializer self._weight_collections = weight_collections @@ -2113,11 +2180,11 @@ def set_weight_collections(self, weight_collections): def build(self, _): self._embedding_weight_var = self.add_variable( - name='embedding_weights', - shape=self._embedding_shape, - dtype=dtypes.float32, - initializer=self._initializer, - trainable=self.trainable, + name='embedding_weights', + shape=self._embedding_shape, + dtype=dtypes.float32, + initializer=self._initializer, + trainable=self.trainable, ) if self._weight_collections and not context.executing_eagerly(): _add_to_collections(self._embedding_weight_var, self._weight_collections) @@ -2250,39 +2317,45 @@ def input_layer(features, feature_columns, ...): def _create_weighted_sum( - column, - builder, - units, - sparse_combiner, - weight_collections, - trainable, - weight_var=None, + column, + builder, + units, + sparse_combiner, + weight_collections, + trainable, + weight_var=None, ): """Creates a weighted sum for a dense/categorical column for linear_model.""" if isinstance(column, _CategoricalColumn): return _create_categorical_column_weighted_sum( - column=column, - builder=builder, - units=units, - sparse_combiner=sparse_combiner, - weight_collections=weight_collections, - trainable=trainable, - weight_var=weight_var, + column=column, + builder=builder, + units=units, + sparse_combiner=sparse_combiner, + weight_collections=weight_collections, + trainable=trainable, + weight_var=weight_var, ) else: return _create_dense_column_weighted_sum( - column=column, - builder=builder, - units=units, - weight_collections=weight_collections, - trainable=trainable, - weight_var=weight_var, + column=column, + builder=builder, + units=units, + weight_collections=weight_collections, + trainable=trainable, + weight_var=weight_var, ) -def _create_dense_column_weighted_sum(column, builder, units, weight_collections, trainable, weight_var=None): +def _create_dense_column_weighted_sum(column, + builder, + units, + weight_collections, + trainable, + weight_var=None): """Create a weighted sum of a dense column for linear_model.""" - tensor = column._get_dense_tensor(builder, weight_collections=weight_collections, trainable=trainable) + tensor = column._get_dense_tensor( + builder, weight_collections=weight_collections, trainable=trainable) num_elements = column._variable_shape.num_elements() batch_size = array_ops.shape(tensor)[0] tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements)) @@ -2290,11 +2363,11 @@ def _create_dense_column_weighted_sum(column, builder, units, weight_collections weight = weight_var else: weight = variable_scope.get_variable( - name='weights', - shape=[num_elements, units], - initializer=init_ops.zeros_initializer(), - trainable=trainable, - collections=weight_collections, + name='weights', + shape=[num_elements, units], + initializer=init_ops.zeros_initializer(), + trainable=trainable, + collections=weight_collections, ) return math_ops.matmul(tensor, weight, name='weighted_sum') @@ -2308,14 +2381,18 @@ class _CategoricalColumn(_FeatureColumn): A categorical feature typically handled with a `tf.SparseTensor` of IDs. """ - IdWeightPair = collections.namedtuple('IdWeightPair', ['id_tensor', 'weight_tensor']) + IdWeightPair = collections.namedtuple('IdWeightPair', + ['id_tensor', 'weight_tensor']) @abc.abstractproperty def _num_buckets(self): """Returns number of buckets in this sparse feature.""" @abc.abstractmethod - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): """Returns an IdWeightPair. `IdWeightPair` is a pair of `SparseTensor`s which represents ids and @@ -2339,13 +2416,13 @@ def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): def _create_categorical_column_weighted_sum( - column, - builder, - units, - sparse_combiner, - weight_collections, - trainable, - weight_var=None, + column, + builder, + units, + sparse_combiner, + weight_collections, + trainable, + weight_var=None, ): """Create a weighted sum of a categorical column for linear_model. @@ -2373,38 +2450,46 @@ def _create_categorical_column_weighted_sum( For both cases, we can implement weighted sum via embedding_lookup with sparse_combiner = "sum". """ - sparse_tensors = column._get_sparse_tensors(builder, weight_collections=weight_collections, trainable=trainable) - id_tensor = sparse_ops.sparse_reshape(sparse_tensors.id_tensor, [array_ops.shape(sparse_tensors.id_tensor)[0], -1]) + sparse_tensors = column._get_sparse_tensors( + builder, weight_collections=weight_collections, trainable=trainable) + id_tensor = sparse_ops.sparse_reshape( + sparse_tensors.id_tensor, + [array_ops.shape(sparse_tensors.id_tensor)[0], -1]) weight_tensor = sparse_tensors.weight_tensor if weight_tensor is not None: - weight_tensor = sparse_ops.sparse_reshape(weight_tensor, [array_ops.shape(weight_tensor)[0], -1]) + weight_tensor = sparse_ops.sparse_reshape( + weight_tensor, [array_ops.shape(weight_tensor)[0], -1]) if weight_var is not None: weight = weight_var else: weight = variable_scope.get_variable( - name='weights', - shape=(column._num_buckets, units), - initializer=init_ops.zeros_initializer(), - trainable=trainable, - collections=weight_collections, + name='weights', + shape=(column._num_buckets, units), + initializer=init_ops.zeros_initializer(), + trainable=trainable, + collections=weight_collections, ) return embedding_ops.safe_embedding_lookup_sparse( - weight, - id_tensor, - sparse_weights=weight_tensor, - combiner=sparse_combiner, - name='weighted_sum', + weight, + id_tensor, + sparse_weights=weight_tensor, + combiner=sparse_combiner, + name='weighted_sum', ) class _SequenceDenseColumn(_FeatureColumn): """Represents dense sequence data.""" - TensorSequenceLengthPair = collections.namedtuple('TensorSequenceLengthPair', ['dense_tensor', 'sequence_length']) + TensorSequenceLengthPair = collections.namedtuple( + 'TensorSequenceLengthPair', ['dense_tensor', 'sequence_length']) @abc.abstractmethod - def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): + def _get_sequence_dense_tensor(self, + inputs, + weight_collections=None, + trainable=None): """Returns a `TensorSequenceLengthPair`.""" @@ -2482,7 +2567,8 @@ def get(self, key): raise ValueError('Feature {} is not in features dictionary.'.format(key)) if not isinstance(key, _FeatureColumn): - raise TypeError('"key" must be either a "str" or "_FeatureColumn". ' 'Provided: {}'.format(key)) + raise TypeError('"key" must be either a "str" or "_FeatureColumn". ' + 'Provided: {}'.format(key)) column = key logging.debug('Transforming feature_column %s.', column) @@ -2514,34 +2600,37 @@ def _get_raw_feature_as_tensor(self, key): if 'RaggedTensor' in str(type(raw_feature)): return raw_feature - feature_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(raw_feature) + feature_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( + raw_feature) def expand_dims(input_tensor): # Input_tensor must have rank 1. if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): - return sparse_ops.sparse_reshape(input_tensor, [array_ops.shape(input_tensor)[0], 1]) + return sparse_ops.sparse_reshape(input_tensor, + [array_ops.shape(input_tensor)[0], 1]) else: return array_ops.expand_dims(input_tensor, -1) rank = feature_tensor.get_shape().ndims if rank is not None: if rank == 0: - raise ValueError('Feature (key: {}) cannot have rank 0. Give: {}'.format(key, feature_tensor)) + raise ValueError( + 'Feature (key: {}) cannot have rank 0. Give: {}'.format( + key, feature_tensor)) return feature_tensor if rank != 1 else expand_dims(feature_tensor) # Handle dynamic rank. - with ops.control_dependencies( - [ + with ops.control_dependencies([ check_ops.assert_positive( - array_ops.rank(feature_tensor), - message='Feature (key: {}) cannot have rank 0. Given: {}'.format(key, feature_tensor), + array_ops.rank(feature_tensor), + message='Feature (key: {}) cannot have rank 0. Given: {}'.format( + key, feature_tensor), ) - ] - ): + ]): return control_flow_ops.cond( - math_ops.equal(1, array_ops.rank(feature_tensor)), - lambda: expand_dims(feature_tensor), - lambda: feature_tensor, + math_ops.equal(1, array_ops.rank(feature_tensor)), + lambda: expand_dims(feature_tensor), + lambda: feature_tensor, ) @@ -2576,16 +2665,17 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): Raises: ValueError: when `input_tensor`'s rank is `None`. """ - input_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(input_tensor) + input_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( + input_tensor) if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): return input_tensor with ops.name_scope( - None, - 'to_sparse_input', - ( - input_tensor, - ignore_value, - ), + None, + 'to_sparse_input', + ( + input_tensor, + ignore_value, + ), ): if ignore_value is None: if input_tensor.dtype == dtypes.string: @@ -2598,12 +2688,15 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): # constructing a new numpy object of the given type, which yields the # default value for that type. ignore_value = input_tensor.dtype.as_numpy_dtype() - ignore_value = math_ops.cast(ignore_value, input_tensor.dtype, name='ignore_value') - indices = array_ops.where(math_ops.not_equal(input_tensor, ignore_value), name='indices') + ignore_value = math_ops.cast( + ignore_value, input_tensor.dtype, name='ignore_value') + indices = array_ops.where( + math_ops.not_equal(input_tensor, ignore_value), name='indices') return sparse_tensor_lib.SparseTensor( - indices=indices, - values=array_ops.gather_nd(input_tensor, indices, name='values'), - dense_shape=array_ops.shape(input_tensor, out_type=dtypes.int64, name='dense_shape'), + indices=indices, + values=array_ops.gather_nd(input_tensor, indices, name='values'), + dense_shape=array_ops.shape( + input_tensor, out_type=dtypes.int64, name='dense_shape'), ) @@ -2634,29 +2727,29 @@ def _normalize_feature_columns(feature_columns): for column in feature_columns: if not isinstance(column, _FeatureColumn): - raise ValueError( - 'Items of feature_columns must be a _FeatureColumn. ' 'Given (type {}): {}.'.format(type(column), column) - ) + raise ValueError('Items of feature_columns must be a _FeatureColumn. ' + 'Given (type {}): {}.'.format(type(column), column)) if not feature_columns: raise ValueError('feature_columns must not be empty.') name_to_column = {} for column in feature_columns: if column.name in name_to_column: - raise ValueError( - 'Duplicate feature column name found for columns: {} ' - 'and {}. This usually means that these columns refer to ' - 'same base feature. Either one must be discarded or a ' - 'duplicated but renamed item must be inserted in ' - 'features dict.'.format(column, name_to_column[column.name]) - ) + raise ValueError('Duplicate feature column name found for columns: {} ' + 'and {}. This usually means that these columns refer to ' + 'same base feature. Either one must be discarded or a ' + 'duplicated but renamed item must be inserted in ' + 'features dict.'.format(column, + name_to_column[column.name])) name_to_column[column.name] = column return feature_columns class _NumericColumn( - _DenseColumn, - collections.namedtuple('_NumericColumn', ['key', 'shape', 'default_value', 'dtype', 'normalizer_fn']), + _DenseColumn, + collections.namedtuple( + '_NumericColumn', + ['key', 'shape', 'default_value', 'dtype', 'normalizer_fn']), ): """See `numeric_column`.""" @@ -2666,15 +2759,18 @@ def name(self): @property def _parse_example_spec(self): - return {self.key: parsing_ops.FixedLenFeature(self.shape, self.dtype, self.default_value)} + return { + self.key: + parsing_ops.FixedLenFeature(self.shape, self.dtype, + self.default_value) + } def _transform_feature(self, inputs): input_tensor = inputs.get(self.key) if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): raise ValueError( - 'The corresponding Tensor of numerical column must be a Tensor. ' - 'SparseTensor is not supported. key: {}'.format(self.key) - ) + 'The corresponding Tensor of numerical column must be a Tensor. ' + 'SparseTensor is not supported. key: {}'.format(self.key)) if self.normalizer_fn is not None: input_tensor = self.normalizer_fn(input_tensor) return math_ops.cast(input_tensor, dtypes.float32) @@ -2706,9 +2802,10 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): class _BucketizedColumn( - _DenseColumn, - _CategoricalColumn, - collections.namedtuple('_BucketizedColumn', ['source_column', 'boundaries']), + _DenseColumn, + _CategoricalColumn, + collections.namedtuple('_BucketizedColumn', + ['source_column', 'boundaries']), ): """See `bucketized_column`.""" @@ -2730,17 +2827,18 @@ def _transform_feature(self, inputs): @property def _variable_shape(self): - return tensor_shape.TensorShape(tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) + return tensor_shape.TensorShape( + tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable input_tensor = inputs.get(self) return array_ops.one_hot( - indices=math_ops.cast(input_tensor, dtypes.int64), - depth=len(self.boundaries) + 1, - on_value=1.0, - off_value=0.0, + indices=math_ops.cast(input_tensor, dtypes.int64), + depth=len(self.boundaries) + 1, + on_value=1.0, + off_value=0.0, ) @property @@ -2748,7 +2846,10 @@ def _num_buckets(self): # By construction, source_column is always one-dimensional. return (len(self.boundaries) + 1) * self.source_column.shape[0] - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): """Converts dense inputs to SparseTensor so downstream code can use it.""" input_tensor = inputs.get(self) batch_size = array_ops.shape(input_tensor)[0] @@ -2756,39 +2857,43 @@ def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): source_dimension = self.source_column.shape[0] i1 = array_ops.reshape( - array_ops.tile( - array_ops.expand_dims(math_ops.range(0, batch_size), 1), - [1, source_dimension], - ), - (-1,), + array_ops.tile( + array_ops.expand_dims(math_ops.range(0, batch_size), 1), + [1, source_dimension], + ), + (-1,), ) i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size]) # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets - bucket_indices = array_ops.reshape(input_tensor, (-1,)) + (len(self.boundaries) + 1) * i2 - - indices = math_ops.cast(array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64) - dense_shape = math_ops.cast(array_ops.stack([batch_size, source_dimension]), dtypes.int64) - sparse_tensor = sparse_tensor_lib.SparseTensor(indices=indices, values=bucket_indices, dense_shape=dense_shape) + bucket_indices = array_ops.reshape(input_tensor, + (-1,)) + (len(self.boundaries) + 1) * i2 + + indices = math_ops.cast( + array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64) + dense_shape = math_ops.cast( + array_ops.stack([batch_size, source_dimension]), dtypes.int64) + sparse_tensor = sparse_tensor_lib.SparseTensor( + indices=indices, values=bucket_indices, dense_shape=dense_shape) return _CategoricalColumn.IdWeightPair(sparse_tensor, None) class _EmbeddingColumn( - _DenseColumn, - _SequenceDenseColumn, - collections.namedtuple( - '_EmbeddingColumn', - ( - 'categorical_column', - 'dimension', - 'combiner', - 'layer_creator', - 'ckpt_to_load_from', - 'tensor_name_in_ckpt', - 'max_norm', - 'trainable', + _DenseColumn, + _SequenceDenseColumn, + collections.namedtuple( + '_EmbeddingColumn', + ( + 'categorical_column', + 'dimension', + 'combiner', + 'layer_creator', + 'ckpt_to_load_from', + 'tensor_name_in_ckpt', + 'max_norm', + 'trainable', + ), ), - ), ): """See `embedding_column`.""" @@ -2811,65 +2916,77 @@ def _variable_shape(self): self._shape = tensor_shape.TensorShape([self.dimension]) return self._shape - def _get_dense_tensor_internal(self, inputs, weight_collections=None, trainable=None): + def _get_dense_tensor_internal(self, + inputs, + weight_collections=None, + trainable=None): """Private method that follows the signature of _get_dense_tensor.""" # Get sparse IDs and weights. sparse_tensors = self.categorical_column._get_sparse_tensors( - inputs, weight_collections=weight_collections, trainable=trainable - ) + inputs, weight_collections=weight_collections, trainable=trainable) sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor embedding_weights = self.layer_creator( - weight_collections=weight_collections, - scope=variable_scope.get_variable_scope(), + weight_collections=weight_collections, + scope=variable_scope.get_variable_scope(), ) if self.ckpt_to_load_from is not None: to_restore = embedding_weights if isinstance(to_restore, variables.PartitionedVariable): to_restore = to_restore._get_variable_list() - checkpoint_utils.init_from_checkpoint(self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) + checkpoint_utils.init_from_checkpoint( + self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) # Return embedding lookup result. return embedding_ops.safe_embedding_lookup_sparse( - embedding_weights=embedding_weights, - sparse_ids=sparse_ids, - sparse_weights=sparse_weights, - combiner=self.combiner, - name='%s_weights' % self.name, - max_norm=self.max_norm, + embedding_weights=embedding_weights, + sparse_ids=sparse_ids, + sparse_weights=sparse_weights, + combiner=self.combiner, + name='%s_weights' % self.name, + max_norm=self.max_norm, ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): if isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must not be of type _SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use input_layer, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'sequence_input_layer instead of input_layer. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) - return self._get_dense_tensor_internal(inputs=inputs, weight_collections=weight_collections, trainable=trainable) - - def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): + 'In embedding_column: {}. ' + 'categorical_column must not be of type _SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use input_layer, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'sequence_input_layer instead of input_layer. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) + return self._get_dense_tensor_internal( + inputs=inputs, + weight_collections=weight_collections, + trainable=trainable) + + def _get_sequence_dense_tensor(self, + inputs, + weight_collections=None, + trainable=None): if not isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must be of type _SequenceCategoricalColumn ' - 'to use sequence_input_layer. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) + 'In embedding_column: {}. ' + 'categorical_column must be of type _SequenceCategoricalColumn ' + 'to use sequence_input_layer. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) dense_tensor = self._get_dense_tensor_internal( - inputs=inputs, weight_collections=weight_collections, trainable=trainable - ) + inputs=inputs, + weight_collections=weight_collections, + trainable=trainable) sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) - sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) - return _SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) + sequence_length = fc_utils.sequence_length_from_sparse_tensor( + sparse_tensors.id_tensor) + return _SequenceDenseColumn.TensorSequenceLengthPair( + dense_tensor=dense_tensor, sequence_length=sequence_length) def _get_graph_for_variable(var): @@ -2880,24 +2997,24 @@ def _get_graph_for_variable(var): class _SharedEmbeddingColumn( - _DenseColumn, - _SequenceDenseColumn, - collections.namedtuple( - '_SharedEmbeddingColumn', - ( - 'categorical_column', - 'dimension', - 'combiner', - 'initializer', - 'shared_embedding_collection_name', - 'ckpt_to_load_from', - 'tensor_name_in_ckpt', - 'max_norm', - 'trainable', - 'partitioner', - 'ev_params', + _DenseColumn, + _SequenceDenseColumn, + collections.namedtuple( + '_SharedEmbeddingColumn', + ( + 'categorical_column', + 'dimension', + 'combiner', + 'initializer', + 'shared_embedding_collection_name', + 'ckpt_to_load_from', + 'tensor_name_in_ckpt', + 'max_norm', + 'trainable', + 'partitioner', + 'ev_params', + ), ), - ), ): """See `embedding_column`.""" @@ -2928,7 +3045,10 @@ def _variable_shape(self): self._shape = tensor_shape.TensorShape([self.dimension]) return self._shape - def _get_dense_tensor_internal(self, inputs, weight_collections=None, trainable=None): + def _get_dense_tensor_internal(self, + inputs, + weight_collections=None, + trainable=None): """Private method that follows the signature of _get_dense_tensor.""" # This method is called from a variable_scope with name _var_scope_name, # which is shared among all shared embeddings. Open a name_scope here, so @@ -2936,130 +3056,143 @@ def _get_dense_tensor_internal(self, inputs, weight_collections=None, trainable= with ops.name_scope(None, default_name=self.name): # Get sparse IDs and weights. sparse_tensors = self.categorical_column._get_sparse_tensors( - inputs, weight_collections=weight_collections, trainable=trainable - ) + inputs, weight_collections=weight_collections, trainable=trainable) sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor embedding_shape = (self.categorical_column._num_buckets, self.dimension) - shared_embedding_collection = ops.get_collection(self.shared_embedding_collection_name) + shared_embedding_collection = ops.get_collection( + self.shared_embedding_collection_name) if shared_embedding_collection: if len(shared_embedding_collection) > 1: raise ValueError( - 'Collection {} can only contain one variable. ' - 'Suggested fix A: Choose a unique name for this collection. ' - 'Suggested fix B: Do not add any variables to this collection. ' - 'The feature_column library already adds a variable under the ' - 'hood.'.format(shared_embedding_collection) - ) + 'Collection {} can only contain one variable. ' + 'Suggested fix A: Choose a unique name for this collection. ' + 'Suggested fix B: Do not add any variables to this collection. ' + 'The feature_column library already adds a variable under the ' + 'hood.'.format(shared_embedding_collection)) embedding_weights = shared_embedding_collection[0] - if embedding_weights.get_shape() != embedding_shape and self.ev_params is None: + if embedding_weights.get_shape( + ) != embedding_shape and self.ev_params is None: raise ValueError( - 'Shared embedding collection {} contains variable {} of ' - 'unexpected shape {}. Expected shape is {}. ' - 'Suggested fix A: Choose a unique name for this collection. ' - 'Suggested fix B: Do not add any variables to this collection. ' - 'The feature_column library already adds a variable under the ' - 'hood.'.format( - self.shared_embedding_collection_name, - embedding_weights.name, - embedding_weights.get_shape(), - embedding_shape, - ) - ) + 'Shared embedding collection {} contains variable {} of ' + 'unexpected shape {}. Expected shape is {}. ' + 'Suggested fix A: Choose a unique name for this collection. ' + 'Suggested fix B: Do not add any variables to this collection. ' + 'The feature_column library already adds a variable under the ' + 'hood.'.format( + self.shared_embedding_collection_name, + embedding_weights.name, + embedding_weights.get_shape(), + embedding_shape, + )) else: if self.ev_params is None: embedding_weights = variable_scope.get_variable( - name='embedding_weights', - shape=embedding_shape, - dtype=dtypes.float32, - initializer=self.initializer, - trainable=self.trainable and trainable, - partitioner=self.partitioner, - collections=weight_collections, + name='embedding_weights', + shape=embedding_shape, + dtype=dtypes.float32, + initializer=self.initializer, + trainable=self.trainable and trainable, + partitioner=self.partitioner, + collections=weight_collections, ) else: # at eval or inference time, it is necessary to set # the initializers to zeros, so that new key will # get zero embedding - if os.environ.get('tf.estimator.mode', '') != os.environ.get('tf.estimator.ModeKeys.TRAIN', 'train'): + if os.environ.get('tf.estimator.mode', '') != os.environ.get( + 'tf.estimator.ModeKeys.TRAIN', 'train'): initializer = init_ops.zeros_initializer() else: initializer = self.initializer extra_args = {} if 'EmbeddingVariableConfig' in dir(variables): ev_option = variables.EmbeddingVariableOption() - ev_option.filter_strategy = variables.CounterFilter(filter_freq=self.ev_params.filter_freq) + ev_option.filter_strategy = variables.CounterFilter( + filter_freq=self.ev_params.filter_freq) extra_args['ev_option'] = ev_option else: - extra_args['filter_options'] = variables.CounterFilterOptions(self.ev_params.filter_freq) + extra_args['filter_options'] = variables.CounterFilterOptions( + self.ev_params.filter_freq) embedding_weights = variable_scope.get_embedding_variable( - name='embedding_weights', - embedding_dim=self.dimension, - initializer=initializer, - trainable=self.trainable and trainable, - partitioner=self.partitioner, - collections=weight_collections, - steps_to_live=self.ev_params.steps_to_live, - **extra_args, + name='embedding_weights', + embedding_dim=self.dimension, + initializer=initializer, + trainable=self.trainable and trainable, + partitioner=self.partitioner, + collections=weight_collections, + steps_to_live=self.ev_params.steps_to_live, + **extra_args, ) - ops.add_to_collection(self.shared_embedding_collection_name, embedding_weights) + ops.add_to_collection(self.shared_embedding_collection_name, + embedding_weights) if self.ckpt_to_load_from is not None: to_restore = embedding_weights if isinstance(to_restore, variables.PartitionedVariable): to_restore = to_restore._get_variable_list() - checkpoint_utils.init_from_checkpoint(self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) + checkpoint_utils.init_from_checkpoint( + self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) if 'RaggedTensor' in str(type(sparse_ids)): assert sparse_weights is None return embedding_lookup_ragged( - embedding_weights=embedding_weights, - ragged_ids=sparse_ids, - ragged_weights=sparse_weights, - combiner=self.combiner, - max_norm=self.max_norm, - name='%s_weights' % self.name, + embedding_weights=embedding_weights, + ragged_ids=sparse_ids, + ragged_weights=sparse_weights, + combiner=self.combiner, + max_norm=self.max_norm, + name='%s_weights' % self.name, ) # Return embedding lookup result. return embedding_ops.safe_embedding_lookup_sparse( - embedding_weights=embedding_weights, - sparse_ids=sparse_ids, - sparse_weights=sparse_weights, - combiner=self.combiner, - name='%s_weights' % self.name, - max_norm=self.max_norm, + embedding_weights=embedding_weights, + sparse_ids=sparse_ids, + sparse_weights=sparse_weights, + combiner=self.combiner, + name='%s_weights' % self.name, + max_norm=self.max_norm, ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): if isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must not be of type _SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use input_layer, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'sequence_input_layer instead of input_layer. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) - return self._get_dense_tensor_internal(inputs=inputs, weight_collections=weight_collections, trainable=trainable) - - def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): + 'In embedding_column: {}. ' + 'categorical_column must not be of type _SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use input_layer, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'sequence_input_layer instead of input_layer. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) + return self._get_dense_tensor_internal( + inputs=inputs, + weight_collections=weight_collections, + trainable=trainable) + + def _get_sequence_dense_tensor(self, + inputs, + weight_collections=None, + trainable=None): if not isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must be of type _SequenceCategoricalColumn ' - 'to use sequence_input_layer. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) + 'In embedding_column: {}. ' + 'categorical_column must be of type _SequenceCategoricalColumn ' + 'to use sequence_input_layer. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) dense_tensor = self._get_dense_tensor_internal( - inputs=inputs, weight_collections=weight_collections, trainable=trainable - ) + inputs=inputs, + weight_collections=weight_collections, + trainable=trainable) sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) - sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) - return _SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) + sequence_length = fc_utils.sequence_length_from_sparse_tensor( + sparse_tensors.id_tensor) + return _SequenceDenseColumn.TensorSequenceLengthPair( + dense_tensor=dense_tensor, sequence_length=sequence_length) def _check_shape(shape, key): @@ -3070,15 +3203,18 @@ def _check_shape(shape, key): shape = tuple(shape) for dimension in shape: if not isinstance(dimension, six.integer_types): - raise TypeError('shape dimensions must be integer. ' 'shape: {}, key: {}'.format(shape, key)) + raise TypeError('shape dimensions must be integer. ' + 'shape: {}, key: {}'.format(shape, key)) if dimension < 1: - raise ValueError('shape dimensions must be greater than 0. ' 'shape: {}, key: {}'.format(shape, key)) + raise ValueError('shape dimensions must be greater than 0. ' + 'shape: {}, key: {}'.format(shape, key)) return shape class _HashedCategoricalColumn( - _CategoricalColumn, - collections.namedtuple('_HashedCategoricalColumn', ['key', 'hash_bucket_size', 'dtype']), + _CategoricalColumn, + collections.namedtuple('_HashedCategoricalColumn', + ['key', 'hash_bucket_size', 'dtype']), ): """See `categorical_column_with_hash_bucket`.""" @@ -3099,44 +3235,52 @@ def _transform_feature(self, inputs): if not isinstance(input_tensor, sparse_tensor_lib.SparseTensor): raise ValueError('SparseColumn input must be a SparseTensor.') - fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) + fc_utils.assert_string_or_int( + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key)) if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) - ) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype)) if self.dtype == dtypes.string: sparse_values = input_tensor.values else: sparse_values = string_ops.as_string(input_tensor.values) - sparse_id_values = string_ops.string_to_hash_bucket_fast(sparse_values, self.hash_bucket_size, name='lookup') - return sparse_tensor_lib.SparseTensor(input_tensor.indices, sparse_id_values, input_tensor.dense_shape) + sparse_id_values = string_ops.string_to_hash_bucket_fast( + sparse_values, self.hash_bucket_size, name='lookup') + return sparse_tensor_lib.SparseTensor(input_tensor.indices, + sparse_id_values, + input_tensor.dense_shape) @property def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.hash_bucket_size - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) class _VocabularyFileCategoricalColumn( - _CategoricalColumn, - collections.namedtuple( - '_VocabularyFileCategoricalColumn', - ( - 'key', - 'vocabulary_file', - 'vocabulary_size', - 'num_oov_buckets', - 'dtype', - 'default_value', + _CategoricalColumn, + collections.namedtuple( + '_VocabularyFileCategoricalColumn', + ( + 'key', + 'vocabulary_file', + 'vocabulary_size', + 'num_oov_buckets', + 'dtype', + 'default_value', + ), ), - ), ): """See `categorical_column_with_vocabulary_file`.""" @@ -3153,11 +3297,13 @@ def _transform_feature(self, inputs): if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) - ) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype)) - fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) + fc_utils.assert_string_or_int( + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key)) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -3166,12 +3312,12 @@ def _transform_feature(self, inputs): input_tensor = math_ops.cast(input_tensor, dtypes.int64) return lookup_ops.index_table_from_file( - vocabulary_file=self.vocabulary_file, - num_oov_buckets=self.num_oov_buckets, - vocab_size=self.vocabulary_size, - default_value=self.default_value, - key_dtype=key_dtype, - name='{}_lookup'.format(self.key), + vocabulary_file=self.vocabulary_file, + num_oov_buckets=self.num_oov_buckets, + vocab_size=self.vocabulary_size, + default_value=self.default_value, + key_dtype=key_dtype, + name='{}_lookup'.format(self.key), ).lookup(input_tensor) @property @@ -3179,16 +3325,19 @@ def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.vocabulary_size + self.num_oov_buckets - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) class _VocabularyListCategoricalColumn( - _CategoricalColumn, - collections.namedtuple( - '_VocabularyListCategoricalColumn', - ('key', 'vocabulary_list', 'dtype', 'default_value', 'num_oov_buckets'), - ), + _CategoricalColumn, + collections.namedtuple( + '_VocabularyListCategoricalColumn', + ('key', 'vocabulary_list', 'dtype', 'default_value', 'num_oov_buckets'), + ), ): """See `categorical_column_with_vocabulary_list`.""" @@ -3205,11 +3354,13 @@ def _transform_feature(self, inputs): if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) - ) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype)) - fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) + fc_utils.assert_string_or_int( + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key)) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -3218,11 +3369,11 @@ def _transform_feature(self, inputs): input_tensor = math_ops.cast(input_tensor, dtypes.int64) return lookup_ops.index_table_from_tensor( - vocabulary_list=tuple(self.vocabulary_list), - default_value=self.default_value, - num_oov_buckets=self.num_oov_buckets, - dtype=key_dtype, - name='{}_lookup'.format(self.key), + vocabulary_list=tuple(self.vocabulary_list), + default_value=self.default_value, + num_oov_buckets=self.num_oov_buckets, + dtype=key_dtype, + name='{}_lookup'.format(self.key), ).lookup(input_tensor) @property @@ -3230,13 +3381,17 @@ def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return len(self.vocabulary_list) + self.num_oov_buckets - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) class _IdentityCategoricalColumn( - _CategoricalColumn, - collections.namedtuple('_IdentityCategoricalColumn', ('key', 'num_buckets', 'default_value')), + _CategoricalColumn, + collections.namedtuple('_IdentityCategoricalColumn', + ('key', 'num_buckets', 'default_value')), ): """See `categorical_column_with_identity`.""" @@ -3252,38 +3407,42 @@ def _transform_feature(self, inputs): input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) if not input_tensor.dtype.is_integer: - raise ValueError('Invalid input, not integer. key: {} dtype: {}'.format(self.key, input_tensor.dtype)) + raise ValueError('Invalid input, not integer. key: {} dtype: {}'.format( + self.key, input_tensor.dtype)) values = math_ops.cast(input_tensor.values, dtypes.int64, name='values') - num_buckets = math_ops.cast(self.num_buckets, dtypes.int64, name='num_buckets') + num_buckets = math_ops.cast( + self.num_buckets, dtypes.int64, name='num_buckets') zero = math_ops.cast(0, dtypes.int64, name='zero') if self.default_value is None: # Fail if values are out-of-range. assert_less = check_ops.assert_less( - values, - num_buckets, - data=(values, num_buckets), - name='assert_less_than_num_buckets', + values, + num_buckets, + data=(values, num_buckets), + name='assert_less_than_num_buckets', ) - assert_greater = check_ops.assert_greater_equal(values, zero, data=(values,), name='assert_greater_or_equal_0') + assert_greater = check_ops.assert_greater_equal( + values, zero, data=(values,), name='assert_greater_or_equal_0') with ops.control_dependencies((assert_less, assert_greater)): values = array_ops.identity(values) else: # Assign default for out-of-range values. values = array_ops.where( - math_ops.logical_or(values < zero, values >= num_buckets, name='out_of_range'), - array_ops.fill( - dims=array_ops.shape(values), - value=math_ops.cast(self.default_value, dtypes.int64), - name='default_values', - ), - values, + math_ops.logical_or( + values < zero, values >= num_buckets, name='out_of_range'), + array_ops.fill( + dims=array_ops.shape(values), + value=math_ops.cast(self.default_value, dtypes.int64), + name='default_values', + ), + values, ) return sparse_tensor_lib.SparseTensor( - indices=input_tensor.indices, - values=values, - dense_shape=input_tensor.dense_shape, + indices=input_tensor.indices, + values=values, + dense_shape=input_tensor.dense_shape, ) @property @@ -3291,30 +3450,33 @@ def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.num_buckets - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) class _WeightedCategoricalColumn( - _CategoricalColumn, - collections.namedtuple( - '_WeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype'), - ), + _CategoricalColumn, + collections.namedtuple( + '_WeightedCategoricalColumn', + ('categorical_column', 'weight_feature_key', 'dtype'), + ), ): """See `weighted_categorical_column`.""" @property def name(self): - return '{}_weighted_by_{}'.format(self.categorical_column.name, self.weight_feature_key) + return '{}_weighted_by_{}'.format(self.categorical_column.name, + self.weight_feature_key) @property def _parse_example_spec(self): config = self.categorical_column._parse_example_spec if self.weight_feature_key in config: - raise ValueError( - 'Parse config {} already exists for {}.'.format(config[self.weight_feature_key], self.weight_feature_key) - ) + raise ValueError('Parse config {} already exists for {}.'.format( + config[self.weight_feature_key], self.weight_feature_key)) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @@ -3326,17 +3488,23 @@ def _transform_feature(self, inputs): weight_tensor = inputs.get(self.weight_feature_key) if weight_tensor is None: raise ValueError('Missing weights {}.'.format(self.weight_feature_key)) - weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(weight_tensor) + weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( + weight_tensor) if self.dtype != weight_tensor.dtype.base_dtype: - raise ValueError('Bad dtype, expected {}, but got {}.'.format(self.dtype, weight_tensor.dtype)) + raise ValueError('Bad dtype, expected {}, but got {}.'.format( + self.dtype, weight_tensor.dtype)) if not isinstance(weight_tensor, sparse_tensor_lib.SparseTensor): # The weight tensor can be a regular Tensor. In this case, sparsify it. - weight_tensor = _to_sparse_input_and_drop_ignore_values(weight_tensor, ignore_value=0.0) + weight_tensor = _to_sparse_input_and_drop_ignore_values( + weight_tensor, ignore_value=0.0) if not weight_tensor.dtype.is_floating: weight_tensor = math_ops.cast(weight_tensor, dtypes.float32) return (inputs.get(self.categorical_column), weight_tensor) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): del weight_collections del trainable tensors = inputs.get(self) @@ -3344,8 +3512,9 @@ def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): class _CrossedColumn( - _CategoricalColumn, - collections.namedtuple('_CrossedColumn', ['keys', 'hash_bucket_size', 'hash_key']), + _CategoricalColumn, + collections.namedtuple('_CrossedColumn', + ['keys', 'hash_bucket_size', 'hash_key']), ): """See `crossed_column`.""" @@ -3378,17 +3547,16 @@ def _transform_feature(self, inputs): ids_and_weights = key._get_sparse_tensors(inputs) if ids_and_weights.weight_tensor is not None: raise ValueError( - 'crossed_column does not support weight_tensor, but the given ' - 'column populates weight_tensor. ' - 'Given column: {}'.format(key.name) - ) + 'crossed_column does not support weight_tensor, but the given ' + 'column populates weight_tensor. ' + 'Given column: {}'.format(key.name)) feature_tensors.append(ids_and_weights.id_tensor) else: raise ValueError('Unsupported column type. Given: {}'.format(key)) return sparse_ops.sparse_cross_hashed( - inputs=feature_tensors, - num_buckets=self.hash_bucket_size, - hash_key=self.hash_key, + inputs=feature_tensors, + num_buckets=self.hash_bucket_size, + hash_key=self.hash_key, ) @property @@ -3396,7 +3564,10 @@ def _num_buckets(self): """Returns number of buckets in this sparse feature.""" return self.hash_bucket_size - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): return _CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -3419,9 +3590,9 @@ def _collect_leaf_level_keys(cross): class _IndicatorColumn( - _DenseColumn, - _SequenceDenseColumn, - collections.namedtuple('_IndicatorColumn', ['categorical_column']), + _DenseColumn, + _SequenceDenseColumn, + collections.namedtuple('_IndicatorColumn', ['categorical_column']), ): """Represents a one-hot column for use in deep networks. @@ -3453,25 +3624,31 @@ def _transform_feature(self, inputs): # If the underlying column is weighted, return the input as a dense tensor. if weight_tensor is not None: weighted_column = sparse_ops.sparse_merge( - sp_ids=id_tensor, - sp_values=weight_tensor, - vocab_size=int(self._variable_shape[-1]), + sp_ids=id_tensor, + sp_values=weight_tensor, + vocab_size=int(self._variable_shape[-1]), ) # Remove (?, -1) index. - weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0], weighted_column.dense_shape) + weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0], + weighted_column.dense_shape) # Use scatter_nd to merge duplicated indices if existed, # instead of sparse_tensor_to_dense. return array_ops.scatter_nd( - weighted_column.indices, - weighted_column.values, - weighted_column.dense_shape, + weighted_column.indices, + weighted_column.values, + weighted_column.dense_shape, ) - dense_id_tensor = sparse_ops.sparse_tensor_to_dense(id_tensor, default_value=-1) + dense_id_tensor = sparse_ops.sparse_tensor_to_dense( + id_tensor, default_value=-1) # One hot must be float for tf.concat reasons since all other inputs to # input_layer are float32. - one_hot_id_tensor = array_ops.one_hot(dense_id_tensor, depth=self._variable_shape[-1], on_value=1.0, off_value=0.0) + one_hot_id_tensor = array_ops.one_hot( + dense_id_tensor, + depth=self._variable_shape[-1], + on_value=1.0, + off_value=0.0) # Reduce to get a multi-hot per example. return math_ops.reduce_sum(one_hot_id_tensor, axis=[-2]) @@ -3507,37 +3684,42 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del trainable if isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must not be of type _SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use input_layer, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'sequence_input_layer instead of input_layer. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) + 'In indicator_column: {}. ' + 'categorical_column must not be of type _SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use input_layer, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'sequence_input_layer instead of input_layer. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) # Feature has been already transformed. Return the intermediate # representation created by _transform_feature. return inputs.get(self) - def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): + def _get_sequence_dense_tensor(self, + inputs, + weight_collections=None, + trainable=None): # Do nothing with weight_collections and trainable since no variables are # created in this function. del weight_collections del trainable if not isinstance(self.categorical_column, _SequenceCategoricalColumn): raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must be of type _SequenceCategoricalColumn ' - 'to use sequence_input_layer. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) + 'In indicator_column: {}. ' + 'categorical_column must be of type _SequenceCategoricalColumn ' + 'to use sequence_input_layer. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) # Feature has been already transformed. Return the intermediate # representation created by _transform_feature. dense_tensor = inputs.get(self) sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) - sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) - return _SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) + sequence_length = fc_utils.sequence_length_from_sparse_tensor( + sparse_tensors.id_tensor) + return _SequenceDenseColumn.TensorSequenceLengthPair( + dense_tensor=dense_tensor, sequence_length=sequence_length) def _verify_static_batch_size_equality(tensors, columns): @@ -3560,19 +3742,19 @@ def _verify_static_batch_size_equality(tensors, columns): expected_batch_size = tensors[i].shape.dims[0] elif not expected_batch_size.is_compatible_with(tensors[i].shape.dims[0]): raise ValueError( - 'Batch size (first dimension) of each feature must be same. ' - 'Batch size of columns ({}, {}): ({}, {})'.format( - columns[bath_size_column_index].name, - columns[i].name, - expected_batch_size, - tensors[i].shape.dims[0], - ) - ) + 'Batch size (first dimension) of each feature must be same. ' + 'Batch size of columns ({}, {}): ({}, {})'.format( + columns[bath_size_column_index].name, + columns[i].name, + expected_batch_size, + tensors[i].shape.dims[0], + )) class _SequenceCategoricalColumn( - _CategoricalColumn, - collections.namedtuple('_SequenceCategoricalColumn', ['categorical_column']), + _CategoricalColumn, + collections.namedtuple('_SequenceCategoricalColumn', + ['categorical_column']), ): """Represents sequences of categorical data.""" @@ -3591,7 +3773,10 @@ def _transform_feature(self, inputs): def _num_buckets(self): return self.categorical_column._num_buckets - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) id_tensor = sparse_tensors.id_tensor weight_tensor = sparse_tensors.weight_tensor diff --git a/easy_rec/python/compat/feature_column/feature_column_v2.py b/easy_rec/python/compat/feature_column/feature_column_v2.py index 7772f60a2..21d4964b1 100644 --- a/easy_rec/python/compat/feature_column/feature_column_v2.py +++ b/easy_rec/python/compat/feature_column/feature_column_v2.py @@ -124,7 +124,9 @@ the API subject to change, and should not be relied upon! """ -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import abc import collections @@ -136,50 +138,41 @@ import six import tensorflow as tf from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes, ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib from tensorflow.python.framework import tensor_shape - # TODO(b/118385027): Dependency on keras can be problematic if Keras moves out # of the main repo. from tensorflow.python.keras import utils from tensorflow.python.keras.engine import training from tensorflow.python.keras.engine.base_layer import Layer -from tensorflow.python.ops import ( # NOQA - array_ops, - check_ops, - control_flow_ops, - embedding_ops, - init_ops, - lookup_ops, - math_ops, - nn_ops, - parsing_ops, - sparse_ops, - string_ops, - variable_scope, - variables, -) from tensorflow.python.platform import gfile from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import checkpoint_utils - # from tensorflow.python.training.tracking import tracking -from tensorflow.python.util import deprecation, nest +from tensorflow.python.util import deprecation +from tensorflow.python.util import nest from easy_rec.python.compat import ops as compat_ops from easy_rec.python.compat.feature_column import feature_column as fc_old from easy_rec.python.compat.feature_column import utils as fc_utils -from easy_rec.python.compat.feature_column.feature_column import ( # NOQA - embedding_lookup_ragged, -) from easy_rec.python.layers import utils as layer_utils -_FEATURE_COLUMN_DEPRECATION_DATE = None -_FEATURE_COLUMN_DEPRECATION = ( - 'The old _FeatureColumn APIs are being ' 'deprecated. Please use the new FeatureColumn ' 'APIs instead.' +from tensorflow.python.ops import ( # NOQA + array_ops, check_ops, control_flow_ops, embedding_ops, init_ops, lookup_ops, + math_ops, nn_ops, parsing_ops, sparse_ops, string_ops, variable_scope, + variables, ) +from easy_rec.python.compat.feature_column.feature_column import ( # NOQA + embedding_lookup_ragged,) + +_FEATURE_COLUMN_DEPRECATION_DATE = None +_FEATURE_COLUMN_DEPRECATION = ('The old _FeatureColumn APIs are being ' + 'deprecated. Please use the new FeatureColumn ' + 'APIs instead.') + if os.getenv('SAFE_EMBEDDING', 'TRUE') == 'TRUE': embedding_lookup_sparse = embedding_ops.safe_embedding_lookup_sparse else: @@ -196,14 +189,14 @@ class StateManager(object): """ def create_variable( - self, - feature_column, - name, - shape, - dtype=None, - trainable=True, - use_resource=True, - initializer=None, + self, + feature_column, + name, + shape, + dtype=None, + trainable=True, + use_resource=True, + initializer=None, ): """Creates a new variable. @@ -287,29 +280,29 @@ def __init__(self, layer, trainable): self._cols_to_vars_map = collections.defaultdict(lambda: {}) def create_variable( - self, - feature_column, - name, - shape, - dtype=None, - trainable=True, - use_resource=True, - initializer=None, + self, + feature_column, + name, + shape, + dtype=None, + trainable=True, + use_resource=True, + initializer=None, ): if name in self._cols_to_vars_map[feature_column]: raise ValueError('Variable already exists.') var = self._layer.add_variable( - name=name, - shape=shape, - dtype=dtype, - initializer=initializer, - trainable=self._trainable and trainable, - use_resource=use_resource, - # TODO(rohanj): Get rid of this hack once we have a mechanism for - # specifying a default partitioner for an entire layer. In that case, - # the default getter for Layers should work. - getter=variable_scope.get_variable, + name=name, + shape=shape, + dtype=dtype, + initializer=initializer, + trainable=self._trainable and trainable, + use_resource=use_resource, + # TODO(rohanj): Get rid of this hack once we have a mechanism for + # specifying a default partitioner for an entire layer. In that case, + # the default getter for Layers should work. + getter=variable_scope.get_variable, ) self._cols_to_vars_map[feature_column][name] = var return var @@ -339,17 +332,19 @@ class _BaseFeaturesLayer(Layer): `expected_column_type`. """ - def __init__(self, feature_columns, expected_column_type, trainable, name, **kwargs): - super(_BaseFeaturesLayer, self).__init__(name=name, trainable=trainable, **kwargs) + def __init__(self, feature_columns, expected_column_type, trainable, name, + **kwargs): + super(_BaseFeaturesLayer, self).__init__( + name=name, trainable=trainable, **kwargs) self._feature_columns = _normalize_feature_columns(feature_columns) self._state_manager = _StateManagerImpl(self, self.trainable) for column in self._feature_columns: if not isinstance(column, expected_column_type): raise ValueError( - 'Items of feature_columns must be a {}. ' - 'You can wrap a categorical column with an ' - 'embedding_column or indicator_column. Given: {}'.format(expected_column_type, column) - ) + 'Items of feature_columns must be a {}. ' + 'You can wrap a categorical column with an ' + 'embedding_column or indicator_column. Given: {}'.format( + expected_column_type, column)) def build(self, _): for column in self._feature_columns: @@ -440,11 +435,11 @@ def __init__(self, feature_columns, trainable=True, name=None, **kwargs): ValueError: if an item in `feature_columns` is not a `DenseColumn`. """ super(DenseFeatures, self).__init__( - feature_columns=feature_columns, - trainable=trainable, - name=name, - expected_column_type=DenseColumn, - **kwargs, + feature_columns=feature_columns, + trainable=trainable, + name=name, + expected_column_type=DenseColumn, + **kwargs, ) @property @@ -474,12 +469,14 @@ def call(self, features, cols_to_output_tensors=None): ValueError: If features are not a dictionary. """ if not isinstance(features, dict): - raise ValueError('We expected a dictionary here. Instead we got: ', features) + raise ValueError('We expected a dictionary here. Instead we got: ', + features) transformation_cache = FeatureTransformationCache(features) output_tensors = [] for column in self._feature_columns: with ops.name_scope(column.name): - tensor = column.get_dense_tensor(transformation_cache, self._state_manager) + tensor = column.get_dense_tensor(transformation_cache, + self._state_manager) processed_tensors = self._process_dense_tensor(column, tensor) if cols_to_output_tensors is not None: cols_to_output_tensors[column] = processed_tensors @@ -491,22 +488,23 @@ class _LinearModelLayer(Layer): """Layer that contains logic for `LinearModel`.""" def __init__( - self, - feature_columns, - units=1, - sparse_combiner='sum', - trainable=True, - name=None, - **kwargs, + self, + feature_columns, + units=1, + sparse_combiner='sum', + trainable=True, + name=None, + **kwargs, ): - super(_LinearModelLayer, self).__init__(name=name, trainable=trainable, **kwargs) + super(_LinearModelLayer, self).__init__( + name=name, trainable=trainable, **kwargs) self._feature_columns = _normalize_feature_columns(feature_columns) for column in self._feature_columns: if not isinstance(column, (DenseColumn, CategoricalColumn)): raise ValueError( - 'Items of feature_columns must be either a ' 'DenseColumn or CategoricalColumn. Given: {}'.format(column) - ) + 'Items of feature_columns must be either a ' + 'DenseColumn or CategoricalColumn. Given: {}'.format(column)) self._units = units self._sparse_combiner = sparse_combiner @@ -531,33 +529,34 @@ def build(self, _): else: first_dim = column.variable_shape.num_elements() self._state_manager.create_variable( - column, - name='weights', - dtype=dtypes.float32, - shape=(first_dim, self._units), - initializer=init_ops.zeros_initializer(), - trainable=self.trainable, + column, + name='weights', + dtype=dtypes.float32, + shape=(first_dim, self._units), + initializer=init_ops.zeros_initializer(), + trainable=self.trainable, ) # Create a bias variable. self.bias = self.add_variable( - name='bias_weights', - dtype=dtypes.float32, - shape=[self._units], - initializer=init_ops.zeros_initializer(), - trainable=self.trainable, - use_resource=True, - # TODO(rohanj): Get rid of this hack once we have a mechanism for - # specifying a default partitioner for an entire layer. In that case, - # the default getter for Layers should work. - getter=variable_scope.get_variable, + name='bias_weights', + dtype=dtypes.float32, + shape=[self._units], + initializer=init_ops.zeros_initializer(), + trainable=self.trainable, + use_resource=True, + # TODO(rohanj): Get rid of this hack once we have a mechanism for + # specifying a default partitioner for an entire layer. In that case, + # the default getter for Layers should work. + getter=variable_scope.get_variable, ) super(_LinearModelLayer, self).build(None) def call(self, features): if not isinstance(features, dict): - raise ValueError('We expected a dictionary here. Instead we got: {}'.format(features)) + raise ValueError( + 'We expected a dictionary here. Instead we got: {}'.format(features)) with ops.name_scope(self.name): transformation_cache = FeatureTransformationCache(features) weighted_sums = [] @@ -568,17 +567,19 @@ def call(self, features): weight_var = self._state_manager.get_variable(column, 'weights') weighted_sum = _create_weighted_sum( - column=column, - transformation_cache=transformation_cache, - state_manager=self._state_manager, - sparse_combiner=self._sparse_combiner, - weight_var=weight_var, + column=column, + transformation_cache=transformation_cache, + state_manager=self._state_manager, + sparse_combiner=self._sparse_combiner, + weight_var=weight_var, ) weighted_sums.append(weighted_sum) _verify_static_batch_size_equality(weighted_sums, self._feature_columns) - predictions_no_bias = math_ops.add_n(weighted_sums, name='weighted_sum_no_bias') - predictions = nn_ops.bias_add(predictions_no_bias, self.bias, name='weighted_sum') + predictions_no_bias = math_ops.add_n( + weighted_sums, name='weighted_sum_no_bias') + predictions = nn_ops.bias_add( + predictions_no_bias, self.bias, name='weighted_sum') return predictions @@ -622,13 +623,13 @@ class LinearModel(training.Model): """ def __init__( - self, - feature_columns, - units=1, - sparse_combiner='sum', - trainable=True, - name=None, - **kwargs, + self, + feature_columns, + units=1, + sparse_combiner='sum', + trainable=True, + name=None, + **kwargs, ): """Constructs a LinearLayer. @@ -687,7 +688,13 @@ def __init__( nor `CategoricalColumn`. """ super(LinearModel, self).__init__(name=name, **kwargs) - self.layer = _LinearModelLayer(feature_columns, units, sparse_combiner, trainable, name=self.name, **kwargs) + self.layer = _LinearModelLayer( + feature_columns, + units, + sparse_combiner, + trainable, + name=self.name, + **kwargs) def call(self, features): """Returns a `Tensor` the represents the predictions of a linear model. @@ -748,7 +755,8 @@ def _transform_features_v2(features, feature_columns, state_manager): """ feature_columns = _normalize_feature_columns(feature_columns) outputs = {} - with ops.name_scope(None, default_name='transform_features', values=features.values()): + with ops.name_scope( + None, default_name='transform_features', values=features.values()): transformation_cache = FeatureTransformationCache(features) for column in feature_columns: with ops.name_scope(None, default_name=column.name): @@ -804,28 +812,28 @@ def make_parse_example_spec_v2(feature_columns): result = {} for column in feature_columns: if not isinstance(column, FeatureColumn): - raise ValueError('All feature_columns must be FeatureColumn instances. ' 'Given: {}'.format(column)) + raise ValueError('All feature_columns must be FeatureColumn instances. ' + 'Given: {}'.format(column)) config = column.parse_example_spec for key, value in six.iteritems(config): if key in result and value != result[key]: - raise ValueError( - 'feature_columns contain different parse_spec for key ' '{}. Given {} and {}'.format(key, value, result[key]) - ) + raise ValueError('feature_columns contain different parse_spec for key ' + '{}. Given {} and {}'.format(key, value, result[key])) result.update(config) return result def embedding_column( - categorical_column, - dimension, - combiner='mean', - initializer=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True, - partitioner=None, - ev_params=None, + categorical_column, + dimension, + combiner='mean', + initializer=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True, + partitioner=None, + ev_params=None, ): """`DenseColumn` that converts from sparse, categorical input. @@ -901,41 +909,43 @@ def model_fn(features, ...): if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError('Must specify both `ckpt_to_load_from` and ' '`tensor_name_in_ckpt` or none of them.') + raise ValueError('Must specify both `ckpt_to_load_from` and ' + '`tensor_name_in_ckpt` or none of them.') if (initializer is not None) and (not callable(initializer)): - raise ValueError( - 'initializer must be callable if specified. ' 'Embedding of column_name: {}'.format(categorical_column.name) - ) + raise ValueError('initializer must be callable if specified. ' + 'Embedding of column_name: {}'.format( + categorical_column.name)) if initializer is None: - initializer = init_ops.truncated_normal_initializer(mean=0.0, stddev=0.01 / math.sqrt(dimension)) + initializer = init_ops.truncated_normal_initializer( + mean=0.0, stddev=0.01 / math.sqrt(dimension)) return EmbeddingColumn( - categorical_column=categorical_column, - dimension=dimension, - combiner=combiner, - initializer=initializer, - ckpt_to_load_from=ckpt_to_load_from, - tensor_name_in_ckpt=tensor_name_in_ckpt, - max_norm=max_norm, - trainable=trainable, - partitioner=partitioner, - ev_params=ev_params, + categorical_column=categorical_column, + dimension=dimension, + combiner=combiner, + initializer=initializer, + ckpt_to_load_from=ckpt_to_load_from, + tensor_name_in_ckpt=tensor_name_in_ckpt, + max_norm=max_norm, + trainable=trainable, + partitioner=partitioner, + ev_params=ev_params, ) def shared_embedding_columns( - categorical_columns, - dimension, - combiner='mean', - initializer=None, - shared_embedding_collection_name=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True, - partitioner=None, - ev_params=None, + categorical_columns, + dimension, + combiner='mean', + initializer=None, + shared_embedding_collection_name=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True, + partitioner=None, + ev_params=None, ): """List of dense columns that convert from sparse, categorical input. @@ -1034,17 +1044,20 @@ def model_fn(features, ...): RuntimeError: if eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('shared_embedding_columns are not supported when eager ' 'execution is enabled.') + raise RuntimeError('shared_embedding_columns are not supported when eager ' + 'execution is enabled.') if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError('Must specify both `ckpt_to_load_from` and ' '`tensor_name_in_ckpt` or none of them.') + raise ValueError('Must specify both `ckpt_to_load_from` and ' + '`tensor_name_in_ckpt` or none of them.') if (initializer is not None) and (not callable(initializer)): raise ValueError('initializer must be callable if specified.') if initializer is None: - initializer = init_ops.truncated_normal_initializer(mean=0.0, stddev=0.01 / math.sqrt(dimension)) + initializer = init_ops.truncated_normal_initializer( + mean=0.0, stddev=0.01 / math.sqrt(dimension)) # Sort the columns so the default collection name is deterministic even if the # user passes columns from an unsorted collection, such as dict.values(). @@ -1054,19 +1067,21 @@ def model_fn(features, ...): num_buckets = c0._num_buckets if not isinstance(c0, fc_old._CategoricalColumn): raise ValueError( - 'All categorical_columns must be subclasses of _CategoricalColumn. ' 'Given: {}, of type: {}'.format(c0, type(c0)) - ) - if isinstance(c0, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): + 'All categorical_columns must be subclasses of _CategoricalColumn. ' + 'Given: {}, of type: {}'.format(c0, type(c0))) + if isinstance(c0, + (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): c0 = c0.categorical_column for c in sorted_columns[1:]: - if isinstance(c, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): + if isinstance( + c, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): c = c.categorical_column if num_buckets != c._num_buckets: raise ValueError( - 'To use shared_embedding_column, all categorical_columns must have ' - 'the same number of buckets. Given column: {} with buckets: {} does ' - 'not match column: {} with buckets: {}'.format(c0, num_buckets, c, c._num_buckets) - ) + 'To use shared_embedding_column, all categorical_columns must have ' + 'the same number of buckets. Given column: {} with buckets: {} does ' + 'not match column: {} with buckets: {}'.format( + c0, num_buckets, c, c._num_buckets)) if not shared_embedding_collection_name: shared_embedding_collection_name = '_'.join(c.name for c in sorted_columns) @@ -1075,34 +1090,33 @@ def model_fn(features, ...): result = [] for column in categorical_columns: result.append( - fc_old._SharedEmbeddingColumn( - categorical_column=column, - initializer=initializer, - dimension=dimension, - combiner=combiner, - shared_embedding_collection_name=shared_embedding_collection_name, - ckpt_to_load_from=ckpt_to_load_from, - tensor_name_in_ckpt=tensor_name_in_ckpt, - max_norm=max_norm, - trainable=trainable, - partitioner=partitioner, - ev_params=ev_params, - ) - ) + fc_old._SharedEmbeddingColumn( + categorical_column=column, + initializer=initializer, + dimension=dimension, + combiner=combiner, + shared_embedding_collection_name=shared_embedding_collection_name, + ckpt_to_load_from=ckpt_to_load_from, + tensor_name_in_ckpt=tensor_name_in_ckpt, + max_norm=max_norm, + trainable=trainable, + partitioner=partitioner, + ev_params=ev_params, + )) return result def shared_embedding_columns_v2( - categorical_columns, - dimension, - combiner='mean', - initializer=None, - shared_embedding_collection_name=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True, + categorical_columns, + dimension, + combiner='mean', + initializer=None, + shared_embedding_collection_name=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True, ): """List of dense columns that convert from sparse, categorical input. @@ -1200,17 +1214,20 @@ def model_fn(features, ...): RuntimeError: if eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('shared_embedding_columns are not supported when eager ' 'execution is enabled.') + raise RuntimeError('shared_embedding_columns are not supported when eager ' + 'execution is enabled.') if (dimension is None) or (dimension < 1): raise ValueError('Invalid dimension {}.'.format(dimension)) if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError('Must specify both `ckpt_to_load_from` and ' '`tensor_name_in_ckpt` or none of them.') + raise ValueError('Must specify both `ckpt_to_load_from` and ' + '`tensor_name_in_ckpt` or none of them.') if (initializer is not None) and (not callable(initializer)): raise ValueError('initializer must be callable if specified.') if initializer is None: - initializer = init_ops.truncated_normal_initializer(mean=0.0, stddev=0.01 / math.sqrt(dimension)) + initializer = init_ops.truncated_normal_initializer( + mean=0.0, stddev=0.01 / math.sqrt(dimension)) # Sort the columns so the default collection name is deterministic even if the # user passes columns from an unsorted collection, such as dict.values(). @@ -1220,8 +1237,8 @@ def model_fn(features, ...): num_buckets = c0.num_buckets if not isinstance(c0, CategoricalColumn): raise ValueError( - 'All categorical_columns must be subclasses of CategoricalColumn. ' 'Given: {}, of type: {}'.format(c0, type(c0)) - ) + 'All categorical_columns must be subclasses of CategoricalColumn. ' + 'Given: {}, of type: {}'.format(c0, type(c0))) if isinstance(c0, WeightedCategoricalColumn): c0 = c0.categorical_column for c in sorted_columns[1:]: @@ -1229,46 +1246,47 @@ def model_fn(features, ...): c = c.categorical_column if not isinstance(c, type(c0)): raise ValueError( - 'To use shared_embedding_column, all categorical_columns must have ' - 'the same type, or be weighted_categorical_column of the same type. ' - 'Given column: {} of type: {} does not match given column: {} of ' - 'type: {}'.format(c0, type(c0), c, type(c)) - ) + 'To use shared_embedding_column, all categorical_columns must have ' + 'the same type, or be weighted_categorical_column of the same type. ' + 'Given column: {} of type: {} does not match given column: {} of ' + 'type: {}'.format(c0, type(c0), c, type(c))) if num_buckets != c.num_buckets: raise ValueError( - 'To use shared_embedding_column, all categorical_columns must have ' - 'the same number of buckets. Given column: {} with buckets: {} does ' - 'not match column: {} with buckets: {}'.format(c0, num_buckets, c, c.num_buckets) - ) + 'To use shared_embedding_column, all categorical_columns must have ' + 'the same number of buckets. Given column: {} with buckets: {} does ' + 'not match column: {} with buckets: {}'.format( + c0, num_buckets, c, c.num_buckets)) if not shared_embedding_collection_name: shared_embedding_collection_name = '_'.join(c.name for c in sorted_columns) shared_embedding_collection_name += '_shared_embedding' column_creator = SharedEmbeddingColumnCreator( - dimension, - initializer, - ckpt_to_load_from, - tensor_name_in_ckpt, - num_buckets, - trainable, - shared_embedding_collection_name, + dimension, + initializer, + ckpt_to_load_from, + tensor_name_in_ckpt, + num_buckets, + trainable, + shared_embedding_collection_name, ) result = [] for column in categorical_columns: - result.append(column_creator(categorical_column=column, combiner=combiner, max_norm=max_norm)) + result.append( + column_creator( + categorical_column=column, combiner=combiner, max_norm=max_norm)) return result def numeric_column( - key, - shape=(1,), - default_value=None, - dtype=dtypes.float32, - normalizer_fn=None, - feature_name=None, + key, + shape=(1,), + default_value=None, + dtype=dtypes.float32, + normalizer_fn=None, + feature_name=None, ): """Represents real valued or numerical features. @@ -1323,20 +1341,22 @@ def numeric_column( """ shape = _check_shape(shape, key) if not (dtype.is_integer or dtype.is_floating): - raise ValueError('dtype must be convertible to float. ' 'dtype: {}, key: {}'.format(dtype, key)) + raise ValueError('dtype must be convertible to float. ' + 'dtype: {}, key: {}'.format(dtype, key)) default_value = fc_utils.check_default_value(shape, default_value, dtype, key) if normalizer_fn is not None and not callable(normalizer_fn): - raise TypeError('normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) + raise TypeError( + 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) fc_utils.assert_key_is_string(key) return NumericColumn( - feature_name=feature_name, - key=key, - shape=shape, - default_value=default_value, - dtype=dtype, - normalizer_fn=normalizer_fn, + feature_name=feature_name, + key=key, + shape=shape, + default_value=default_value, + dtype=dtype, + normalizer_fn=normalizer_fn, ) @@ -1408,10 +1428,11 @@ def bucketized_column(source_column, boundaries): """ if not isinstance(source_column, (NumericColumn, fc_old._NumericColumn)): raise ValueError( - 'source_column must be a column generated with numeric_column(). ' 'Given: {}'.format(source_column) - ) + 'source_column must be a column generated with numeric_column(). ' + 'Given: {}'.format(source_column)) if len(source_column.shape) > 1: - raise ValueError('source_column must be one-dimensional column. ' 'Given: {}'.format(source_column)) + raise ValueError('source_column must be one-dimensional column. ' + 'Given: {}'.format(source_column)) if not boundaries: raise ValueError('boundaries must not be empty.') if not (isinstance(boundaries, list) or isinstance(boundaries, tuple)): @@ -1422,7 +1443,10 @@ def bucketized_column(source_column, boundaries): return BucketizedColumn(source_column, tuple(boundaries)) -def categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dtypes.string, feature_name=None): +def categorical_column_with_hash_bucket(key, + hash_bucket_size, + dtype=dtypes.string, + feature_name=None): """Represents sparse feature where ids are set by hashing. Use this when your sparse features are in string or integer format, and you @@ -1468,9 +1492,9 @@ def categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dtypes.stri raise ValueError('hash_bucket_size must be set. ' 'key: {}'.format(key)) if hash_bucket_size < 1: - raise ValueError( - 'hash_bucket_size must be at least 1. ' 'hash_bucket_size: {}, key: {}'.format(hash_bucket_size, key) - ) + raise ValueError('hash_bucket_size must be at least 1. ' + 'hash_bucket_size: {}, key: {}'.format( + hash_bucket_size, key)) fc_utils.assert_key_is_string(key) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) @@ -1479,13 +1503,13 @@ def categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dtypes.stri def categorical_column_with_vocabulary_file_v2( - key, - vocabulary_file, - vocabulary_size=None, - dtype=dtypes.string, - default_value=None, - num_oov_buckets=0, - feature_name=None, + key, + vocabulary_file, + vocabulary_size=None, + dtype=dtypes.string, + default_value=None, + num_oov_buckets=0, + feature_name=None, ): """A `CategoricalColumn` with a vocabulary file. @@ -1575,10 +1599,11 @@ def categorical_column_with_vocabulary_file_v2( with gfile.GFile(vocabulary_file) as f: vocabulary_size = sum(1 for _ in f) logging.info( - 'vocabulary_size = %d in %s is inferred from the number of elements ' 'in the vocabulary_file %s.', - vocabulary_size, - key, - vocabulary_file, + 'vocabulary_size = %d in %s is inferred from the number of elements ' + 'in the vocabulary_file %s.', + vocabulary_size, + key, + vocabulary_file, ) # `vocabulary_size` isn't required for lookup, but it is for `_num_buckets`. @@ -1586,29 +1611,32 @@ def categorical_column_with_vocabulary_file_v2( raise ValueError('Invalid vocabulary_size in {}.'.format(key)) if num_oov_buckets: if default_value is not None: - raise ValueError("Can't specify both num_oov_buckets and default_value in {}.".format(key)) + raise ValueError( + "Can't specify both num_oov_buckets and default_value in {}.".format( + key)) if num_oov_buckets < 0: - raise ValueError('Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key)) + raise ValueError('Invalid num_oov_buckets {} in {}.'.format( + num_oov_buckets, key)) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) return VocabularyFileCategoricalColumn( - feature_name=feature_name, - key=key, - vocabulary_file=vocabulary_file, - vocabulary_size=vocabulary_size, - num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, - default_value=-1 if default_value is None else default_value, - dtype=dtype, + feature_name=feature_name, + key=key, + vocabulary_file=vocabulary_file, + vocabulary_size=vocabulary_size, + num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, + default_value=-1 if default_value is None else default_value, + dtype=dtype, ) def categorical_column_with_vocabulary_list( - key, - vocabulary_list, - dtype=None, - default_value=-1, - num_oov_buckets=0, - feature_name=None, + key, + vocabulary_list, + dtype=None, + default_value=-1, + num_oov_buckets=0, + feature_name=None, ): """A `CategoricalColumn` with in-memory vocabulary. @@ -1686,36 +1714,47 @@ def categorical_column_with_vocabulary_list( ValueError: if `dtype` is not integer or string. """ if (vocabulary_list is None) or (len(vocabulary_list) < 1): - raise ValueError('vocabulary_list {} must be non-empty, column_name: {}'.format(vocabulary_list, key)) + raise ValueError( + 'vocabulary_list {} must be non-empty, column_name: {}'.format( + vocabulary_list, key)) if len(set(vocabulary_list)) != len(vocabulary_list): - raise ValueError('Duplicate keys in vocabulary_list {}, column_name: {}'.format(vocabulary_list, key)) + raise ValueError( + 'Duplicate keys in vocabulary_list {}, column_name: {}'.format( + vocabulary_list, key)) vocabulary_dtype = dtypes.as_dtype(np.array(vocabulary_list).dtype) if num_oov_buckets: if default_value != -1: - raise ValueError("Can't specify both num_oov_buckets and default_value in {}.".format(key)) + raise ValueError( + "Can't specify both num_oov_buckets and default_value in {}.".format( + key)) if num_oov_buckets < 0: - raise ValueError('Invalid num_oov_buckets {} in {}.'.format(num_oov_buckets, key)) - fc_utils.assert_string_or_int(vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key)) + raise ValueError('Invalid num_oov_buckets {} in {}.'.format( + num_oov_buckets, key)) + fc_utils.assert_string_or_int( + vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key)) if dtype is None: dtype = vocabulary_dtype elif dtype.is_integer != vocabulary_dtype.is_integer: raise ValueError( - 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format(dtype, vocabulary_dtype, key) - ) + 'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format( + dtype, vocabulary_dtype, key)) fc_utils.assert_string_or_int(dtype, prefix='column_name: {}'.format(key)) fc_utils.assert_key_is_string(key) return VocabularyListCategoricalColumn( - feature_name=feature_name, - key=key, - vocabulary_list=tuple(vocabulary_list), - dtype=dtype, - default_value=default_value, - num_oov_buckets=num_oov_buckets, + feature_name=feature_name, + key=key, + vocabulary_list=tuple(vocabulary_list), + dtype=dtype, + default_value=default_value, + num_oov_buckets=num_oov_buckets, ) -def categorical_column_with_identity(key, num_buckets, default_value=None, feature_name=None): +def categorical_column_with_identity(key, + num_buckets, + default_value=None, + feature_name=None): """A `CategoricalColumn` that returns identity values. Use this when your inputs are integers in the range `[0, num_buckets)`, and @@ -1770,15 +1809,19 @@ def categorical_column_with_identity(key, num_buckets, default_value=None, featu ValueError: if `default_value` is not in range `[0, num_buckets)`. """ if num_buckets < 1: - raise ValueError('num_buckets {} < 1, column_name {}'.format(num_buckets, key)) - if (default_value is not None) and ((default_value < 0) or (default_value >= num_buckets)): - raise ValueError('default_value {} not in range [0, {}), column_name {}'.format(default_value, num_buckets, key)) + raise ValueError('num_buckets {} < 1, column_name {}'.format( + num_buckets, key)) + if (default_value is not None) and ((default_value < 0) or + (default_value >= num_buckets)): + raise ValueError( + 'default_value {} not in range [0, {}), column_name {}'.format( + default_value, num_buckets, key)) fc_utils.assert_key_is_string(key) return IdentityCategoricalColumn( - feature_name=feature_name, - key=key, - number_buckets=num_buckets, - default_value=default_value, + feature_name=feature_name, + key=key, + number_buckets=num_buckets, + default_value=default_value, ) @@ -1816,7 +1859,9 @@ def indicator_column(categorical_column): return IndicatorColumn(categorical_column) -def weighted_categorical_column(categorical_column, weight_feature_key, dtype=dtypes.float32): +def weighted_categorical_column(categorical_column, + weight_feature_key, + dtype=dtypes.float32): """Applies weight values to a `CategoricalColumn`. Use this when each of your sparse inputs has both an ID and a value. For @@ -1884,9 +1929,9 @@ def weighted_categorical_column(categorical_column, weight_feature_key, dtype=dt if (dtype is None) or not (dtype.is_integer or dtype.is_floating): raise ValueError('dtype {} is not convertible to float.'.format(dtype)) return WeightedCategoricalColumn( - categorical_column=categorical_column, - weight_feature_key=weight_feature_key, - dtype=dtype, + categorical_column=categorical_column, + weight_feature_key=weight_feature_key, + dtype=dtype, ) @@ -1995,27 +2040,29 @@ def crossed_column(keys, hash_bucket_size, hash_key=None, feature_name=None): ValueError: If `hash_bucket_size < 1`. """ if not hash_bucket_size or hash_bucket_size < 1: - raise ValueError('hash_bucket_size must be > 1. ' 'hash_bucket_size: {}'.format(hash_bucket_size)) + raise ValueError('hash_bucket_size must be > 1. ' + 'hash_bucket_size: {}'.format(hash_bucket_size)) if not keys or len(keys) < 2: - raise ValueError('keys must be a list with length > 1. Given: {}'.format(keys)) + raise ValueError( + 'keys must be a list with length > 1. Given: {}'.format(keys)) for key in keys: - if not isinstance(key, six.string_types) and not isinstance(key, (CategoricalColumn, fc_old._CategoricalColumn)): + if not isinstance(key, six.string_types) and not isinstance( + key, (CategoricalColumn, fc_old._CategoricalColumn)): raise ValueError( - 'Unsupported key type. All keys must be either string, or ' - 'categorical column except HashedCategoricalColumn. ' - 'Given: {}'.format(key) - ) - if isinstance(key, (HashedCategoricalColumn, fc_old._HashedCategoricalColumn)): + 'Unsupported key type. All keys must be either string, or ' + 'categorical column except HashedCategoricalColumn. ' + 'Given: {}'.format(key)) + if isinstance(key, + (HashedCategoricalColumn, fc_old._HashedCategoricalColumn)): raise ValueError( - 'categorical_column_with_hash_bucket is not supported for crossing. ' - 'Hashing before crossing will increase probability of collision. ' - 'Instead, use the feature name as a string. Given: {}'.format(key) - ) + 'categorical_column_with_hash_bucket is not supported for crossing. ' + 'Hashing before crossing will increase probability of collision. ' + 'Instead, use the feature name as a string. Given: {}'.format(key)) return CrossedColumn( - feature_name=feature_name, - keys=tuple(keys), - hash_bucket_size=hash_bucket_size, - hash_key=hash_key, + feature_name=feature_name, + keys=tuple(keys), + hash_bucket_size=hash_bucket_size, + hash_key=hash_key, ) @@ -2246,26 +2293,28 @@ def is_feature_column_v2(feature_columns): return True -def _create_weighted_sum(column, transformation_cache, state_manager, sparse_combiner, weight_var): +def _create_weighted_sum(column, transformation_cache, state_manager, + sparse_combiner, weight_var): """Creates a weighted sum for a dense/categorical column for linear_model.""" if isinstance(column, CategoricalColumn): return _create_categorical_column_weighted_sum( - column=column, - transformation_cache=transformation_cache, - state_manager=state_manager, - sparse_combiner=sparse_combiner, - weight_var=weight_var, + column=column, + transformation_cache=transformation_cache, + state_manager=state_manager, + sparse_combiner=sparse_combiner, + weight_var=weight_var, ) else: return _create_dense_column_weighted_sum( - column=column, - transformation_cache=transformation_cache, - state_manager=state_manager, - weight_var=weight_var, + column=column, + transformation_cache=transformation_cache, + state_manager=state_manager, + weight_var=weight_var, ) -def _create_dense_column_weighted_sum(column, transformation_cache, state_manager, weight_var): +def _create_dense_column_weighted_sum(column, transformation_cache, + state_manager, weight_var): """Create a weighted sum of a dense column for linear_model.""" tensor = column.get_dense_tensor(transformation_cache, state_manager) num_elements = column.variable_shape.num_elements() @@ -2280,7 +2329,8 @@ class CategoricalColumn(FeatureColumn): A categorical feature typically handled with a `tf.SparseTensor` of IDs. """ - IdWeightPair = collections.namedtuple('IdWeightPair', ('id_tensor', 'weight_tensor')) + IdWeightPair = collections.namedtuple('IdWeightPair', + ('id_tensor', 'weight_tensor')) @abc.abstractproperty def num_buckets(self): @@ -2308,7 +2358,9 @@ def get_sparse_tensors(self, transformation_cache, state_manager): """ -def _create_categorical_column_weighted_sum(column, transformation_cache, state_manager, sparse_combiner, weight_var): +def _create_categorical_column_weighted_sum(column, transformation_cache, + state_manager, sparse_combiner, + weight_var): """Create a weighted sum of a categorical column for linear_model. Note to maintainer: As implementation details, the weighted sum is @@ -2335,25 +2387,30 @@ def _create_categorical_column_weighted_sum(column, transformation_cache, state_ For both cases, we can implement weighted sum via embedding_lookup with sparse_combiner = "sum". """ - sparse_tensors = column.get_sparse_tensors(transformation_cache, state_manager) - id_tensor = sparse_ops.sparse_reshape(sparse_tensors.id_tensor, [array_ops.shape(sparse_tensors.id_tensor)[0], -1]) + sparse_tensors = column.get_sparse_tensors(transformation_cache, + state_manager) + id_tensor = sparse_ops.sparse_reshape( + sparse_tensors.id_tensor, + [array_ops.shape(sparse_tensors.id_tensor)[0], -1]) weight_tensor = sparse_tensors.weight_tensor if weight_tensor is not None: - weight_tensor = sparse_ops.sparse_reshape(weight_tensor, [array_ops.shape(weight_tensor)[0], -1]) + weight_tensor = sparse_ops.sparse_reshape( + weight_tensor, [array_ops.shape(weight_tensor)[0], -1]) return embedding_lookup_sparse( - weight_var, - id_tensor, - sparse_weights=weight_tensor, - combiner=sparse_combiner, - name='weighted_sum', + weight_var, + id_tensor, + sparse_weights=weight_tensor, + combiner=sparse_combiner, + name='weighted_sum', ) class SequenceDenseColumn(FeatureColumn): """Represents dense sequence data.""" - TensorSequenceLengthPair = collections.namedtuple('TensorSequenceLengthPair', ('dense_tensor', 'sequence_length')) + TensorSequenceLengthPair = collections.namedtuple( + 'TensorSequenceLengthPair', ('dense_tensor', 'sequence_length')) @abc.abstractmethod def get_sequence_dense_tensor(self, transformation_cache, state_manager): @@ -2442,7 +2499,8 @@ def get(self, key, state_manager): raise ValueError('Feature {} is not in features dictionary.'.format(key)) if not isinstance(key, FeatureColumn): - raise TypeError('"key" must be either a "str" or "FeatureColumn". ' 'Provided: {}'.format(key)) + raise TypeError('"key" must be either a "str" or "FeatureColumn". ' + 'Provided: {}'.format(key)) column = key logging.debug('Transforming feature_column %s.', column) @@ -2471,34 +2529,37 @@ def _get_raw_feature_as_tensor(self, key): ValueError: if the raw feature has rank 0. """ raw_feature = self._features[key] - feature_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(raw_feature) + feature_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( + raw_feature) def expand_dims(input_tensor): # Input_tensor must have rank 1. if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): - return sparse_ops.sparse_reshape(input_tensor, [array_ops.shape(input_tensor)[0], 1]) + return sparse_ops.sparse_reshape(input_tensor, + [array_ops.shape(input_tensor)[0], 1]) else: return array_ops.expand_dims(input_tensor, -1) rank = feature_tensor.get_shape().ndims if rank is not None: if rank == 0: - raise ValueError('Feature (key: {}) cannot have rank 0. Give: {}'.format(key, feature_tensor)) + raise ValueError( + 'Feature (key: {}) cannot have rank 0. Give: {}'.format( + key, feature_tensor)) return feature_tensor if rank != 1 else expand_dims(feature_tensor) # Handle dynamic rank. - with ops.control_dependencies( - [ + with ops.control_dependencies([ check_ops.assert_positive( - array_ops.rank(feature_tensor), - message='Feature (key: {}) cannot have rank 0. Given: {}'.format(key, feature_tensor), + array_ops.rank(feature_tensor), + message='Feature (key: {}) cannot have rank 0. Given: {}'.format( + key, feature_tensor), ) - ] - ): + ]): return control_flow_ops.cond( - math_ops.equal(1, array_ops.rank(feature_tensor)), - lambda: expand_dims(feature_tensor), - lambda: feature_tensor, + math_ops.equal(1, array_ops.rank(feature_tensor)), + lambda: expand_dims(feature_tensor), + lambda: feature_tensor, ) @@ -2523,16 +2584,17 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): if 'RaggedTensor' in str(type(input_tensor)): return input_tensor - input_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(input_tensor) + input_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( + input_tensor) if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): return input_tensor with ops.name_scope( - None, - 'to_sparse_input', - ( - input_tensor, - ignore_value, - ), + None, + 'to_sparse_input', + ( + input_tensor, + ignore_value, + ), ): if ignore_value is None: if input_tensor.dtype == dtypes.string: @@ -2545,12 +2607,15 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): # constructing a new numpy object of the given type, which yields the # default value for that type. ignore_value = input_tensor.dtype.as_numpy_dtype() - ignore_value = math_ops.cast(ignore_value, input_tensor.dtype, name='ignore_value') - indices = array_ops.where(math_ops.not_equal(input_tensor, ignore_value), name='indices') + ignore_value = math_ops.cast( + ignore_value, input_tensor.dtype, name='ignore_value') + indices = array_ops.where( + math_ops.not_equal(input_tensor, ignore_value), name='indices') return sparse_tensor_lib.SparseTensor( - indices=indices, - values=array_ops.gather_nd(input_tensor, indices, name='values'), - dense_shape=array_ops.shape(input_tensor, out_type=dtypes.int64, name='dense_shape'), + indices=indices, + values=array_ops.gather_nd(input_tensor, indices, name='values'), + dense_shape=array_ops.shape( + input_tensor, out_type=dtypes.int64, name='dense_shape'), ) @@ -2581,33 +2646,32 @@ def _normalize_feature_columns(feature_columns): for column in feature_columns: if not isinstance(column, FeatureColumn): - raise ValueError( - 'Items of feature_columns must be a FeatureColumn. ' 'Given (type {}): {}.'.format(type(column), column) - ) + raise ValueError('Items of feature_columns must be a FeatureColumn. ' + 'Given (type {}): {}.'.format(type(column), column)) if not feature_columns: raise ValueError('feature_columns must not be empty.') name_to_column = {} for column in feature_columns: if column.name in name_to_column: - raise ValueError( - 'Duplicate feature column name found for columns: {} ' - 'and {}. This usually means that these columns refer to ' - 'same base feature. Either one must be discarded or a ' - 'duplicated but renamed item must be inserted in ' - 'features dict.'.format(column, name_to_column[column.name]) - ) + raise ValueError('Duplicate feature column name found for columns: {} ' + 'and {}. This usually means that these columns refer to ' + 'same base feature. Either one must be discarded or a ' + 'duplicated but renamed item must be inserted in ' + 'features dict.'.format(column, + name_to_column[column.name])) name_to_column[column.name] = column return sorted(feature_columns, key=lambda x: x.name) class NumericColumn( - DenseColumn, - fc_old._DenseColumn, - collections.namedtuple( - 'NumericColumn', - ('feature_name', 'key', 'shape', 'default_value', 'dtype', 'normalizer_fn'), - ), + DenseColumn, + fc_old._DenseColumn, + collections.namedtuple( + 'NumericColumn', + ('feature_name', 'key', 'shape', 'default_value', 'dtype', + 'normalizer_fn'), + ), ): """See `numeric_column`.""" @@ -2628,24 +2692,29 @@ def raw_name(self): @property def parse_example_spec(self): """See `FeatureColumn` base class.""" - return {self.key: parsing_ops.FixedLenFeature(self.shape, self.dtype, self.default_value)} + return { + self.key: + parsing_ops.FixedLenFeature(self.shape, self.dtype, + self.default_value) + } @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec def _transform_input_tensor(self, input_tensor): if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): raise ValueError( - 'The corresponding Tensor of numerical column must be a Tensor. ' - 'SparseTensor is not supported. key: {}'.format(self.key) - ) + 'The corresponding Tensor of numerical column must be a Tensor. ' + 'SparseTensor is not supported. key: {}'.format(self.key)) if self.normalizer_fn is not None: input_tensor = self.normalizer_fn(input_tensor) return math_ops.cast(input_tensor, dtypes.float32) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): input_tensor = inputs.get(self.key) return self._transform_input_tensor(input_tensor) @@ -2676,7 +2745,8 @@ def variable_shape(self): return tensor_shape.TensorShape(self.shape) @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): return self.variable_shape @@ -2696,7 +2766,8 @@ def get_dense_tensor(self, transformation_cache, state_manager): # representation created by _transform_feature. return transformation_cache.get(self, state_manager) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -2719,23 +2790,25 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): """See 'FeatureColumn` base class.""" _check_config_keys(config, cls._fields) kwargs = config.copy() - kwargs['normalizer_fn'] = utils.deserialize_keras_object(config['normalizer_fn'], custom_objects=custom_objects) + kwargs['normalizer_fn'] = utils.deserialize_keras_object( + config['normalizer_fn'], custom_objects=custom_objects) kwargs['dtype'] = dtypes.as_dtype(config['dtype']) return cls(**kwargs) class BucketizedColumn( - DenseColumn, - CategoricalColumn, - fc_old._DenseColumn, - fc_old._CategoricalColumn, - collections.namedtuple('BucketizedColumn', ('source_column', 'boundaries')), + DenseColumn, + CategoricalColumn, + fc_old._DenseColumn, + fc_old._CategoricalColumn, + collections.namedtuple('BucketizedColumn', ('source_column', 'boundaries')), ): """See `bucketized_column`.""" @property def _is_v2_column(self): - return isinstance(self.source_column, FeatureColumn) and self.source_column._is_v2_column + return isinstance(self.source_column, + FeatureColumn) and self.source_column._is_v2_column @property def name(self): @@ -2753,11 +2826,13 @@ def parse_example_spec(self): return self.source_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.source_column._parse_example_spec - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Returns bucketized categorical `source_column` tensor.""" source_tensor = inputs.get(self.source_column) @@ -2771,19 +2846,21 @@ def transform_feature(self, transformation_cache, state_manager): @property def variable_shape(self): """See `DenseColumn` base class.""" - return tensor_shape.TensorShape(tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) + return tensor_shape.TensorShape( + tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): return self.variable_shape def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( - indices=math_ops.cast(input_tensor, dtypes.int64), - depth=len(self.boundaries) + 1, - on_value=1.0, - off_value=0.0, + indices=math_ops.cast(input_tensor, dtypes.int64), + depth=len(self.boundaries) + 1, + on_value=1.0, + off_value=0.0, ) def get_dense_tensor(self, transformation_cache, state_manager): @@ -2791,7 +2868,8 @@ def get_dense_tensor(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_dense_tensor_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -2805,7 +2883,8 @@ def num_buckets(self): return (len(self.boundaries) + 1) * self.source_column.shape[0] @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets @@ -2815,20 +2894,24 @@ def _get_sparse_tensors_for_input_tensor(self, input_tensor): source_dimension = self.source_column.shape[0] i1 = array_ops.reshape( - array_ops.tile( - array_ops.expand_dims(math_ops.range(0, batch_size), 1), - [1, source_dimension], - ), - (-1,), + array_ops.tile( + array_ops.expand_dims(math_ops.range(0, batch_size), 1), + [1, source_dimension], + ), + (-1,), ) i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size]) # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets - bucket_indices = array_ops.reshape(input_tensor, (-1,)) + (len(self.boundaries) + 1) * i2 + bucket_indices = array_ops.reshape(input_tensor, + (-1,)) + (len(self.boundaries) + 1) * i2 - indices = math_ops.cast(array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64) - dense_shape = math_ops.cast(array_ops.stack([batch_size, source_dimension]), dtypes.int64) - sparse_tensor = sparse_tensor_lib.SparseTensor(indices=indices, values=bucket_indices, dense_shape=dense_shape) + indices = math_ops.cast( + array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64) + dense_shape = math_ops.cast( + array_ops.stack([batch_size, source_dimension]), dtypes.int64) + sparse_tensor = sparse_tensor_lib.SparseTensor( + indices=indices, values=bucket_indices, dense_shape=dense_shape) return CategoricalColumn.IdWeightPair(sparse_tensor, None) def get_sparse_tensors(self, transformation_cache, state_manager): @@ -2836,8 +2919,12 @@ def get_sparse_tensors(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_sparse_tensors_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): """Converts dense inputs to SparseTensor so downstream code can use it.""" del weight_collections del trainable @@ -2860,22 +2947,25 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): """See 'FeatureColumn` base class.""" _check_config_keys(config, cls._fields) kwargs = config.copy() - kwargs['source_column'] = deserialize_feature_column(config['source_column'], custom_objects, columns_by_name) + kwargs['source_column'] = deserialize_feature_column( + config['source_column'], custom_objects, columns_by_name) return cls(**kwargs) class SequenceBucketizedColumn( - DenseColumn, - CategoricalColumn, - fc_old._DenseColumn, - fc_old._CategoricalColumn, - collections.namedtuple('SequenceBucketizedColumn', ('source_column', 'boundaries')), + DenseColumn, + CategoricalColumn, + fc_old._DenseColumn, + fc_old._CategoricalColumn, + collections.namedtuple('SequenceBucketizedColumn', + ('source_column', 'boundaries')), ): """See `bucketized_column`.""" @property def _is_v2_column(self): - return isinstance(self.source_column, FeatureColumn) and self.source_column._is_v2_column + return isinstance(self.source_column, + FeatureColumn) and self.source_column._is_v2_column @property def name(self): @@ -2893,19 +2983,22 @@ def parse_example_spec(self): return self.source_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.source_column._parse_example_spec - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Returns bucketized categorical `source_column` tensor.""" source_tensor = inputs.get(self.source_column) - bucketize_values = math_ops._bucketize(source_tensor.values, boundaries=self.boundaries) + bucketize_values = math_ops._bucketize( + source_tensor.values, boundaries=self.boundaries) bucketize_tensor = sparse_tensor_lib.SparseTensor( - indices=source_tensor.indices, - values=bucketize_values, - dense_shape=source_tensor.dense_shape, + indices=source_tensor.indices, + values=bucketize_values, + dense_shape=source_tensor.dense_shape, ) return bucketize_tensor @@ -2917,19 +3010,21 @@ def transform_feature(self, transformation_cache, state_manager): @property def variable_shape(self): """See `DenseColumn` base class.""" - return tensor_shape.TensorShape(tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) + return tensor_shape.TensorShape( + tuple(self.source_column.shape) + (len(self.boundaries) + 1,)) @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): return self.variable_shape def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( - indices=math_ops.cast(input_tensor, dtypes.int64), - depth=len(self.boundaries) + 1, - on_value=1.0, - off_value=0.0, + indices=math_ops.cast(input_tensor, dtypes.int64), + depth=len(self.boundaries) + 1, + on_value=1.0, + off_value=0.0, ) def get_dense_tensor(self, transformation_cache, state_manager): @@ -2937,7 +3032,8 @@ def get_dense_tensor(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_dense_tensor_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -2951,7 +3047,8 @@ def num_buckets(self): return (len(self.boundaries) + 1) * self.source_column.shape[0] @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets @@ -2965,12 +3062,13 @@ def _get_sparse_tensors_for_input_tensor(self, input_sparse_tensor): i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size]) # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets - bucket_indices = array_ops.reshape(input_tensor, (-1,)) + (len(self.boundaries) + 1) * i2 + bucket_indices = array_ops.reshape(input_tensor, + (-1,)) + (len(self.boundaries) + 1) * i2 sparse_tensor = sparse_tensor_lib.SparseTensor( - indices=input_indices, - values=bucket_indices, - dense_shape=input_sparse_tensor.dense_shape, + indices=input_indices, + values=bucket_indices, + dense_shape=input_sparse_tensor.dense_shape, ) # Compute the third dimension explicitly instead of setting it to -1, as # that doesn't work for dynamically shaped tensors with 0-length at runtime. @@ -2985,8 +3083,12 @@ def get_sparse_tensors(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_sparse_tensors_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): """Converts dense inputs to SparseTensor so downstream code can use it.""" del weight_collections del trainable @@ -3009,22 +3111,25 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): """See 'FeatureColumn` base class.""" _check_config_keys(config, cls._fields) kwargs = config.copy() - kwargs['source_column'] = deserialize_feature_column(config['source_column'], custom_objects, columns_by_name) + kwargs['source_column'] = deserialize_feature_column( + config['source_column'], custom_objects, columns_by_name) return cls(**kwargs) class SequenceNumericColumn( - DenseColumn, - CategoricalColumn, - fc_old._DenseColumn, - fc_old._CategoricalColumn, - collections.namedtuple('SequenceNumericColumn', ('source_column', 'sequence_length')), + DenseColumn, + CategoricalColumn, + fc_old._DenseColumn, + fc_old._CategoricalColumn, + collections.namedtuple('SequenceNumericColumn', + ('source_column', 'sequence_length')), ): """See `SequenceNumericColumn`.""" @property def _is_v2_column(self): - return isinstance(self.source_column, FeatureColumn) and self.source_column._is_v2_column + return isinstance(self.source_column, + FeatureColumn) and self.source_column._is_v2_column @property def name(self): @@ -3042,11 +3147,13 @@ def parse_example_spec(self): return self.source_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.source_column._parse_example_spec - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Returns bucketized categorical `source_column` tensor.""" source_tensor = inputs.get(self.source_column) @@ -3060,19 +3167,21 @@ def transform_feature(self, transformation_cache, state_manager): @property def variable_shape(self): """See `DenseColumn` base class.""" - return tensor_shape.TensorShape(tuple(self.source_column.shape) + (self.sequence_length,)) + return tensor_shape.TensorShape( + tuple(self.source_column.shape) + (self.sequence_length,)) @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): return self.variable_shape def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( - indices=math_ops.cast(input_tensor, dtypes.int64), - depth=self.sequence_length, - on_value=1.0, - off_value=0.0, + indices=math_ops.cast(input_tensor, dtypes.int64), + depth=self.sequence_length, + on_value=1.0, + off_value=0.0, ) def get_dense_tensor(self, transformation_cache, state_manager): @@ -3080,7 +3189,8 @@ def get_dense_tensor(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_dense_tensor_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable @@ -3089,12 +3199,15 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): def _get_sequence_dense_tensor(self, inputs): input_tensor = inputs.get(self) - sparse_tensors = self._get_sparse_tensors_for_input_tensor(input_tensor).id_tensor - sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors) + sparse_tensors = self._get_sparse_tensors_for_input_tensor( + input_tensor).id_tensor + sequence_length = fc_utils.sequence_length_from_sparse_tensor( + sparse_tensors) sequence_length = tf.cast(sequence_length, tf.int32) shape = array_ops.shape(sparse_tensors) target_shape = [shape[0], shape[1], math_ops.reduce_prod(shape[2:])] - ret_tensor = tf.sparse_to_dense(sparse_tensors.indices, target_shape, sparse_tensors.values) + ret_tensor = tf.sparse_to_dense(sparse_tensors.indices, target_shape, + sparse_tensors.values) return CategoricalColumn.IdWeightPair(ret_tensor, sequence_length) @property @@ -3104,7 +3217,8 @@ def num_buckets(self): return self.sequence_length * self.source_column.shape[0] @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets @@ -3122,8 +3236,12 @@ def get_sparse_tensors(self, transformation_cache, state_manager): input_tensor = transformation_cache.get(self, state_manager) return self._get_sparse_tensors_for_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): """Converts dense inputs to SparseTensor so downstream code can use it.""" del weight_collections del trainable @@ -3146,28 +3264,31 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): """See 'FeatureColumn` base class.""" _check_config_keys(config, cls._fields) kwargs = config.copy() - kwargs['source_column'] = deserialize_feature_column(config['source_column'], custom_objects, columns_by_name) + kwargs['source_column'] = deserialize_feature_column( + config['source_column'], custom_objects, columns_by_name) return cls(**kwargs) class SequenceWeightedCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, - collections.namedtuple( - 'SequenceWeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype'), - ), + CategoricalColumn, + fc_old._CategoricalColumn, + collections.namedtuple( + 'SequenceWeightedCategoricalColumn', + ('categorical_column', 'weight_feature_key', 'dtype'), + ), ): """See `weighted_categorical_column`.""" @property def _is_v2_column(self): - return isinstance(self.categorical_column, FeatureColumn) and self.categorical_column._is_v2_column + return isinstance(self.categorical_column, + FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): """See `FeatureColumn` base class.""" - return '{}_weighted_by_{}'.format(self.categorical_column.name, self.weight_feature_key) + return '{}_weighted_by_{}'.format(self.categorical_column.name, + self.weight_feature_key) @property def raw_name(self): @@ -3179,20 +3300,19 @@ def parse_example_spec(self): """See `FeatureColumn` base class.""" config = self.categorical_column.parse_example_spec if self.weight_feature_key in config: - raise ValueError( - 'Parse config {} already exists for {}.'.format(config[self.weight_feature_key], self.weight_feature_key) - ) + raise ValueError('Parse config {} already exists for {}.'.format( + config[self.weight_feature_key], self.weight_feature_key)) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): config = self.categorical_column._parse_example_spec if self.weight_feature_key in config: - raise ValueError( - 'Parse config {} already exists for {}.'.format(config[self.weight_feature_key], self.weight_feature_key) - ) + raise ValueError('Parse config {} already exists for {}.'.format( + config[self.weight_feature_key], self.weight_feature_key)) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @@ -3202,19 +3322,23 @@ def num_buckets(self): return self.categorical_column.num_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.categorical_column._num_buckets def _transform_weight_tensor(self, weight_tensor): if weight_tensor is None: raise ValueError('Missing weights {}.'.format(self.weight_feature_key)) - weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(weight_tensor) + weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( + weight_tensor) if self.dtype != weight_tensor.dtype.base_dtype: - raise ValueError('Bad dtype, expected {}, but got {}.'.format(self.dtype, weight_tensor.dtype)) + raise ValueError('Bad dtype, expected {}, but got {}.'.format( + self.dtype, weight_tensor.dtype)) if not isinstance(weight_tensor, sparse_tensor_lib.SparseTensor): # The weight tensor can be a regular Tensor. In this case, sparsify it. - weight_tensor = _to_sparse_input_and_drop_ignore_values(weight_tensor, ignore_value=0.0) + weight_tensor = _to_sparse_input_and_drop_ignore_values( + weight_tensor, ignore_value=0.0) if not weight_tensor.dtype.is_floating: weight_tensor = math_ops.cast(weight_tensor, dtypes.float32) shape = tf.shape(weight_tensor) @@ -3224,14 +3348,16 @@ def _transform_weight_tensor(self, weight_tensor): def transform_feature(self, transformation_cache, state_manager): """Applies weights to tensor generated from `categorical_column`'.""" - weight_tensor = transformation_cache.get(self.weight_feature_key, state_manager) + weight_tensor = transformation_cache.get(self.weight_feature_key, + state_manager) weight_tensor = self._transform_weight_tensor(weight_tensor) return ( - transformation_cache.get(self.categorical_column, state_manager), - weight_tensor, + transformation_cache.get(self.categorical_column, state_manager), + weight_tensor, ) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Applies weights to tensor generated from `categorical_column`'.""" weight_tensor = inputs.get(self.weight_feature_key) @@ -3243,8 +3369,12 @@ def get_sparse_tensors(self, transformation_cache, state_manager): tensors = transformation_cache.get(self, state_manager) return CategoricalColumn.IdWeightPair(tensors[0], tensors[1]) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): del weight_collections del trainable tensors = inputs.get(self) @@ -3258,7 +3388,8 @@ def parents(self): def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) - config['categorical_column'] = serialize_feature_column(self.categorical_column) + config['categorical_column'] = serialize_feature_column( + self.categorical_column) config['dtype'] = self.dtype.name return config @@ -3268,38 +3399,38 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['categorical_column'] = deserialize_feature_column( - config['categorical_column'], custom_objects, columns_by_name - ) + config['categorical_column'], custom_objects, columns_by_name) kwargs['dtype'] = dtypes.as_dtype(config['dtype']) return cls(**kwargs) class EmbeddingColumn( - DenseColumn, - SequenceDenseColumn, - fc_old._DenseColumn, - fc_old._SequenceDenseColumn, - collections.namedtuple( - 'EmbeddingColumn', - ( - 'categorical_column', - 'dimension', - 'combiner', - 'initializer', - 'ckpt_to_load_from', - 'tensor_name_in_ckpt', - 'max_norm', - 'trainable', - 'partitioner', - 'ev_params', + DenseColumn, + SequenceDenseColumn, + fc_old._DenseColumn, + fc_old._SequenceDenseColumn, + collections.namedtuple( + 'EmbeddingColumn', + ( + 'categorical_column', + 'dimension', + 'combiner', + 'initializer', + 'ckpt_to_load_from', + 'tensor_name_in_ckpt', + 'max_norm', + 'trainable', + 'partitioner', + 'ev_params', + ), ), - ), ): """See `embedding_column`.""" @property def _is_v2_column(self): - return isinstance(self.categorical_column, FeatureColumn) and self.categorical_column._is_v2_column + return isinstance(self.categorical_column, + FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): @@ -3317,7 +3448,8 @@ def parse_example_spec(self): return self.categorical_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.categorical_column._parse_example_spec @@ -3325,7 +3457,8 @@ def transform_feature(self, transformation_cache, state_manager): """Transforms underlying `categorical_column`.""" return transformation_cache.get(self.categorical_column, state_manager) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): return inputs.get(self.categorical_column) @@ -3335,7 +3468,8 @@ def variable_shape(self): return tensor_shape.TensorShape([self.dimension]) @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): return self.variable_shape @@ -3343,16 +3477,17 @@ def create_state(self, state_manager): """Creates the embedding lookup variable.""" embedding_shape = (self.categorical_column._num_buckets, self.dimension) state_manager.create_variable( - self, - name='embedding_weights', - shape=embedding_shape, - dtype=dtypes.float32, - trainable=self.trainable, - use_resource=True, - initializer=self.initializer, + self, + name='embedding_weights', + shape=embedding_shape, + dtype=dtypes.float32, + trainable=self.trainable, + use_resource=True, + initializer=self.initializer, ) - def _get_dense_tensor_internal_helper(self, sparse_tensors, embedding_weights): + def _get_dense_tensor_internal_helper(self, sparse_tensors, + embedding_weights): sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor @@ -3360,106 +3495,114 @@ def _get_dense_tensor_internal_helper(self, sparse_tensors, embedding_weights): to_restore = embedding_weights if isinstance(to_restore, variables.PartitionedVariable): to_restore = to_restore._get_variable_list() - checkpoint_utils.init_from_checkpoint(self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) + checkpoint_utils.init_from_checkpoint( + self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) if 'RaggedTensor' in str(type(sparse_ids)): return embedding_lookup_ragged( + embedding_weights, + sparse_ids, + sparse_weights, + combiner=self.combiner, + max_norm=self.max_norm, + name='%s_weights' % self.name, + ) + + # Return embedding lookup result. + return embedding_lookup_sparse( embedding_weights, sparse_ids, sparse_weights, combiner=self.combiner, - max_norm=self.max_norm, name='%s_weights' % self.name, - ) - - # Return embedding lookup result. - return embedding_lookup_sparse( - embedding_weights, - sparse_ids, - sparse_weights, - combiner=self.combiner, - name='%s_weights' % self.name, - max_norm=self.max_norm, + max_norm=self.max_norm, ) def _get_dense_tensor_internal(self, sparse_tensors, state_manager): """Private method that follows the signature of get_dense_tensor.""" - embedding_weights = state_manager.get_variable(self, name='embedding_weights') - return self._get_dense_tensor_internal_helper(sparse_tensors, embedding_weights) + embedding_weights = state_manager.get_variable( + self, name='embedding_weights') + return self._get_dense_tensor_internal_helper(sparse_tensors, + embedding_weights) - def _old_get_dense_tensor_internal(self, sparse_tensors, weight_collections, trainable): + def _old_get_dense_tensor_internal(self, sparse_tensors, weight_collections, + trainable): """Private method that follows the signature of _get_dense_tensor.""" embedding_shape = (self.categorical_column._num_buckets, self.dimension) if weight_collections and ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections: weight_collections.append(ops.GraphKeys.GLOBAL_VARIABLES) if self.ev_params is None: embedding_weights = variable_scope.get_variable( - name='embedding_weights', - shape=embedding_shape, - dtype=dtypes.float32, # bfloat16, - initializer=self.initializer, - trainable=self.trainable and trainable, - partitioner=self.partitioner, - collections=weight_collections, + name='embedding_weights', + shape=embedding_shape, + dtype=dtypes.float32, # bfloat16, + initializer=self.initializer, + trainable=self.trainable and trainable, + partitioner=self.partitioner, + collections=weight_collections, ) else: # at eval or inference time, it is necessary to set # the initializers to zeros, so that new key will # get zero embedding - if os.environ.get('tf.estimator.mode', '') != os.environ.get('tf.estimator.ModeKeys.TRAIN', 'train'): + if os.environ.get('tf.estimator.mode', '') != os.environ.get( + 'tf.estimator.ModeKeys.TRAIN', 'train'): initializer = init_ops.zeros_initializer() else: initializer = self.initializer extra_args = {} if 'EmbeddingVariableConfig' in dir(variables): ev_option = variables.EmbeddingVariableOption() - ev_option.filter_strategy = variables.CounterFilter(filter_freq=self.ev_params.filter_freq) + ev_option.filter_strategy = variables.CounterFilter( + filter_freq=self.ev_params.filter_freq) extra_args['ev_option'] = ev_option else: - extra_args['filter_options'] = variables.CounterFilterOptions(self.ev_params.filter_freq) + extra_args['filter_options'] = variables.CounterFilterOptions( + self.ev_params.filter_freq) embedding_weights = variable_scope.get_embedding_variable( - name='embedding_weights', - embedding_dim=self.dimension, - initializer=initializer, - trainable=self.trainable and trainable, - partitioner=self.partitioner, - collections=weight_collections, - steps_to_live=self.ev_params.steps_to_live, - **extra_args, + name='embedding_weights', + embedding_dim=self.dimension, + initializer=initializer, + trainable=self.trainable and trainable, + partitioner=self.partitioner, + collections=weight_collections, + steps_to_live=self.ev_params.steps_to_live, + **extra_args, ) # Write the embedding configuration to RTP-specified collections. This will inform RTP to # optimize this embedding operation. embedding_attrs = layer_utils.gen_embedding_attrs( - column=self, - variable=embedding_weights, - bucket_size=self.categorical_column._num_buckets, - combiner=self.combiner, - is_embedding_var=(self.ev_params is not None), + column=self, + variable=embedding_weights, + bucket_size=self.categorical_column._num_buckets, + combiner=self.combiner, + is_embedding_var=(self.ev_params is not None), ) embedding_attrs['name'] = layer_utils.unique_name_in_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name'] - ) - layer_utils.update_attr_to_collection(compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs) + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name']) + layer_utils.update_attr_to_collection( + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs) # operate embedding - predictions = self._get_dense_tensor_internal_helper(sparse_tensors, embedding_weights) + predictions = self._get_dense_tensor_internal_helper( + sparse_tensors, embedding_weights) # Update the information about the output and input nodes of embedding operation to the # previous written RTP-specific collection entry. RTP uses these informations to extract # the embedding subgraph. if isinstance(sparse_tensors.id_tensor, sparse_tensor_lib.SparseTensor): layer_utils.append_tensor_to_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, - embedding_attrs['name'], - 'tensor', - predictions, + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, + embedding_attrs['name'], + 'tensor', + predictions, ) layer_utils.append_tensor_to_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, - embedding_attrs['name'], - 'input', - sparse_tensors.id_tensor, + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, + embedding_attrs['name'], + 'input', + sparse_tensors.id_tensor, ) return predictions @@ -3481,76 +3624,91 @@ def get_dense_tensor(self, transformation_cache, state_manager): """ if isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must not be of type SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use DenseFeatures, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) + 'In embedding_column: {}. ' + 'categorical_column must not be of type SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use DenseFeatures, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'SequenceFeatures instead of DenseFeatures. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) # Get sparse IDs and weights. - sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) + sparse_tensors = self.categorical_column.get_sparse_tensors( + transformation_cache, state_manager) return self._get_dense_tensor_internal(sparse_tensors, state_manager) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): if isinstance( - self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), + self.categorical_column, + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), ): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must not be of type _SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use DenseFeatures, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs, weight_collections, trainable) - return self._old_get_dense_tensor_internal(sparse_tensors, weight_collections, trainable) + 'In embedding_column: {}. ' + 'categorical_column must not be of type _SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use DenseFeatures, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'SequenceFeatures instead of DenseFeatures. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) + sparse_tensors = self.categorical_column._get_sparse_tensors( + inputs, weight_collections, trainable) + return self._old_get_dense_tensor_internal(sparse_tensors, + weight_collections, trainable) def get_sequence_dense_tensor(self, transformation_cache, state_manager): """See `SequenceDenseColumn` base class.""" if not isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must be of type SequenceCategoricalColumn ' - 'to use SequenceFeatures. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) - sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) - dense_tensor = self._get_dense_tensor_internal(sparse_tensors, state_manager) - sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) - return SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): + 'In embedding_column: {}. ' + 'categorical_column must be of type SequenceCategoricalColumn ' + 'to use SequenceFeatures. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) + sparse_tensors = self.categorical_column.get_sparse_tensors( + transformation_cache, state_manager) + dense_tensor = self._get_dense_tensor_internal(sparse_tensors, + state_manager) + sequence_length = fc_utils.sequence_length_from_sparse_tensor( + sparse_tensors.id_tensor) + return SequenceDenseColumn.TensorSequenceLengthPair( + dense_tensor=dense_tensor, sequence_length=sequence_length) + + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sequence_dense_tensor(self, + inputs, + weight_collections=None, + trainable=None): if not isinstance( - self.categorical_column, - ( - SequenceCategoricalColumn, - fc_old._SequenceCategoricalColumn, - SequenceBucketizedColumn, - SequenceNumericColumn, - SequenceWeightedCategoricalColumn, - ), + self.categorical_column, + ( + SequenceCategoricalColumn, + fc_old._SequenceCategoricalColumn, + SequenceBucketizedColumn, + SequenceNumericColumn, + SequenceWeightedCategoricalColumn, + ), ): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must be of type SequenceCategoricalColumn ' - 'to use SequenceFeatures. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) + 'In embedding_column: {}. ' + 'categorical_column must be of type SequenceCategoricalColumn ' + 'to use SequenceFeatures. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) dense_tensor = self._old_get_dense_tensor_internal( - sparse_tensors, weight_collections=weight_collections, trainable=trainable - ) - sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) - return SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) + sparse_tensors, + weight_collections=weight_collections, + trainable=trainable) + sequence_length = fc_utils.sequence_length_from_sparse_tensor( + sparse_tensors.id_tensor) + return SequenceDenseColumn.TensorSequenceLengthPair( + dense_tensor=dense_tensor, sequence_length=sequence_length) @property def parents(self): @@ -3560,7 +3718,8 @@ def parents(self): def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) - config['categorical_column'] = serialize_feature_column(self.categorical_column) + config['categorical_column'] = serialize_feature_column( + self.categorical_column) config['initializer'] = utils.serialize_keras_object(self.initializer) return config @@ -3570,31 +3729,30 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['categorical_column'] = deserialize_feature_column( - config['categorical_column'], custom_objects, columns_by_name - ) - kwargs['initializer'] = utils.deserialize_keras_object(config['initializer'], custom_objects=custom_objects) + config['categorical_column'], custom_objects, columns_by_name) + kwargs['initializer'] = utils.deserialize_keras_object( + config['initializer'], custom_objects=custom_objects) return cls(**kwargs) def _raise_shared_embedding_column_error(): - raise ValueError( - 'SharedEmbeddingColumns are not supported in ' - '`linear_model` or `input_layer`. Please use ' - '`DenseFeatures` or `LinearModel` instead.' - ) + raise ValueError('SharedEmbeddingColumns are not supported in ' + '`linear_model` or `input_layer`. Please use ' + '`DenseFeatures` or `LinearModel` instead.') # class SharedEmbeddingColumnCreator(tracking.AutoTrackable): class SharedEmbeddingColumnCreator: + def __init__( - self, - dimension, - initializer, - ckpt_to_load_from, - tensor_name_in_ckpt, - num_buckets, - trainable, - name='shared_embedding_column_creator', + self, + dimension, + initializer, + ckpt_to_load_from, + tensor_name_in_ckpt, + num_buckets, + trainable, + name='shared_embedding_column_creator', ): self._dimension = dimension self._initializer = initializer @@ -3615,18 +3773,19 @@ def embedding_weights(self): if key not in self._embedding_weights: embedding_shape = (self._num_buckets, self._dimension) var = variable_scope.get_variable( - name=self._name, - shape=embedding_shape, - dtype=dtypes.float32, - initializer=self._initializer, - trainable=self._trainable, + name=self._name, + shape=embedding_shape, + dtype=dtypes.float32, + initializer=self._initializer, + trainable=self._trainable, ) if self._ckpt_to_load_from is not None: to_restore = var if isinstance(to_restore, variables.PartitionedVariable): to_restore = to_restore._get_variable_list() - checkpoint_utils.init_from_checkpoint(self._ckpt_to_load_from, {self._tensor_name_in_ckpt: to_restore}) + checkpoint_utils.init_from_checkpoint( + self._ckpt_to_load_from, {self._tensor_name_in_ckpt: to_restore}) self._embedding_weights[key] = var return self._embedding_weights[key] @@ -3636,19 +3795,19 @@ def dimension(self): class SharedEmbeddingColumn( - DenseColumn, - SequenceDenseColumn, - fc_old._DenseColumn, - fc_old._SequenceDenseColumn, - collections.namedtuple( - 'SharedEmbeddingColumn', - ( - 'categorical_column', - 'shared_embedding_column_creator', - 'combiner', - 'max_norm', + DenseColumn, + SequenceDenseColumn, + fc_old._DenseColumn, + fc_old._SequenceDenseColumn, + collections.namedtuple( + 'SharedEmbeddingColumn', + ( + 'categorical_column', + 'shared_embedding_column_creator', + 'combiner', + 'max_norm', + ), ), - ), ): """See `embedding_column`.""" @@ -3685,7 +3844,8 @@ def _transform_feature(self, inputs): @property def variable_shape(self): """See `DenseColumn` base class.""" - return tensor_shape.TensorShape([self.shared_embedding_column_creator.dimension]) + return tensor_shape.TensorShape( + [self.shared_embedding_column_creator.dimension]) @property def _variable_shape(self): @@ -3698,7 +3858,8 @@ def _get_dense_tensor_internal(self, transformation_cache, state_manager): # that the ops for different columns have distinct names. with ops.name_scope(None, default_name=self.name): # Get sparse IDs and weights. - sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) + sparse_tensors = self.categorical_column.get_sparse_tensors( + transformation_cache, state_manager) sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor @@ -3706,26 +3867,26 @@ def _get_dense_tensor_internal(self, transformation_cache, state_manager): # Return embedding lookup result. return embedding_lookup_sparse( - embedding_weights=embedding_weights, - sparse_ids=sparse_ids, - sparse_weights=sparse_weights, - combiner=self.combiner, - name='%s_weights' % self.name, - max_norm=self.max_norm, + embedding_weights=embedding_weights, + sparse_ids=sparse_ids, + sparse_weights=sparse_weights, + combiner=self.combiner, + name='%s_weights' % self.name, + max_norm=self.max_norm, ) def get_dense_tensor(self, transformation_cache, state_manager): """Returns the embedding lookup result.""" if isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must not be of type SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use DenseFeatures, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) + 'In embedding_column: {}. ' + 'categorical_column must not be of type SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use DenseFeatures, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'SequenceFeatures instead of DenseFeatures. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) return self._get_dense_tensor_internal(transformation_cache, state_manager) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): @@ -3735,18 +3896,25 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): """See `SequenceDenseColumn` base class.""" if not isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In embedding_column: {}. ' - 'categorical_column must be of type SequenceCategoricalColumn ' - 'to use SequenceFeatures. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) - dense_tensor = self._get_dense_tensor_internal(transformation_cache, state_manager) - sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) - sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) - return SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) - - def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): + 'In embedding_column: {}. ' + 'categorical_column must be of type SequenceCategoricalColumn ' + 'to use SequenceFeatures. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) + dense_tensor = self._get_dense_tensor_internal(transformation_cache, + state_manager) + sparse_tensors = self.categorical_column.get_sparse_tensors( + transformation_cache, state_manager) + sequence_length = fc_utils.sequence_length_from_sparse_tensor( + sparse_tensors.id_tensor) + return SequenceDenseColumn.TensorSequenceLengthPair( + dense_tensor=dense_tensor, sequence_length=sequence_length) + + def _get_sequence_dense_tensor(self, + inputs, + weight_collections=None, + trainable=None): return _raise_shared_embedding_column_error() @property @@ -3772,16 +3940,20 @@ def _check_shape(shape, key): shape = tuple(shape) for dimension in shape: if not isinstance(dimension, int): - raise TypeError('shape dimensions must be integer. ' 'shape: {}, key: {}'.format(shape, key)) + raise TypeError('shape dimensions must be integer. ' + 'shape: {}, key: {}'.format(shape, key)) if dimension < 1: - raise ValueError('shape dimensions must be greater than 0. ' 'shape: {}, key: {}'.format(shape, key)) + raise ValueError('shape dimensions must be greater than 0. ' + 'shape: {}, key: {}'.format(shape, key)) return shape class HashedCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, - collections.namedtuple('HashedCategoricalColumn', ('feature_name', 'key', 'hash_bucket_size', 'dtype')), + CategoricalColumn, + fc_old._CategoricalColumn, + collections.namedtuple( + 'HashedCategoricalColumn', + ('feature_name', 'key', 'hash_bucket_size', 'dtype')), ): """See `categorical_column_with_hash_bucket`.""" @@ -3805,40 +3977,49 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec def _transform_input_tensor(self, input_tensor): """Hashes the values in the feature_column.""" - fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) + fc_utils.assert_string_or_int( + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key)) if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) - ) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype)) if input_tensor.dtype == dtypes.string: sparse_values = input_tensor.values else: sparse_values = string_ops.as_string(input_tensor.values) - sparse_id_values = string_ops.string_to_hash_bucket_fast(sparse_values, self.hash_bucket_size, name='lookup') + sparse_id_values = string_ops.string_to_hash_bucket_fast( + sparse_values, self.hash_bucket_size, name='lookup') if 'RaggedTensor' in str(type(input_tensor)): from tensorflow.python.ops.ragged import ragged_tensor - return ragged_tensor.RaggedTensor.from_row_splits(values=sparse_id_values, row_splits=input_tensor.row_splits) + return ragged_tensor.RaggedTensor.from_row_splits( + values=sparse_id_values, row_splits=input_tensor.row_splits) - return sparse_tensor_lib.SparseTensor(input_tensor.indices, sparse_id_values, input_tensor.dense_shape) + return sparse_tensor_lib.SparseTensor(input_tensor.indices, + sparse_id_values, + input_tensor.dense_shape) def transform_feature(self, transformation_cache, state_manager): """Hashes the values in the feature_column.""" - input_tensor = _to_sparse_input_and_drop_ignore_values(transformation_cache.get(self.key, state_manager)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + transformation_cache.get(self.key, state_manager)) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) return self._transform_input_tensor(input_tensor) @@ -3849,16 +4030,22 @@ def num_buckets(self): return self.hash_bucket_size @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" - return CategoricalColumn.IdWeightPair(transformation_cache.get(self, state_manager), None) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + return CategoricalColumn.IdWeightPair( + transformation_cache.get(self, state_manager), None) + + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -3884,20 +4071,20 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class VocabularyFileCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, - collections.namedtuple( - 'VocabularyFileCategoricalColumn', - ( - 'feature_name', - 'key', - 'vocabulary_file', - 'vocabulary_size', - 'num_oov_buckets', - 'dtype', - 'default_value', + CategoricalColumn, + fc_old._CategoricalColumn, + collections.namedtuple( + 'VocabularyFileCategoricalColumn', + ( + 'feature_name', + 'key', + 'vocabulary_file', + 'vocabulary_size', + 'num_oov_buckets', + 'dtype', + 'default_value', + ), ), - ), ): """See `categorical_column_with_vocabulary_file`.""" @@ -3921,7 +4108,8 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec @@ -3929,11 +4117,13 @@ def _transform_input_tensor(self, input_tensor): """Creates a lookup table for the vocabulary.""" if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) - ) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype)) - fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) + fc_utils.assert_string_or_int( + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key)) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -3943,20 +4133,22 @@ def _transform_input_tensor(self, input_tensor): # TODO(rohanj): Use state manager to manage the index table creation. return lookup_ops.index_table_from_file( - vocabulary_file=self.vocabulary_file, - num_oov_buckets=self.num_oov_buckets, - vocab_size=self.vocabulary_size, - default_value=self.default_value, - key_dtype=key_dtype, - name='{}_lookup'.format(self.key), + vocabulary_file=self.vocabulary_file, + num_oov_buckets=self.num_oov_buckets, + vocab_size=self.vocabulary_size, + default_value=self.default_value, + key_dtype=key_dtype, + name='{}_lookup'.format(self.key), ).lookup(input_tensor) def transform_feature(self, transformation_cache, state_manager): """Creates a lookup table for the vocabulary.""" - input_tensor = _to_sparse_input_and_drop_ignore_values(transformation_cache.get(self.key, state_manager)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + transformation_cache.get(self.key, state_manager)) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) return self._transform_input_tensor(input_tensor) @@ -3967,16 +4159,22 @@ def num_buckets(self): return self.vocabulary_size + self.num_oov_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" - return CategoricalColumn.IdWeightPair(transformation_cache.get(self, state_manager), None) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + return CategoricalColumn.IdWeightPair( + transformation_cache.get(self, state_manager), None) + + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -4002,19 +4200,19 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class VocabularyListCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, - collections.namedtuple( - 'VocabularyListCategoricalColumn', - ( - 'feature_name', - 'key', - 'vocabulary_list', - 'dtype', - 'default_value', - 'num_oov_buckets', + CategoricalColumn, + fc_old._CategoricalColumn, + collections.namedtuple( + 'VocabularyListCategoricalColumn', + ( + 'feature_name', + 'key', + 'vocabulary_list', + 'dtype', + 'default_value', + 'num_oov_buckets', + ), ), - ), ): """See `categorical_column_with_vocabulary_list`.""" @@ -4038,7 +4236,8 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(self.dtype)} @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec @@ -4046,11 +4245,13 @@ def _transform_input_tensor(self, input_tensor): """Creates a lookup table for the vocabulary list.""" if self.dtype.is_integer != input_tensor.dtype.is_integer: raise ValueError( - 'Column dtype and SparseTensors dtype must be compatible. ' - 'key: {}, column dtype: {}, tensor dtype: {}'.format(self.key, self.dtype, input_tensor.dtype) - ) + 'Column dtype and SparseTensors dtype must be compatible. ' + 'key: {}, column dtype: {}, tensor dtype: {}'.format( + self.key, self.dtype, input_tensor.dtype)) - fc_utils.assert_string_or_int(input_tensor.dtype, prefix='column_name: {} input_tensor'.format(self.key)) + fc_utils.assert_string_or_int( + input_tensor.dtype, + prefix='column_name: {} input_tensor'.format(self.key)) key_dtype = self.dtype if input_tensor.dtype.is_integer: @@ -4060,19 +4261,21 @@ def _transform_input_tensor(self, input_tensor): # TODO(rohanj): Use state manager to manage the index table creation. return lookup_ops.index_table_from_tensor( - vocabulary_list=tuple(self.vocabulary_list), - default_value=self.default_value, - num_oov_buckets=self.num_oov_buckets, - dtype=key_dtype, - name='{}_lookup'.format(self.key), + vocabulary_list=tuple(self.vocabulary_list), + default_value=self.default_value, + num_oov_buckets=self.num_oov_buckets, + dtype=key_dtype, + name='{}_lookup'.format(self.key), ).lookup(input_tensor) def transform_feature(self, transformation_cache, state_manager): """Creates a lookup table for the vocabulary list.""" - input_tensor = _to_sparse_input_and_drop_ignore_values(transformation_cache.get(self.key, state_manager)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + transformation_cache.get(self.key, state_manager)) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) return self._transform_input_tensor(input_tensor) @@ -4083,16 +4286,22 @@ def num_buckets(self): return len(self.vocabulary_list) + self.num_oov_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" - return CategoricalColumn.IdWeightPair(transformation_cache.get(self, state_manager), None) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + return CategoricalColumn.IdWeightPair( + transformation_cache.get(self, state_manager), None) + + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -4118,12 +4327,12 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class IdentityCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, - collections.namedtuple( - 'IdentityCategoricalColumn', - ('feature_name', 'key', 'number_buckets', 'default_value'), - ), + CategoricalColumn, + fc_old._CategoricalColumn, + collections.namedtuple( + 'IdentityCategoricalColumn', + ('feature_name', 'key', 'number_buckets', 'default_value'), + ), ): """See `categorical_column_with_identity`.""" @@ -4147,57 +4356,64 @@ def parse_example_spec(self): return {self.key: parsing_ops.VarLenFeature(dtypes.int64)} @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec def _transform_input_tensor(self, input_tensor): """Returns a SparseTensor with identity values.""" if not input_tensor.dtype.is_integer: - raise ValueError('Invalid input, not integer. key: {} dtype: {}'.format(self.key, input_tensor.dtype)) + raise ValueError('Invalid input, not integer. key: {} dtype: {}'.format( + self.key, input_tensor.dtype)) if 'RaggedTensor' in str(type(input_tensor)): return input_tensor values = math_ops.cast(input_tensor.values, dtypes.int64, name='values') if self.num_buckets < sys.maxsize: - num_buckets = math_ops.cast(self.num_buckets, dtypes.int64, name='num_buckets') + num_buckets = math_ops.cast( + self.num_buckets, dtypes.int64, name='num_buckets') zero = math_ops.cast(0, dtypes.int64, name='zero') if self.default_value is None: # Fail if values are out-of-range. assert_less = check_ops.assert_less( - values, - num_buckets, - data=(values, num_buckets), - name='assert_less_than_num_buckets', + values, + num_buckets, + data=(values, num_buckets), + name='assert_less_than_num_buckets', ) - assert_greater = check_ops.assert_greater_equal(values, zero, data=(values,), name='assert_greater_or_equal_0') + assert_greater = check_ops.assert_greater_equal( + values, zero, data=(values,), name='assert_greater_or_equal_0') with ops.control_dependencies((assert_less, assert_greater)): values = array_ops.identity(values) else: # Assign default for out-of-range values. values = array_ops.where( - math_ops.logical_or(values < zero, values >= num_buckets, name='out_of_range'), - array_ops.fill( - dims=array_ops.shape(values), - value=math_ops.cast(self.default_value, dtypes.int64), - name='default_values', - ), - values, + math_ops.logical_or( + values < zero, values >= num_buckets, name='out_of_range'), + array_ops.fill( + dims=array_ops.shape(values), + value=math_ops.cast(self.default_value, dtypes.int64), + name='default_values', + ), + values, ) return sparse_tensor_lib.SparseTensor( - indices=input_tensor.indices, - values=values, - dense_shape=input_tensor.dense_shape, + indices=input_tensor.indices, + values=values, + dense_shape=input_tensor.dense_shape, ) def transform_feature(self, transformation_cache, state_manager): """Returns a SparseTensor with identity values.""" - input_tensor = _to_sparse_input_and_drop_ignore_values(transformation_cache.get(self.key, state_manager)) + input_tensor = _to_sparse_input_and_drop_ignore_values( + transformation_cache.get(self.key, state_manager)) return self._transform_input_tensor(input_tensor) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): input_tensor = _to_sparse_input_and_drop_ignore_values(inputs.get(self.key)) return self._transform_input_tensor(input_tensor) @@ -4208,16 +4424,22 @@ def num_buckets(self): return self.number_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" - return CategoricalColumn.IdWeightPair(transformation_cache.get(self, state_manager), None) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + return CategoricalColumn.IdWeightPair( + transformation_cache.get(self, state_manager), None) + + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): del weight_collections del trainable return CategoricalColumn.IdWeightPair(inputs.get(self), None) @@ -4239,23 +4461,25 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class WeightedCategoricalColumn( - CategoricalColumn, - fc_old._CategoricalColumn, - collections.namedtuple( - 'WeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype'), - ), + CategoricalColumn, + fc_old._CategoricalColumn, + collections.namedtuple( + 'WeightedCategoricalColumn', + ('categorical_column', 'weight_feature_key', 'dtype'), + ), ): """See `weighted_categorical_column`.""" @property def _is_v2_column(self): - return isinstance(self.categorical_column, FeatureColumn) and self.categorical_column._is_v2_column + return isinstance(self.categorical_column, + FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): """See `FeatureColumn` base class.""" - return '{}_weighted_by_{}'.format(self.categorical_column.name, self.weight_feature_key) + return '{}_weighted_by_{}'.format(self.categorical_column.name, + self.weight_feature_key) @property def raw_name(self): @@ -4267,20 +4491,19 @@ def parse_example_spec(self): """See `FeatureColumn` base class.""" config = self.categorical_column.parse_example_spec if self.weight_feature_key in config: - raise ValueError( - 'Parse config {} already exists for {}.'.format(config[self.weight_feature_key], self.weight_feature_key) - ) + raise ValueError('Parse config {} already exists for {}.'.format( + config[self.weight_feature_key], self.weight_feature_key)) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): config = self.categorical_column._parse_example_spec if self.weight_feature_key in config: - raise ValueError( - 'Parse config {} already exists for {}.'.format(config[self.weight_feature_key], self.weight_feature_key) - ) + raise ValueError('Parse config {} already exists for {}.'.format( + config[self.weight_feature_key], self.weight_feature_key)) config[self.weight_feature_key] = parsing_ops.VarLenFeature(self.dtype) return config @@ -4290,33 +4513,39 @@ def num_buckets(self): return self.categorical_column.num_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.categorical_column._num_buckets def _transform_weight_tensor(self, weight_tensor): if weight_tensor is None: raise ValueError('Missing weights {}.'.format(self.weight_feature_key)) - weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(weight_tensor) + weight_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor( + weight_tensor) if self.dtype != weight_tensor.dtype.base_dtype: - raise ValueError('Bad dtype, expected {}, but got {}.'.format(self.dtype, weight_tensor.dtype)) + raise ValueError('Bad dtype, expected {}, but got {}.'.format( + self.dtype, weight_tensor.dtype)) if not isinstance(weight_tensor, sparse_tensor_lib.SparseTensor): # The weight tensor can be a regular Tensor. In this case, sparsify it. - weight_tensor = _to_sparse_input_and_drop_ignore_values(weight_tensor, ignore_value=0.0) + weight_tensor = _to_sparse_input_and_drop_ignore_values( + weight_tensor, ignore_value=0.0) if not weight_tensor.dtype.is_floating: weight_tensor = math_ops.cast(weight_tensor, dtypes.float32) return weight_tensor def transform_feature(self, transformation_cache, state_manager): """Applies weights to tensor generated from `categorical_column`'.""" - weight_tensor = transformation_cache.get(self.weight_feature_key, state_manager) + weight_tensor = transformation_cache.get(self.weight_feature_key, + state_manager) weight_tensor = self._transform_weight_tensor(weight_tensor) return ( - transformation_cache.get(self.categorical_column, state_manager), - weight_tensor, + transformation_cache.get(self.categorical_column, state_manager), + weight_tensor, ) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Applies weights to tensor generated from `categorical_column`'.""" weight_tensor = inputs.get(self.weight_feature_key) @@ -4328,8 +4557,12 @@ def get_sparse_tensors(self, transformation_cache, state_manager): tensors = transformation_cache.get(self, state_manager) return CategoricalColumn.IdWeightPair(tensors[0], tensors[1]) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): del weight_collections del trainable tensors = inputs.get(self) @@ -4343,7 +4576,8 @@ def parents(self): def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) - config['categorical_column'] = serialize_feature_column(self.categorical_column) + config['categorical_column'] = serialize_feature_column( + self.categorical_column) config['dtype'] = self.dtype.name return config @@ -4353,16 +4587,17 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['categorical_column'] = deserialize_feature_column( - config['categorical_column'], custom_objects, columns_by_name - ) + config['categorical_column'], custom_objects, columns_by_name) kwargs['dtype'] = dtypes.as_dtype(config['dtype']) return cls(**kwargs) class CrossedColumn( - CategoricalColumn, - fc_old._CategoricalColumn, - collections.namedtuple('CrossedColumn', ('feature_name', 'keys', 'hash_bucket_size', 'hash_key')), + CategoricalColumn, + fc_old._CategoricalColumn, + collections.namedtuple( + 'CrossedColumn', + ('feature_name', 'keys', 'hash_bucket_size', 'hash_key')), ): """See `crossed_column`.""" @@ -4404,7 +4639,8 @@ def parse_example_spec(self): return config @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.parse_example_spec @@ -4415,23 +4651,24 @@ def transform_feature(self, transformation_cache, state_manager): if isinstance(key, six.string_types): feature_tensors.append(transformation_cache.get(key, state_manager)) elif isinstance(key, (fc_old._CategoricalColumn, CategoricalColumn)): - ids_and_weights = key.get_sparse_tensors(transformation_cache, state_manager) + ids_and_weights = key.get_sparse_tensors(transformation_cache, + state_manager) if ids_and_weights.weight_tensor is not None: raise ValueError( - 'crossed_column does not support weight_tensor, but the given ' - 'column populates weight_tensor. ' - 'Given column: {}'.format(key.name) - ) + 'crossed_column does not support weight_tensor, but the given ' + 'column populates weight_tensor. ' + 'Given column: {}'.format(key.name)) feature_tensors.append(ids_and_weights.id_tensor) else: raise ValueError('Unsupported column type. Given: {}'.format(key)) return sparse_ops.sparse_cross_hashed( - inputs=feature_tensors, - num_buckets=self.hash_bucket_size, - hash_key=self.hash_key, + inputs=feature_tensors, + num_buckets=self.hash_bucket_size, + hash_key=self.hash_key, ) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Generates a hashed sparse cross from the input tensors.""" feature_tensors = [] @@ -4442,17 +4679,16 @@ def _transform_feature(self, inputs): ids_and_weights = key._get_sparse_tensors(inputs) if ids_and_weights.weight_tensor is not None: raise ValueError( - 'crossed_column does not support weight_tensor, but the given ' - 'column populates weight_tensor. ' - 'Given column: {}'.format(key.name) - ) + 'crossed_column does not support weight_tensor, but the given ' + 'column populates weight_tensor. ' + 'Given column: {}'.format(key.name)) feature_tensors.append(ids_and_weights.id_tensor) else: raise ValueError('Unsupported column type. Given: {}'.format(key)) return sparse_ops.sparse_cross_hashed( - inputs=feature_tensors, - num_buckets=self.hash_bucket_size, - hash_key=self.hash_key, + inputs=feature_tensors, + num_buckets=self.hash_bucket_size, + hash_key=self.hash_key, ) @property @@ -4461,16 +4697,22 @@ def num_buckets(self): return self.hash_bucket_size @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.num_buckets def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" - return CategoricalColumn.IdWeightPair(transformation_cache.get(self, state_manager), None) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + return CategoricalColumn.IdWeightPair( + transformation_cache.get(self, state_manager), None) + + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): """See `CategoricalColumn` base class.""" del weight_collections del trainable @@ -4492,7 +4734,10 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): """See 'FeatureColumn` base class.""" _check_config_keys(config, cls._fields) kwargs = config.copy() - kwargs['keys'] = tuple([deserialize_feature_column(c, custom_objects, columns_by_name) for c in config['keys']]) + kwargs['keys'] = tuple([ + deserialize_feature_column(c, custom_objects, columns_by_name) + for c in config['keys'] + ]) return cls(**kwargs) @@ -4518,7 +4763,9 @@ def _prune_invalid_ids(sparse_ids, sparse_weights): """Prune invalid IDs (< 0) from the input ids and weights.""" is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) if sparse_weights is not None: - is_id_valid = math_ops.logical_and(is_id_valid, array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) + is_id_valid = math_ops.logical_and( + is_id_valid, + array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) if sparse_weights is not None: sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) @@ -4535,11 +4782,11 @@ def _prune_invalid_weights(sparse_ids, sparse_weights): class IndicatorColumn( - DenseColumn, - SequenceDenseColumn, - fc_old._DenseColumn, - fc_old._SequenceDenseColumn, - collections.namedtuple('IndicatorColumn', ('categorical_column')), + DenseColumn, + SequenceDenseColumn, + fc_old._DenseColumn, + fc_old._SequenceDenseColumn, + collections.namedtuple('IndicatorColumn', ('categorical_column')), ): """Represents a one-hot column for use in deep networks. @@ -4550,7 +4797,8 @@ class IndicatorColumn( @property def _is_v2_column(self): - return isinstance(self.categorical_column, FeatureColumn) and self.categorical_column._is_v2_column + return isinstance(self.categorical_column, + FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): @@ -4569,25 +4817,31 @@ def _transform_id_weight_pair(self, id_weight_pair): # If the underlying column is weighted, return the input as a dense tensor. if weight_tensor is not None: weighted_column = sparse_ops.sparse_merge( - sp_ids=id_tensor, - sp_values=weight_tensor, - vocab_size=int(self._variable_shape[-1]), + sp_ids=id_tensor, + sp_values=weight_tensor, + vocab_size=int(self._variable_shape[-1]), ) # Remove (?, -1) index. - weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0], weighted_column.dense_shape) + weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0], + weighted_column.dense_shape) # Use scatter_nd to merge duplicated indices if existed, # instead of sparse_tensor_to_dense. return array_ops.scatter_nd( - weighted_column.indices, - weighted_column.values, - weighted_column.dense_shape, + weighted_column.indices, + weighted_column.values, + weighted_column.dense_shape, ) - dense_id_tensor = sparse_ops.sparse_tensor_to_dense(id_tensor, default_value=-1) + dense_id_tensor = sparse_ops.sparse_tensor_to_dense( + id_tensor, default_value=-1) # One hot must be float for tf.concat reasons since all other inputs to # input_layer are float32. - one_hot_id_tensor = array_ops.one_hot(dense_id_tensor, depth=self._variable_shape[-1], on_value=1.0, off_value=0.0) + one_hot_id_tensor = array_ops.one_hot( + dense_id_tensor, + depth=self._variable_shape[-1], + on_value=1.0, + off_value=0.0) # Reduce to get a multi-hot per example. return math_ops.reduce_sum(one_hot_id_tensor, axis=[-2]) @@ -4607,10 +4861,12 @@ def transform_feature(self, transformation_cache, state_manager): Raises: ValueError: if input rank is not known at graph building time. """ - id_weight_pair = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) + id_weight_pair = self.categorical_column.get_sparse_tensors( + transformation_cache, state_manager) return self._transform_id_weight_pair(id_weight_pair) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): id_weight_pair = self.categorical_column._get_sparse_tensors(inputs) return self._transform_id_weight_pair(id_weight_pair) @@ -4621,7 +4877,8 @@ def parse_example_spec(self): return self.categorical_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.categorical_column._parse_example_spec @@ -4634,7 +4891,8 @@ def variable_shape(self): return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) @@ -4655,35 +4913,36 @@ def get_dense_tensor(self, transformation_cache, state_manager): """ if isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must not be of type SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use DenseFeatures, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) + 'In indicator_column: {}. ' + 'categorical_column must not be of type SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use DenseFeatures, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'SequenceFeatures instead of DenseFeatures. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) # Feature has been already transformed. Return the intermediate # representation created by transform_feature. return transformation_cache.get(self, state_manager) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del weight_collections del trainable if isinstance( - self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), + self.categorical_column, + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), ): raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must not be of type _SequenceCategoricalColumn. ' - 'Suggested fix A: If you wish to use DenseFeatures, use a ' - 'non-sequence categorical_column_with_*. ' - 'Suggested fix B: If you wish to create sequence input, use ' - 'SequenceFeatures instead of DenseFeatures. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) + 'In indicator_column: {}. ' + 'categorical_column must not be of type _SequenceCategoricalColumn. ' + 'Suggested fix A: If you wish to use DenseFeatures, use a ' + 'non-sequence categorical_column_with_*. ' + 'Suggested fix B: If you wish to create sequence input, use ' + 'SequenceFeatures instead of DenseFeatures. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) # Feature has been already transformed. Return the intermediate # representation created by transform_feature. return inputs.get(self) @@ -4692,42 +4951,51 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): """See `SequenceDenseColumn` base class.""" if not isinstance(self.categorical_column, SequenceCategoricalColumn): raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must be of type SequenceCategoricalColumn ' - 'to use SequenceFeatures. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) + 'In indicator_column: {}. ' + 'categorical_column must be of type SequenceCategoricalColumn ' + 'to use SequenceFeatures. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) # Feature has been already transformed. Return the intermediate # representation created by transform_feature. dense_tensor = transformation_cache.get(self, state_manager) - sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) - sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) - return SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) - - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sequence_dense_tensor(self, inputs, weight_collections=None, trainable=None): + sparse_tensors = self.categorical_column.get_sparse_tensors( + transformation_cache, state_manager) + sequence_length = fc_utils.sequence_length_from_sparse_tensor( + sparse_tensors.id_tensor) + return SequenceDenseColumn.TensorSequenceLengthPair( + dense_tensor=dense_tensor, sequence_length=sequence_length) + + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sequence_dense_tensor(self, + inputs, + weight_collections=None, + trainable=None): # Do nothing with weight_collections and trainable since no variables are # created in this function. del weight_collections del trainable if not isinstance( - self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), + self.categorical_column, + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), ): raise ValueError( - 'In indicator_column: {}. ' - 'categorical_column must be of type _SequenceCategoricalColumn ' - 'to use SequenceFeatures. ' - 'Suggested fix: Use one of sequence_categorical_column_with_*. ' - 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column) - ) + 'In indicator_column: {}. ' + 'categorical_column must be of type _SequenceCategoricalColumn ' + 'to use SequenceFeatures. ' + 'Suggested fix: Use one of sequence_categorical_column_with_*. ' + 'Given (type {}): {}'.format(self.name, type(self.categorical_column), + self.categorical_column)) # Feature has been already transformed. Return the intermediate # representation created by _transform_feature. dense_tensor = inputs.get(self) sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) - sequence_length = fc_utils.sequence_length_from_sparse_tensor(sparse_tensors.id_tensor) - return SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=sequence_length) + sequence_length = fc_utils.sequence_length_from_sparse_tensor( + sparse_tensors.id_tensor) + return SequenceDenseColumn.TensorSequenceLengthPair( + dense_tensor=dense_tensor, sequence_length=sequence_length) @property def parents(self): @@ -4737,7 +5005,8 @@ def parents(self): def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) - config['categorical_column'] = serialize_feature_column(self.categorical_column) + config['categorical_column'] = serialize_feature_column( + self.categorical_column) return config @classmethod @@ -4746,8 +5015,7 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['categorical_column'] = deserialize_feature_column( - config['categorical_column'], custom_objects, columns_by_name - ) + config['categorical_column'], custom_objects, columns_by_name) return cls(**kwargs) @@ -4764,33 +5032,34 @@ def _verify_static_batch_size_equality(tensors, columns): # bath_size is a tf.compat.v1.Dimension object. expected_batch_size = None for i in range(0, len(tensors)): - batch_size = tensor_shape.Dimension(tensor_shape.dimension_value(tensors[i].shape[0])) + batch_size = tensor_shape.Dimension( + tensor_shape.dimension_value(tensors[i].shape[0])) if batch_size.value is not None: if expected_batch_size is None: bath_size_column_index = i expected_batch_size = batch_size elif not expected_batch_size.is_compatible_with(batch_size): raise ValueError( - 'Batch size (first dimension) of each feature must be same. ' - 'Batch size of columns ({}, {}): ({}, {})'.format( - columns[bath_size_column_index].name, - columns[i].name, - expected_batch_size, - batch_size, - ) - ) + 'Batch size (first dimension) of each feature must be same. ' + 'Batch size of columns ({}, {}): ({}, {})'.format( + columns[bath_size_column_index].name, + columns[i].name, + expected_batch_size, + batch_size, + )) class SequenceCategoricalColumn( - CategoricalColumn, - fc_old._SequenceCategoricalColumn, - collections.namedtuple('SequenceCategoricalColumn', ('categorical_column')), + CategoricalColumn, + fc_old._SequenceCategoricalColumn, + collections.namedtuple('SequenceCategoricalColumn', ('categorical_column')), ): """Represents sequences of categorical data.""" @property def _is_v2_column(self): - return isinstance(self.categorical_column, FeatureColumn) and self.categorical_column._is_v2_column + return isinstance(self.categorical_column, + FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): @@ -4808,19 +5077,22 @@ def parse_example_spec(self): return self.categorical_column.parse_example_spec @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): return self.categorical_column._parse_example_spec def transform_feature(self, transformation_cache, state_manager): """See `FeatureColumn` base class.""" - ret_tensor = self.categorical_column.transform_feature(transformation_cache, state_manager) + ret_tensor = self.categorical_column.transform_feature( + transformation_cache, state_manager) shape = array_ops.shape(ret_tensor) target_shape = [shape[0], shape[1], math_ops.reduce_prod(shape[2:])] ret_tensor = sparse_ops.sparse_reshape(ret_tensor, target_shape) return ret_tensor - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): ret_tensor = self.categorical_column._transform_feature(inputs) shape = array_ops.shape(ret_tensor) @@ -4834,7 +5106,8 @@ def num_buckets(self): return self.categorical_column.num_buckets @property - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): return self.categorical_column._num_buckets @@ -4873,11 +5146,16 @@ def get_sparse_tensors(self, transformation_cache, state_manager): state_manager: A `StateManager` to create / access resources such as lookup tables. """ - sparse_tensors = self.categorical_column.get_sparse_tensors(transformation_cache, state_manager) + sparse_tensors = self.categorical_column.get_sparse_tensors( + transformation_cache, state_manager) return self._get_sparse_tensors_helper(sparse_tensors) - @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) - def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): + @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, + _FEATURE_COLUMN_DEPRECATION) + def _get_sparse_tensors(self, + inputs, + weight_collections=None, + trainable=None): sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) return self._get_sparse_tensors_helper(sparse_tensors) @@ -4889,7 +5167,8 @@ def parents(self): def _get_config(self): """See 'FeatureColumn` base class.""" config = dict(zip(self._fields, self)) - config['categorical_column'] = serialize_feature_column(self.categorical_column) + config['categorical_column'] = serialize_feature_column( + self.categorical_column) return config @classmethod @@ -4898,8 +5177,7 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): _check_config_keys(config, cls._fields) kwargs = config.copy() kwargs['categorical_column'] = deserialize_feature_column( - config['categorical_column'], custom_objects, columns_by_name - ) + config['categorical_column'], custom_objects, columns_by_name) return cls(**kwargs) @@ -4909,7 +5187,8 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): def _check_config_keys(config, expected_keys): """Checks that a config has all expected_keys.""" if set(config.keys()) != set(expected_keys): - raise ValueError('Invalid config: {}, expected keys: {}'.format(config, expected_keys)) + raise ValueError('Invalid config: {}, expected keys: {}'.format( + config, expected_keys)) def serialize_feature_column(fc): @@ -4955,12 +5234,15 @@ def serialize_feature_column(fc): if isinstance(fc, six.string_types): return fc elif isinstance(fc, FeatureColumn): - return utils.serialize_keras_class_and_config(fc.__class__.__name__, fc._get_config()) + return utils.serialize_keras_class_and_config(fc.__class__.__name__, + fc._get_config()) else: raise ValueError('Instance: {} is not a FeatureColumn'.format(fc)) -def deserialize_feature_column(config, custom_objects=None, columns_by_name=None): +def deserialize_feature_column(config, + custom_objects=None, + columns_by_name=None): """Deserializes a `config` generated with `serialize_feature_column`. This method should only be used to deserialize parent FeatureColumns when @@ -4988,38 +5270,41 @@ def deserialize_feature_column(config, custom_objects=None, columns_by_name=None # A dict from class_name to class for all FeatureColumns in this module. # FeatureColumns not part of the module can be passed as custom_objects. module_feature_column_classes = { - cls.__name__: cls - for cls in [ - BucketizedColumn, - EmbeddingColumn, - HashedCategoricalColumn, - IdentityCategoricalColumn, - IndicatorColumn, - NumericColumn, - SequenceCategoricalColumn, - SequenceDenseColumn, - SharedEmbeddingColumn, - VocabularyFileCategoricalColumn, - VocabularyListCategoricalColumn, - WeightedCategoricalColumn, - init_ops.TruncatedNormal, - ] + cls.__name__: cls for cls in [ + BucketizedColumn, + EmbeddingColumn, + HashedCategoricalColumn, + IdentityCategoricalColumn, + IndicatorColumn, + NumericColumn, + SequenceCategoricalColumn, + SequenceDenseColumn, + SharedEmbeddingColumn, + VocabularyFileCategoricalColumn, + VocabularyListCategoricalColumn, + WeightedCategoricalColumn, + init_ops.TruncatedNormal, + ] } if columns_by_name is None: columns_by_name = {} (cls, cls_config) = utils.class_and_config_for_serialized_keras_object( - config, - module_objects=module_feature_column_classes, - custom_objects=custom_objects, - printable_module_name='feature_column_v2', + config, + module_objects=module_feature_column_classes, + custom_objects=custom_objects, + printable_module_name='feature_column_v2', ) if not issubclass(cls, FeatureColumn): - raise ValueError('Expected FeatureColumn class, instead found: {}'.format(cls)) + raise ValueError( + 'Expected FeatureColumn class, instead found: {}'.format(cls)) # Always deserialize the FeatureColumn, in order to get the name. - new_instance = cls._from_config(cls_config, custom_objects=custom_objects, columns_by_name=columns_by_name) + new_instance = cls._from_config( + cls_config, + custom_objects=custom_objects, + columns_by_name=columns_by_name) # If the name already exists, re-use the column from columns_by_name, # (new_instance remains unused). @@ -5064,7 +5349,10 @@ def deserialize_feature_columns(configs, custom_objects=None): ValueError if called with input that is not a list of FeatureColumns. """ columns_by_name = {} - return [deserialize_feature_column(c, custom_objects, columns_by_name) for c in configs] + return [ + deserialize_feature_column(c, custom_objects, columns_by_name) + for c in configs + ] def is_embedding_column(fc): diff --git a/easy_rec/python/compat/feature_column/sequence_feature_column.py b/easy_rec/python/compat/feature_column/sequence_feature_column.py index 862f84068..2ac498d81 100644 --- a/easy_rec/python/compat/feature_column/sequence_feature_column.py +++ b/easy_rec/python/compat/feature_column/sequence_feature_column.py @@ -18,23 +18,24 @@ NOTE: This API is a work in progress and will likely be changing frequently. """ -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import collections -from tensorflow.python.framework import dtypes, ops, tensor_shape -from tensorflow.python.ops import ( # NOQA - array_ops, - check_ops, - math_ops, - parsing_ops, - sparse_ops, -) +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from easy_rec.python.compat.feature_column import feature_column as fc_v1 from easy_rec.python.compat.feature_column import feature_column_v2 as fc from easy_rec.python.compat.feature_column import utils as fc_utils +from tensorflow.python.ops import ( # NOQA + array_ops, check_ops, math_ops, parsing_ops, sparse_ops, +) + # pylint: disable=protected-access @@ -90,11 +91,11 @@ def __init__(self, feature_columns, trainable=True, name=None, **kwargs): `SequenceDenseColumn`. """ super(SequenceFeatures, self).__init__( - feature_columns=feature_columns, - trainable=trainable, - name=name, - expected_column_type=fc.SequenceDenseColumn, - **kwargs, + feature_columns=feature_columns, + trainable=trainable, + name=name, + expected_column_type=fc.SequenceDenseColumn, + **kwargs, ) def _target_shape(self, input_shape, total_elements): @@ -119,20 +120,23 @@ def call(self, features): ValueError: If features are not a dictionary. """ if not isinstance(features, dict): - raise ValueError('We expected a dictionary here. Instead we got: ', features) + raise ValueError('We expected a dictionary here. Instead we got: ', + features) transformation_cache = fc.FeatureTransformationCache(features) output_tensors = [] sequence_lengths = [] for column in self._feature_columns: with ops.name_scope(column.name): - dense_tensor, sequence_length = column.get_sequence_dense_tensor(transformation_cache, self._state_manager) + dense_tensor, sequence_length = column.get_sequence_dense_tensor( + transformation_cache, self._state_manager) # Flattens the final dimension to produce a 3D Tensor. output_tensors.append(self._process_dense_tensor(column, dense_tensor)) sequence_lengths.append(sequence_length) # Check and process sequence lengths. - fc._verify_static_batch_size_equality(sequence_lengths, self._feature_columns) + fc._verify_static_batch_size_equality(sequence_lengths, + self._feature_columns) sequence_length = _assert_all_equal_and_return(sequence_lengths) return self._verify_and_concat_tensors(output_tensors), sequence_length @@ -159,37 +163,43 @@ def concatenate_context_input(context_input, sequence_input): not have rank 2. """ seq_rank_check = check_ops.assert_rank( - sequence_input, - 3, - message='sequence_input must have rank 3', - data=[array_ops.shape(sequence_input)], + sequence_input, + 3, + message='sequence_input must have rank 3', + data=[array_ops.shape(sequence_input)], ) seq_type_check = check_ops.assert_type( - sequence_input, - dtypes.float32, - message='sequence_input must have dtype float32; got {}.'.format(sequence_input.dtype), + sequence_input, + dtypes.float32, + message='sequence_input must have dtype float32; got {}.'.format( + sequence_input.dtype), ) ctx_rank_check = check_ops.assert_rank( - context_input, - 2, - message='context_input must have rank 2', - data=[array_ops.shape(context_input)], + context_input, + 2, + message='context_input must have rank 2', + data=[array_ops.shape(context_input)], ) ctx_type_check = check_ops.assert_type( - context_input, - dtypes.float32, - message='context_input must have dtype float32; got {}.'.format(context_input.dtype), + context_input, + dtypes.float32, + message='context_input must have dtype float32; got {}.'.format( + context_input.dtype), ) - with ops.control_dependencies([seq_rank_check, seq_type_check, ctx_rank_check, ctx_type_check]): + with ops.control_dependencies( + [seq_rank_check, seq_type_check, ctx_rank_check, ctx_type_check]): padded_length = array_ops.shape(sequence_input)[1] tiled_context_input = array_ops.tile( - array_ops.expand_dims(context_input, 1), - array_ops.concat([[1], [padded_length], [1]], 0), + array_ops.expand_dims(context_input, 1), + array_ops.concat([[1], [padded_length], [1]], 0), ) return array_ops.concat([sequence_input, tiled_context_input], 2) -def sequence_categorical_column_with_identity(key, num_buckets, default_value=None, feature_name=None): +def sequence_categorical_column_with_identity(key, + num_buckets, + default_value=None, + feature_name=None): """Returns a feature column that represents sequences of integers. Pass this to `embedding_column` or `indicator_column` to convert sequence @@ -230,22 +240,22 @@ def sequence_categorical_column_with_identity(key, num_buckets, default_value=No ValueError: if `default_value` is not in range `[0, num_buckets)`. """ return fc.SequenceCategoricalColumn( - fc.categorical_column_with_identity( - feature_name=feature_name, - key=key, - num_buckets=num_buckets, - default_value=default_value, - ) - ) + fc.categorical_column_with_identity( + feature_name=feature_name, + key=key, + num_buckets=num_buckets, + default_value=default_value, + )) def sequence_numeric_column_with_bucketized_column(source_column, boundaries): if not isinstance(source_column, (SequenceNumericColumn,)): # pylint: disable=protected-access raise ValueError( - 'source_column must be a column generated with sequence_numeric_column(). ' 'Given: {}'.format(source_column) - ) + 'source_column must be a column generated with sequence_numeric_column(). ' + 'Given: {}'.format(source_column)) if len(source_column.shape) > 1: - raise ValueError('source_column must be one-dimensional column. ' 'Given: {}'.format(source_column)) + raise ValueError('source_column must be one-dimensional column. ' + 'Given: {}'.format(source_column)) if not boundaries: raise ValueError('boundaries must not be empty.') if not (isinstance(boundaries, list) or isinstance(boundaries, tuple)): @@ -259,25 +269,31 @@ def sequence_numeric_column_with_bucketized_column(source_column, boundaries): def sequence_numeric_column_with_raw_column(source_column, sequence_length): if not isinstance(source_column, (SequenceNumericColumn,)): # pylint: disable=protected-access raise ValueError( - 'source_column must be a column generated with sequence_numeric_column(). ' 'Given: {}'.format(source_column) - ) + 'source_column must be a column generated with sequence_numeric_column(). ' + 'Given: {}'.format(source_column)) if len(source_column.shape) > 1: - raise ValueError('source_column must be one-dimensional column. ' 'Given: {}'.format(source_column)) + raise ValueError('source_column must be one-dimensional column. ' + 'Given: {}'.format(source_column)) return fc.SequenceNumericColumn(source_column, sequence_length) -def sequence_weighted_categorical_column(categorical_column, weight_feature_key, dtype=dtypes.float32): +def sequence_weighted_categorical_column(categorical_column, + weight_feature_key, + dtype=dtypes.float32): if (dtype is None) or not (dtype.is_integer or dtype.is_floating): raise ValueError('dtype {} is not convertible to float.'.format(dtype)) return fc.SequenceWeightedCategoricalColumn( - categorical_column=categorical_column, - weight_feature_key=weight_feature_key, - dtype=dtype, + categorical_column=categorical_column, + weight_feature_key=weight_feature_key, + dtype=dtype, ) -def sequence_categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dtypes.string, feature_name=None): +def sequence_categorical_column_with_hash_bucket(key, + hash_bucket_size, + dtype=dtypes.string, + feature_name=None): """A sequence of categorical terms where ids are set by hashing. Pass this to `embedding_column` or `indicator_column` to convert sequence @@ -315,23 +331,22 @@ def sequence_categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dt ValueError: `dtype` is neither string nor integer. """ return fc.SequenceCategoricalColumn( - fc.categorical_column_with_hash_bucket( - feature_name=feature_name, - key=key, - hash_bucket_size=hash_bucket_size, - dtype=dtype, - ) - ) + fc.categorical_column_with_hash_bucket( + feature_name=feature_name, + key=key, + hash_bucket_size=hash_bucket_size, + dtype=dtype, + )) def sequence_categorical_column_with_vocabulary_file( - key, - vocabulary_file, - vocabulary_size=None, - num_oov_buckets=0, - default_value=None, - dtype=dtypes.string, - feature_name=None, + key, + vocabulary_file, + vocabulary_size=None, + num_oov_buckets=0, + default_value=None, + dtype=dtypes.string, + feature_name=None, ): """A sequence of categorical terms where ids use a vocabulary file. @@ -385,25 +400,24 @@ def sequence_categorical_column_with_vocabulary_file( ValueError: `dtype` is neither string nor integer. """ return fc.SequenceCategoricalColumn( - fc.categorical_column_with_vocabulary_file( - feature_name=feature_name, - key=key, - vocabulary_file=vocabulary_file, - vocabulary_size=vocabulary_size, - num_oov_buckets=num_oov_buckets, - default_value=default_value, - dtype=dtype, - ) - ) + fc.categorical_column_with_vocabulary_file( + feature_name=feature_name, + key=key, + vocabulary_file=vocabulary_file, + vocabulary_size=vocabulary_size, + num_oov_buckets=num_oov_buckets, + default_value=default_value, + dtype=dtype, + )) def sequence_categorical_column_with_vocabulary_list( - key, - vocabulary_list, - dtype=None, - default_value=-1, - num_oov_buckets=0, - feature_name=None, + key, + vocabulary_list, + dtype=None, + default_value=-1, + num_oov_buckets=0, + feature_name=None, ): """A sequence of categorical terms where ids use an in-memory list. @@ -456,24 +470,23 @@ def sequence_categorical_column_with_vocabulary_list( ValueError: if `dtype` is not integer or string. """ return fc.SequenceCategoricalColumn( - fc.categorical_column_with_vocabulary_list( - feature_name=feature_name, - key=key, - vocabulary_list=vocabulary_list, - dtype=dtype, - default_value=default_value, - num_oov_buckets=num_oov_buckets, - ) - ) + fc.categorical_column_with_vocabulary_list( + feature_name=feature_name, + key=key, + vocabulary_list=vocabulary_list, + dtype=dtype, + default_value=default_value, + num_oov_buckets=num_oov_buckets, + )) def sequence_numeric_column( - key, - shape=(1,), - default_value=0.0, - dtype=dtypes.float32, - normalizer_fn=None, - feature_name=None, + key, + shape=(1,), + default_value=0.0, + dtype=dtypes.float32, + normalizer_fn=None, + feature_name=None, ): """Returns a feature column that represents sequences of numeric data. @@ -517,17 +530,19 @@ def sequence_numeric_column( """ shape = fc._check_shape(shape=shape, key=key) if not (dtype.is_integer or dtype.is_floating): - raise ValueError('dtype must be convertible to float. ' 'dtype: {}, key: {}'.format(dtype, key)) + raise ValueError('dtype must be convertible to float. ' + 'dtype: {}, key: {}'.format(dtype, key)) if normalizer_fn is not None and not callable(normalizer_fn): - raise TypeError('normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) + raise TypeError( + 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) return SequenceNumericColumn( - feature_name=feature_name, - key=key, - shape=shape, - default_value=default_value, - dtype=dtype, - normalizer_fn=normalizer_fn, + feature_name=feature_name, + key=key, + shape=shape, + default_value=default_value, + dtype=dtype, + normalizer_fn=normalizer_fn, ) @@ -544,12 +559,13 @@ def _assert_all_equal_and_return(tensors, name=None): class SequenceNumericColumn( - fc.SequenceDenseColumn, - fc_v1._FeatureColumn, - collections.namedtuple( - 'SequenceNumericColumn', - ('feature_name', 'key', 'shape', 'default_value', 'dtype', 'normalizer_fn'), - ), + fc.SequenceDenseColumn, + fc_v1._FeatureColumn, + collections.namedtuple( + 'SequenceNumericColumn', + ('feature_name', 'key', 'shape', 'default_value', 'dtype', + 'normalizer_fn'), + ), ): """Represents sequences of numeric data.""" @@ -613,9 +629,11 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): lookup tables. """ sp_tensor = transformation_cache.get(self, state_manager) - dense_tensor = sparse_ops.sparse_tensor_to_dense(sp_tensor, default_value=self.default_value) + dense_tensor = sparse_ops.sparse_tensor_to_dense( + sp_tensor, default_value=self.default_value) # Reshape into [batch_size, T, variable_shape]. - dense_shape = array_ops.concat([array_ops.shape(dense_tensor)[:1], [-1], self.variable_shape], axis=0) + dense_shape = array_ops.concat( + [array_ops.shape(dense_tensor)[:1], [-1], self.variable_shape], axis=0) dense_tensor = array_ops.reshape(dense_tensor, shape=dense_shape) # Get the number of timesteps per example @@ -626,9 +644,11 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): num_elements = self.variable_shape.num_elements() else: num_elements = 1 - seq_length = fc_utils.sequence_length_from_sparse_tensor(sp_tensor, num_elements=num_elements) + seq_length = fc_utils.sequence_length_from_sparse_tensor( + sp_tensor, num_elements=num_elements) - return fc.SequenceDenseColumn.TensorSequenceLengthPair(dense_tensor=dense_tensor, sequence_length=seq_length) + return fc.SequenceDenseColumn.TensorSequenceLengthPair( + dense_tensor=dense_tensor, sequence_length=seq_length) # TODO(b/119409767): Implement parents, _{get,from}_config. @property diff --git a/easy_rec/python/compat/feature_column/utils.py b/easy_rec/python/compat/feature_column/utils.py index da303c7d3..35870e89d 100644 --- a/easy_rec/python/compat/feature_column/utils.py +++ b/easy_rec/python/compat/feature_column/utils.py @@ -15,11 +15,15 @@ # ============================================================================== """Defines functions common to multiple feature column files.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import six -from tensorflow.python.framework import dtypes, ops -from tensorflow.python.ops import array_ops, math_ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops from tensorflow.python.util import nest @@ -38,7 +42,8 @@ def sequence_length_from_sparse_tensor(sp_tensor, num_elements=1): # Example: orig tensor [[1, 2], [3]], col_ids = (0, 1, 1), # row_ids = (0, 0, 1), seq_length = [2, 1]. If num_elements = 2, # these will get grouped, and the final seq_length is [1, 1] - seq_length = math_ops.cast(math_ops.ceil(seq_length / num_elements), dtypes.int64) + seq_length = math_ops.cast( + math_ops.ceil(seq_length / num_elements), dtypes.int64) # If the last n rows do not have ids, seq_length will have shape # [batch_size - n]. Pad the remaining values with zeros. @@ -49,12 +54,15 @@ def sequence_length_from_sparse_tensor(sp_tensor, num_elements=1): def assert_string_or_int(dtype, prefix): if (dtype != dtypes.string) and (not dtype.is_integer): - raise ValueError('{} dtype must be string or integer. dtype: {}.'.format(prefix, dtype)) + raise ValueError('{} dtype must be string or integer. dtype: {}.'.format( + prefix, dtype)) def assert_key_is_string(key): if not isinstance(key, six.string_types): - raise ValueError('key must be a string. Got: type {}. Given key: {}.'.format(type(key), key)) + raise ValueError( + 'key must be a string. Got: type {}. Given key: {}.'.format( + type(key), key)) def check_default_value(shape, default_value, dtype, key): @@ -98,23 +106,22 @@ def check_default_value(shape, default_value, dtype, key): if nest.is_sequence(default_value): if not _is_shape_and_default_value_compatible(default_value, shape): raise ValueError( - 'The shape of default_value must be equal to given shape. ' 'default_value: {}, shape: {}, key: {}'.format( - default_value, shape, key - ) - ) + 'The shape of default_value must be equal to given shape. ' + 'default_value: {}, shape: {}, key: {}'.format( + default_value, shape, key)) # Check if the values in the list are all integers or are convertible to # floats. - is_list_all_int = all(isinstance(v, int) for v in nest.flatten(default_value)) - is_list_has_float = any(isinstance(v, float) for v in nest.flatten(default_value)) + is_list_all_int = all( + isinstance(v, int) for v in nest.flatten(default_value)) + is_list_has_float = any( + isinstance(v, float) for v in nest.flatten(default_value)) if is_list_all_int: return _as_tuple(default_value) if is_list_has_float and dtype.is_floating: return _as_tuple(default_value) - raise TypeError( - 'default_value must be compatible with dtype. ' 'default_value: {}, dtype: {}, key: {}'.format( - default_value, dtype, key - ) - ) + raise TypeError('default_value must be compatible with dtype. ' + 'default_value: {}, dtype: {}, key: {}'.format( + default_value, dtype, key)) def _create_tuple(shape, value): diff --git a/easy_rec/python/compat/layers.py b/easy_rec/python/compat/layers.py index f94c15320..3eb4fd17c 100644 --- a/easy_rec/python/compat/layers.py +++ b/easy_rec/python/compat/layers.py @@ -15,26 +15,31 @@ # ============================================================================== """Higher level ops for building layers.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import functools -from tensorflow.python.framework import dtypes, ops -from tensorflow.python.ops import init_ops, nn, variable_scope +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import variable_scope def layer_norm( - inputs, - center=True, - scale=True, - activation_fn=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - begin_norm_axis=1, - begin_params_axis=-1, - scope=None, + inputs, + center=True, + scale=True, + activation_fn=None, + reuse=None, + variables_collections=None, + outputs_collections=None, + trainable=True, + begin_norm_axis=1, + begin_params_axis=-1, + scope=None, ): """Adds a Layer Normalization layer. @@ -94,7 +99,8 @@ def layer_norm( or if `inputs.shape[begin_params_axis:]` is not fully defined at graph build time. """ - with variable_scope.variable_scope(scope, 'LayerNorm', [inputs], reuse=reuse) as sc: + with variable_scope.variable_scope( + scope, 'LayerNorm', [inputs], reuse=reuse) as sc: inputs = ops.convert_to_tensor(inputs) inputs_shape = inputs.shape inputs_rank = inputs_shape.ndims @@ -104,36 +110,36 @@ def layer_norm( if begin_norm_axis < 0: begin_norm_axis = inputs_rank + begin_norm_axis if begin_params_axis >= inputs_rank or begin_norm_axis >= inputs_rank: - raise ValueError( - 'begin_params_axis (%d) and begin_norm_axis (%d) ' - 'must be < rank(inputs) (%d)' % (begin_params_axis, begin_norm_axis, inputs_rank) - ) + raise ValueError('begin_params_axis (%d) and begin_norm_axis (%d) ' + 'must be < rank(inputs) (%d)' % + (begin_params_axis, begin_norm_axis, inputs_rank)) params_shape = inputs_shape[begin_params_axis:] if not params_shape.is_fully_defined(): raise ValueError( - 'Inputs %s: shape(inputs)[%s:] is not fully defined: %s' % (inputs.name, begin_params_axis, inputs_shape) - ) + 'Inputs %s: shape(inputs)[%s:] is not fully defined: %s' % + (inputs.name, begin_params_axis, inputs_shape)) # Allocate parameters for the beta and gamma of the normalization. beta, gamma = None, None if center: beta_collections = get_variable_collections(variables_collections, 'beta') beta = model_variable( - 'beta', - shape=params_shape, - dtype=dtype, - initializer=init_ops.zeros_initializer(), - collections=beta_collections, - trainable=trainable, + 'beta', + shape=params_shape, + dtype=dtype, + initializer=init_ops.zeros_initializer(), + collections=beta_collections, + trainable=trainable, ) if scale: - gamma_collections = get_variable_collections(variables_collections, 'gamma') + gamma_collections = get_variable_collections(variables_collections, + 'gamma') gamma = model_variable( - 'gamma', - shape=params_shape, - dtype=dtype, - initializer=init_ops.ones_initializer(), - collections=gamma_collections, - trainable=trainable, + 'gamma', + shape=params_shape, + dtype=dtype, + initializer=init_ops.ones_initializer(), + collections=gamma_collections, + trainable=trainable, ) # Calculate the moments on the last axis (layer activations). norm_axes = list(range(begin_norm_axis, inputs_rank)) @@ -141,12 +147,12 @@ def layer_norm( # Compute layer normalization using the batch_normalization function. variance_epsilon = 1e-12 outputs = nn.batch_normalization( - inputs, - mean, - variance, - offset=beta, - scale=gamma, - variance_epsilon=variance_epsilon, + inputs, + mean, + variance, + offset=beta, + scale=gamma, + variance_epsilon=variance_epsilon, ) outputs.set_shape(inputs_shape) if activation_fn is not None: @@ -205,18 +211,18 @@ def append_tensor_alias(tensor, alias): def variable( - name, - shape=None, - dtype=None, - initializer=None, - regularizer=None, - trainable=True, - collections=None, - caching_device=None, - device=None, - partitioner=None, - custom_getter=None, - use_resource=None, + name, + shape=None, + dtype=None, + initializer=None, + regularizer=None, + trainable=True, + collections=None, + caching_device=None, + device=None, + partitioner=None, + custom_getter=None, + use_resource=None, ): """Gets an existing variable with these parameters or creates a new one. @@ -247,41 +253,43 @@ def variable( Returns: The created or existing variable. """ - collections = list(collections if collections is not None else [ops.GraphKeys.GLOBAL_VARIABLES]) + collections = list(collections if collections is not None else + [ops.GraphKeys.GLOBAL_VARIABLES]) # Remove duplicates collections = list(set(collections)) getter = variable_scope.get_variable if custom_getter is not None: - getter = functools.partial(custom_getter, reuse=variable_scope.get_variable_scope().reuse) + getter = functools.partial( + custom_getter, reuse=variable_scope.get_variable_scope().reuse) with ops.device(device or ''): return getter( - name, - shape=shape, - dtype=dtype, - initializer=initializer, - regularizer=regularizer, - trainable=trainable, - collections=collections, - caching_device=caching_device, - partitioner=partitioner, - use_resource=use_resource, + name, + shape=shape, + dtype=dtype, + initializer=initializer, + regularizer=regularizer, + trainable=trainable, + collections=collections, + caching_device=caching_device, + partitioner=partitioner, + use_resource=use_resource, ) def model_variable( - name, - shape=None, - dtype=dtypes.float32, - initializer=None, - regularizer=None, - trainable=True, - collections=None, - caching_device=None, - device=None, - partitioner=None, - custom_getter=None, - use_resource=None, + name, + shape=None, + dtype=dtypes.float32, + initializer=None, + regularizer=None, + trainable=True, + collections=None, + caching_device=None, + device=None, + partitioner=None, + custom_getter=None, + use_resource=None, ): """Gets an existing model variable with these parameters or creates a new one. @@ -316,17 +324,17 @@ def model_variable( collections = list(collections or []) collections += [ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.MODEL_VARIABLES] var = variable( - name, - shape=shape, - dtype=dtype, - initializer=initializer, - regularizer=regularizer, - trainable=trainable, - collections=collections, - caching_device=caching_device, - device=device, - partitioner=partitioner, - custom_getter=custom_getter, - use_resource=use_resource, + name, + shape=shape, + dtype=dtype, + initializer=initializer, + regularizer=regularizer, + trainable=trainable, + collections=collections, + caching_device=caching_device, + device=device, + partitioner=partitioner, + custom_getter=custom_getter, + use_resource=use_resource, ) return var diff --git a/easy_rec/python/compat/optimizers.py b/easy_rec/python/compat/optimizers.py index 2ef07ed68..ed3fadb7c 100644 --- a/easy_rec/python/compat/optimizers.py +++ b/easy_rec/python/compat/optimizers.py @@ -15,26 +15,17 @@ # ============================================================================== """Optimizer ops for use in layers and tf.learn.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import logging import six import tensorflow as tf - # from tensorflow.contrib import framework as contrib_framework -from tensorflow.python.framework import dtypes, ops - -# from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import ( # NOQA - array_ops, - clip_ops, - control_flow_ops, - gen_nn_ops, - init_ops, - math_ops, - random_ops, -) +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.ops import variable_scope as vs from tensorflow.python.ops import variables as vars_ from tensorflow.python.summary import summary @@ -43,7 +34,14 @@ from tensorflow.python.training import training as train from easy_rec.python.ops.incr_record import set_sparse_indices -from easy_rec.python.utils import constant, estimator_utils +from easy_rec.python.utils import constant +from easy_rec.python.utils import estimator_utils + +# from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import ( # NOQA + array_ops, clip_ops, control_flow_ops, gen_nn_ops, init_ops, math_ops, + random_ops, +) try: from tensorflow.python.framework import indexed_slices @@ -63,41 +61,48 @@ sok = None OPTIMIZER_CLS_NAMES = { - 'Adagrad': train.AdagradOptimizer, - 'Adam': train.AdamOptimizer, - 'Ftrl': train.FtrlOptimizer, - 'Momentum': lambda learning_rate: train.MomentumOptimizer(learning_rate, momentum=0.9), - 'RMSProp': train.RMSPropOptimizer, - 'SGD': train.GradientDescentOptimizer, + 'Adagrad': + train.AdagradOptimizer, + 'Adam': + train.AdamOptimizer, + 'Ftrl': + train.FtrlOptimizer, + 'Momentum': + lambda learning_rate: train.MomentumOptimizer( + learning_rate, momentum=0.9), + 'RMSProp': + train.RMSPropOptimizer, + 'SGD': + train.GradientDescentOptimizer, } OPTIMIZER_SUMMARIES = [ - 'learning_rate', - 'loss', - 'gradients', - 'gradient_norm', - 'global_gradient_norm', + 'learning_rate', + 'loss', + 'gradients', + 'gradient_norm', + 'global_gradient_norm', ] def optimize_loss( - loss, - global_step, - learning_rate, - optimizer, - gradient_noise_scale=None, - gradient_multipliers=None, - clip_gradients=None, - learning_rate_decay_fn=None, - update_ops=None, - variables=None, - name=None, - summaries=None, - colocate_gradients_with_ops=False, - not_apply_grad_after_first_step=False, - increment_global_step=True, - incr_save=False, - embedding_parallel=False, + loss, + global_step, + learning_rate, + optimizer, + gradient_noise_scale=None, + gradient_multipliers=None, + clip_gradients=None, + learning_rate_decay_fn=None, + update_ops=None, + variables=None, + name=None, + summaries=None, + colocate_gradients_with_ops=False, + not_apply_grad_after_first_step=False, + increment_global_step=True, + incr_save=False, + embedding_parallel=False, ): """Given loss and parameters for optimizer, returns a training op. @@ -204,28 +209,29 @@ def optimize_loss( # Learning rate variable, with possible decay. lr = None if learning_rate is not None: - if isinstance(learning_rate, ops.Tensor) and learning_rate.get_shape().ndims == 0: + if isinstance(learning_rate, + ops.Tensor) and learning_rate.get_shape().ndims == 0: lr = learning_rate elif isinstance(learning_rate, float): if learning_rate < 0.0: raise ValueError('Invalid learning_rate %s.', learning_rate) lr = vs.get_variable( - 'learning_rate', - [], - trainable=False, - initializer=init_ops.constant_initializer(learning_rate), + 'learning_rate', + [], + trainable=False, + initializer=init_ops.constant_initializer(learning_rate), ) else: - raise ValueError( - 'Learning rate should be 0d Tensor or float. ' - 'Got %s of type %s' % (str(learning_rate), str(type(learning_rate))) - ) + raise ValueError('Learning rate should be 0d Tensor or float. ' + 'Got %s of type %s' % + (str(learning_rate), str(type(learning_rate)))) if summaries is None: summaries = ['loss', 'learning_rate', 'global_gradient_norm'] else: for summ in summaries: if summ not in OPTIMIZER_SUMMARIES: - raise ValueError('Summaries should be one of [%s], you provided %s.' % (', '.join(OPTIMIZER_SUMMARIES), summ)) + raise ValueError('Summaries should be one of [%s], you provided %s.' % + (', '.join(OPTIMIZER_SUMMARIES), summ)) if learning_rate is not None and learning_rate_decay_fn is not None: if global_step is None: raise ValueError('global_step is required for learning_rate_decay_fn.') @@ -236,15 +242,18 @@ def optimize_loss( # Create optimizer, given specified parameters. if isinstance(optimizer, six.string_types): if lr is None: - raise ValueError('Learning rate is None, but should be specified if ' 'optimizer is string (%s).' % optimizer) + raise ValueError('Learning rate is None, but should be specified if ' + 'optimizer is string (%s).' % optimizer) if optimizer not in OPTIMIZER_CLS_NAMES: raise ValueError( - 'Optimizer name should be one of [%s], you provided %s.' % (', '.join(OPTIMIZER_CLS_NAMES), optimizer) - ) + 'Optimizer name should be one of [%s], you provided %s.' % + (', '.join(OPTIMIZER_CLS_NAMES), optimizer)) opt = OPTIMIZER_CLS_NAMES[optimizer](learning_rate=lr) - elif isinstance(optimizer, type) and issubclass(optimizer, optimizer_.Optimizer): + elif isinstance(optimizer, type) and issubclass(optimizer, + optimizer_.Optimizer): if lr is None: - raise ValueError('Learning rate is None, but should be specified if ' 'optimizer is class (%s).' % optimizer) + raise ValueError('Learning rate is None, but should be specified if ' + 'optimizer is class (%s).' % optimizer) opt = optimizer(learning_rate=lr) elif isinstance(optimizer, optimizer_.Optimizer): opt = optimizer @@ -254,41 +263,41 @@ def optimize_loss( else: opt = optimizer() if not isinstance(opt, optimizer_.Optimizer): - raise ValueError('Unrecognized optimizer: function should return ' 'subclass of Optimizer. Got %s.' % str(opt)) + raise ValueError('Unrecognized optimizer: function should return ' + 'subclass of Optimizer. Got %s.' % str(opt)) elif isinstance(optimizer, sok_optimizer.OptimizerWrapperV1) or isinstance( - optimizer, sok_optimizer.OptimizerWrapperV2 - ): + optimizer, sok_optimizer.OptimizerWrapperV2): opt = optimizer else: - raise ValueError( - 'Unrecognized optimizer: should be string, ' - 'subclass of Optimizer, instance of ' - 'subclass of Optimizer or function with one argument. ' - 'Got %s[type=%s].' % (str(optimizer), str(type(optimizer))) - ) + raise ValueError('Unrecognized optimizer: should be string, ' + 'subclass of Optimizer, instance of ' + 'subclass of Optimizer or function with one argument. ' + 'Got %s[type=%s].' % + (str(optimizer), str(type(optimizer)))) # All trainable variables, if specific variables are not specified. if variables is None: variables = vars_.trainable_variables() # Compute gradients. - gradients = opt.compute_gradients(loss, variables, colocate_gradients_with_ops=colocate_gradients_with_ops) + gradients = opt.compute_gradients( + loss, + variables, + colocate_gradients_with_ops=colocate_gradients_with_ops) if estimator_utils.has_hvd() and hvd.size() > 1: if not embedding_parallel: # embedding parameters not partitioned reduced_grads = [] for g, v in gradients: - reduced_grads.append( - ( + reduced_grads.append(( hvd.allreduce( - g, - op=hvd.Average, - compression=hvd.compression.NoneCompressor, + g, + op=hvd.Average, + compression=hvd.compression.NoneCompressor, ), v, - ) - ) + )) gradients = reduced_grads else: # embedding parameters partitioned: @@ -311,62 +320,61 @@ def optimize_loss( part_grads.append(g) part_vars.append(v) else: - reduced_grads.append( - ( - indexed_slices.IndexedSlices(indices=g.indices, values=g.values / hvd.size()), + reduced_grads.append(( + indexed_slices.IndexedSlices( + indices=g.indices, values=g.values / hvd.size()), v, - ) - ) + )) group_allreduce = False if len(part_grads) > 0: if group_allreduce: reduced_part_grads = hvd.grouped_allreduce( - part_grads, - op=hvd.Average, - compression=hvd.compression.NoneCompressor, + part_grads, + op=hvd.Average, + compression=hvd.compression.NoneCompressor, ) for g, v in zip(reduced_part_grads, part_vars): reduced_grads.append((g, v)) else: for g, v in zip(part_grads, part_vars): g = hvd.allreduce( - g, - op=hvd.Average, - compression=hvd.compression.NoneCompressor, + g, + op=hvd.Average, + compression=hvd.compression.NoneCompressor, ) reduced_grads.append((g, v)) if len(part_sparse_grads) > 0: if group_allreduce: reduced_part_grads = hvd.grouped_allreduce( - part_sparse_grads, - op=hvd.Average, - compression=hvd.compression.NoneCompressor, + part_sparse_grads, + op=hvd.Average, + compression=hvd.compression.NoneCompressor, ) for g, v in zip(reduced_part_grads, part_sparse_vars): reduced_grads.append((g, v)) else: for g, v in zip(part_sparse_grads, part_sparse_vars): g = hvd.allreduce( - g, - op=hvd.Average, - compression=hvd.compression.NoneCompressor, + g, + op=hvd.Average, + compression=hvd.compression.NoneCompressor, ) reduced_grads.append((g, v)) gradients = reduced_grads # Optionally add gradient noise. if gradient_noise_scale is not None: - gradients = _add_scaled_noise_to_gradients(gradients, gradient_noise_scale) + gradients = _add_scaled_noise_to_gradients(gradients, + gradient_noise_scale) # Multiply some gradients. if gradient_multipliers is not None: gradients = _multiply_gradients(gradients, gradient_multipliers) if not gradients: raise ValueError( - 'Empty list of (gradient, var) pairs encountered. This is most ' - 'likely to be caused by an improper value of gradient_multipliers.' - ) + 'Empty list of (gradient, var) pairs encountered. This is most ' + 'likely to be caused by an improper value of gradient_multipliers.') # if 'global_gradient_norm' in summaries or 'gradient_norm' in summaries: # summary.scalar('global_norm/gradient_norm', @@ -375,18 +383,21 @@ def optimize_loss( # Optionally clip gradients by global norm. if isinstance(clip_gradients, float): # gradients = _clip_gradients_by_norm(gradients, clip_gradients) - sparse_norm, dense_norm, grad_norm = _get_grad_norm(gradients, embedding_parallel) + sparse_norm, dense_norm, grad_norm = _get_grad_norm( + gradients, embedding_parallel) summary.scalar('global_norm/sparse_grad', sparse_norm) summary.scalar('global_norm/dense_grad', dense_norm) summary.scalar('global_norm/gradient_norm', grad_norm) grads = [x[0] for x in gradients] vars = [x[1] for x in gradients] - clipped_grads, _ = clip_ops.clip_by_global_norm(grads, clip_gradients, use_norm=grad_norm) + clipped_grads, _ = clip_ops.clip_by_global_norm( + grads, clip_gradients, use_norm=grad_norm) gradients = list(zip(clipped_grads, vars)) elif callable(clip_gradients): gradients = clip_gradients(gradients) elif clip_gradients is not None: - raise ValueError('Unknown type %s for clip_gradients' % type(clip_gradients)) + raise ValueError('Unknown type %s for clip_gradients' % + type(clip_gradients)) # Add scalar summary for loss. if 'loss' in summaries: @@ -406,12 +417,14 @@ def optimize_loss( summary.histogram('gradients/%s' % var_name, grad_values) if 'gradient_norm' in summaries: summary.scalar( - 'gradient_norm/%s' % var_name, - clip_ops.global_norm([grad_values]), + 'gradient_norm/%s' % var_name, + clip_ops.global_norm([grad_values]), ) - if clip_gradients is not None and ('global_gradient_norm' in summaries or 'gradient_norm' in summaries): - sparse_norm, dense_norm, grad_norm = _get_grad_norm(gradients, embedding_parallel) + if clip_gradients is not None and ('global_gradient_norm' in summaries or + 'gradient_norm' in summaries): + sparse_norm, dense_norm, grad_norm = _get_grad_norm( + gradients, embedding_parallel) summary.scalar('global_norm/clipped_sparse_grad', sparse_norm) summary.scalar('global_norm/clipped_dense_grad', dense_norm) summary.scalar('global_norm/clipped_gradient_norm', grad_norm) @@ -419,9 +432,9 @@ def optimize_loss( # Create gradient updates. def _apply_grad(): grad_updates = opt.apply_gradients( - gradients, - global_step=global_step if increment_global_step else None, - name='train', + gradients, + global_step=global_step if increment_global_step else None, + name='train', ) embed_para_vars = ops.get_collection(constant.EmbeddingParallel) @@ -438,10 +451,12 @@ def _apply_grad(): for grad, var in gradients: if isinstance(grad, indexed_slices.IndexedSlices): indices = grad.indices - with ops.colocate_with(var), ops.control_dependencies([grad_updates]): + with ops.colocate_with(var), ops.control_dependencies( + [grad_updates]): incr_save_op = set_sparse_indices(indices, var_name=var.op.name) incr_save_ops.append(incr_save_op) - ops.add_to_collection('SPARSE_UPDATE_VARIABLES', (var, grad.indices.dtype)) + ops.add_to_collection('SPARSE_UPDATE_VARIABLES', + (var, grad.indices.dtype)) else: ops.add_to_collection('DENSE_UPDATE_VARIABLES', var) return tf.group(incr_save_ops) @@ -472,16 +487,19 @@ def _get_grad_norm(grads_and_vars, embedding_parallel=False): else: dense_norms.append(gen_nn_ops.l2_loss(grad)) if hvd is not None and part_norms: - reduced_norms = hvd.grouped_allreduce(part_norms, op=hvd.Sum, compression=hvd.compression.NoneCompressor) + reduced_norms = hvd.grouped_allreduce( + part_norms, op=hvd.Sum, compression=hvd.compression.NoneCompressor) sparse_norms = sparse_norms + reduced_norms all_norms = sparse_norms + dense_norms sparse_norm = ( - math_ops.sqrt(math_ops.reduce_sum(array_ops.stack(sparse_norms) * 2.0)) if sparse_norms else tf.constant(0.0) - ) + math_ops.sqrt(math_ops.reduce_sum(array_ops.stack(sparse_norms) * 2.0)) + if sparse_norms else tf.constant(0.0)) dense_norm = ( - math_ops.sqrt(math_ops.reduce_sum(array_ops.stack(dense_norms) * 2.0)) if dense_norms else tf.constant(0.0) - ) - grad_norm = math_ops.sqrt(math_ops.reduce_sum(array_ops.stack(all_norms)) * 2.0) if all_norms else tf.constant(0.0) + math_ops.sqrt(math_ops.reduce_sum(array_ops.stack(dense_norms) * 2.0)) + if dense_norms else tf.constant(0.0)) + grad_norm = math_ops.sqrt( + math_ops.reduce_sum(array_ops.stack(all_norms)) * + 2.0) if all_norms else tf.constant(0.0) return sparse_norm, dense_norm, grad_norm @@ -500,13 +518,14 @@ def _adaptive_max_norm(norm, std_factor, decay, global_step, epsilon, name): def moving_average(name, value, decay): moving_average_variable = vs.get_variable( - name, - shape=value.get_shape(), - dtype=value.dtype, - initializer=init_ops.zeros_initializer(), - trainable=False, + name, + shape=value.get_shape(), + dtype=value.dtype, + initializer=init_ops.zeros_initializer(), + trainable=False, ) - return moving_averages.assign_moving_average(moving_average_variable, value, decay, zero_debias=False) + return moving_averages.assign_moving_average( + moving_average_variable, value, decay, zero_debias=False) # quicker adaptation at the beginning if global_step is not None: @@ -524,13 +543,13 @@ def moving_average(name, value, decay): def adaptive_clipping_fn( - std_factor=2.0, - decay=0.95, - static_max_norm=None, - global_step=None, - report_summary=False, - epsilon=1e-8, - name=None, + std_factor=2.0, + decay=0.95, + static_max_norm=None, + global_step=None, + report_summary=False, + epsilon=1e-8, + name=None, ): """Adapt the clipping value using statistics on the norms. @@ -563,14 +582,16 @@ def gradient_clipping(grads_and_vars): norm = clip_ops.global_norm(grads) - max_norm, log_mean = _adaptive_max_norm(norm, std_factor, decay, global_step, epsilon, name) + max_norm, log_mean = _adaptive_max_norm(norm, std_factor, decay, + global_step, epsilon, name) # reports the max gradient norm for debugging if report_summary: summary.scalar('global_norm/adaptive_max_gradient_norm', max_norm) # factor will be 1. if norm is smaller than max_norm - factor = array_ops.where(norm < max_norm, array_ops.ones_like(norm), math_ops.exp(log_mean) / norm) + factor = array_ops.where(norm < max_norm, array_ops.ones_like(norm), + math_ops.exp(log_mean) / norm) if static_max_norm is not None: factor = math_ops.minimum(static_max_norm / norm, factor) @@ -581,7 +602,9 @@ def gradient_clipping(grads_and_vars): if grad is None: clipped_grads.append(None) elif isinstance(grad, indexed_slices.IndexedSlices): - clipped_grads.append(indexed_slices.IndexedSlices(grad.values * factor, grad.indices, grad.dense_shape)) + clipped_grads.append( + indexed_slices.IndexedSlices(grad.values * factor, grad.indices, + grad.dense_shape)) else: clipped_grads.append(grad * factor) @@ -611,12 +634,14 @@ def _multiply_gradients(grads_and_vars, gradient_multipliers): """Multiply specified gradients.""" multiplied_grads_and_vars = [] for grad, var in grads_and_vars: - if grad is not None and (var in gradient_multipliers or var.name in gradient_multipliers): + if grad is not None and (var in gradient_multipliers or + var.name in gradient_multipliers): key = var if var in gradient_multipliers else var.name multiplier = gradient_multipliers[key] if isinstance(grad, indexed_slices.IndexedSlices): grad_values = grad.values * multiplier - grad = indexed_slices.IndexedSlices(grad_values, grad.indices, grad.dense_shape) + grad = indexed_slices.IndexedSlices(grad_values, grad.indices, + grad.dense_shape) else: grad *= math_ops.cast(multiplier, grad.dtype) multiplied_grads_and_vars.append((grad, var)) diff --git a/easy_rec/python/compat/queues.py b/easy_rec/python/compat/queues.py index ac2c14cf8..a71a43bc2 100644 --- a/easy_rec/python/compat/queues.py +++ b/easy_rec/python/compat/queues.py @@ -16,8 +16,11 @@ import time import weakref from multiprocessing import connection -from multiprocessing.util import Finalize, is_exiting, register_after_fork -from queue import Empty, Full +from multiprocessing.util import Finalize +from multiprocessing.util import is_exiting +from multiprocessing.util import register_after_fork +from queue import Empty +from queue import Full import six @@ -65,30 +68,30 @@ def __init__(self, ctx, maxsize=0, name=''): def __getstate__(self): context.assert_spawning(self) return ( - self._ignore_epipe, - self._maxsize, - self._reader, - self._writer, - self._rlock, - self._wlock, - self._sem, - self._opid, - self._name, - self._run, + self._ignore_epipe, + self._maxsize, + self._reader, + self._writer, + self._rlock, + self._wlock, + self._sem, + self._opid, + self._name, + self._run, ) def __setstate__(self, state): ( - self._ignore_epipe, - self._maxsize, - self._reader, - self._writer, - self._rlock, - self._wlock, - self._sem, - self._opid, - self._name, - self._run, + self._ignore_epipe, + self._maxsize, + self._reader, + self._writer, + self._rlock, + self._wlock, + self._sem, + self._opid, + self._name, + self._run, ) = state self._reset() @@ -168,7 +171,8 @@ def put_nowait(self, obj): def close(self, wait_send_finish=True): self._closed = True close = self._close - if not wait_send_finish and self._thread is not None and self._thread.is_alive(): + if not wait_send_finish and self._thread is not None and self._thread.is_alive( + ): try: if self._reader is not None: self._reader.close() @@ -205,19 +209,19 @@ def _start_thread(self): # Start thread which transfers data from buffer to pipe self._buffer.clear() self._thread = threading.Thread( - target=self._feed, - args=( - self._buffer, - self._notempty, - self._send_bytes, - self._wlock, - self._reader.close, - self._writer.close, - self._ignore_epipe, - self._on_queue_feeder_error, - self._sem, - ), - name='QueueFeederThread', + target=self._feed, + args=( + self._buffer, + self._notempty, + self._send_bytes, + self._wlock, + self._reader.close, + self._writer.close, + self._ignore_epipe, + self._on_queue_feeder_error, + self._sem, + ), + name='QueueFeederThread', ) self._thread.daemon = True @@ -227,14 +231,17 @@ def _start_thread(self): if not self._joincancelled: self._jointhread = Finalize( - self._thread, - Queue._finalize_join, - [weakref.ref(self._thread)], - exitpriority=-5, + self._thread, + Queue._finalize_join, + [weakref.ref(self._thread)], + exitpriority=-5, ) # Send sentinel to the thread queue object when garbage collected - self._close = Finalize(self, Queue._finalize_close, [self._buffer, self._notempty], exitpriority=10) + self._close = Finalize( + self, + Queue._finalize_close, [self._buffer, self._notempty], + exitpriority=10) @staticmethod def _finalize_join(twr): @@ -254,16 +261,16 @@ def _finalize_close(buffer, notempty): notempty.notify() def _feed( - self, - buffer, - notempty, - send_bytes, - writelock, - reader_close, - writer_close, - ignore_epipe, - onerror, - queue_sem, + self, + buffer, + notempty, + send_bytes, + writelock, + reader_close, + writer_close, + ignore_epipe, + onerror, + queue_sem, ): logging.debug('starting thread to feed data to pipe') nacquire = notempty.acquire @@ -309,16 +316,16 @@ def _feed( pass except Exception as e: if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: - logging.warning('Queue[' + name + '] exception: pid=' + str(pid) + ' run=' + str(self._run) + ' e=' + str(e)) + logging.warning('Queue[' + name + '] exception: pid=' + str(pid) + + ' run=' + str(self._run) + ' e=' + str(e)) return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. if is_exiting(): - logging.warning( - 'Queue[' + name + '] thread error in exiting: pid=' + str(pid) + ' run=' + str(self._run) + ' e=' + str(e) - ) + logging.warning('Queue[' + name + '] thread error in exiting: pid=' + + str(pid) + ' run=' + str(self._run) + ' e=' + str(e)) return else: # Since the object has not been sent in the queue, we need diff --git a/easy_rec/python/compat/regularizers.py b/easy_rec/python/compat/regularizers.py index a7a448170..0ea11662f 100644 --- a/easy_rec/python/compat/regularizers.py +++ b/easy_rec/python/compat/regularizers.py @@ -16,20 +16,25 @@ # from tf.contrib """Regularizers for use with layers.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import numbers -from tensorflow.python.framework import constant_op, ops -from tensorflow.python.ops import math_ops, nn, standard_ops +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import standard_ops from tensorflow.python.platform import tf_logging as logging __all__ = [ - 'l1_regularizer', - 'l2_regularizer', - 'l1_l2_regularizer', - 'sum_regularizer', - 'apply_regularization', + 'l1_regularizer', + 'l2_regularizer', + 'l1_l2_regularizer', + 'sum_regularizer', + 'apply_regularization', ] @@ -52,7 +57,8 @@ def l1_regularizer(scale, scope=None): raise ValueError('scale cannot be an integer: %s' % scale) if isinstance(scale, numbers.Real): if scale < 0.0: - raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale) + raise ValueError('Setting a scale less than 0 on a regularizer: %g' % + scale) if scale == 0.0: logging.info('Scale of 0 disables regularizer.') return lambda _: None @@ -60,8 +66,12 @@ def l1_regularizer(scale, scope=None): def l1(weights, name=None): """Applies L1 regularization to weights.""" with ops.name_scope(scope, 'l1_regularizer', [weights]) as name: - my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') - return standard_ops.multiply(my_scale, standard_ops.reduce_sum(standard_ops.abs(weights)), name=name) + my_scale = ops.convert_to_tensor( + scale, dtype=weights.dtype.base_dtype, name='scale') + return standard_ops.multiply( + my_scale, + standard_ops.reduce_sum(standard_ops.abs(weights)), + name=name) return l1 @@ -85,7 +95,8 @@ def l2_regularizer(scale, scope=None): raise ValueError('scale cannot be an integer: %s' % (scale,)) if isinstance(scale, numbers.Real): if scale < 0.0: - raise ValueError('Setting a scale less than 0 on a regularizer: %g.' % scale) + raise ValueError('Setting a scale less than 0 on a regularizer: %g.' % + scale) if scale == 0.0: logging.info('Scale of 0 disables regularizer.') return lambda _: None @@ -93,7 +104,8 @@ def l2_regularizer(scale, scope=None): def l2(weights): """Applies l2 regularization to weights.""" with ops.name_scope(scope, 'l2_regularizer', [weights]) as name: - my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') + my_scale = ops.convert_to_tensor( + scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.multiply(my_scale, nn.l2_loss(weights), name=name) return l2 @@ -123,7 +135,9 @@ def l1_l2_regularizer(scale_l1=1.0, scale_l2=1.0, scope=None): return l2_regularizer(scale_l2, scope) if scale_l2 == 0.0: return l1_regularizer(scale_l1, scope) - return sum_regularizer([l1_regularizer(scale_l1), l2_regularizer(scale_l2)], scope=scope) + return sum_regularizer([l1_regularizer(scale_l1), + l2_regularizer(scale_l2)], + scope=scope) def sum_regularizer(regularizer_list, scope=None): @@ -149,7 +163,8 @@ def sum_reg(weights): tensor = reg(weights) if tensor is not None: regularizer_tensors.append(tensor) - return math_ops.add_n(regularizer_tensors, name=name) if regularizer_tensors else None + return math_ops.add_n( + regularizer_tensors, name=name) if regularizer_tensors else None return sum_reg @@ -180,14 +195,16 @@ def apply_regularization(regularizer, weights_list=None): weights_list = ops.get_collection(ops.GraphKeys.WEIGHTS) if not weights_list: raise ValueError('No weights to regularize.') - with ops.name_scope('get_regularization_penalty', values=weights_list) as scope: + with ops.name_scope( + 'get_regularization_penalty', values=weights_list) as scope: penalties = [regularizer(w) for w in weights_list] - penalties = [p if p is not None else constant_op.constant(0.0) for p in penalties] + penalties = [ + p if p is not None else constant_op.constant(0.0) for p in penalties + ] for p in penalties: if p.get_shape().ndims != 0: - raise ValueError( - 'regularizer must return a scalar Tensor instead of a ' 'Tensor with rank %d.' % p.get_shape().ndims - ) + raise ValueError('regularizer must return a scalar Tensor instead of a ' + 'Tensor with rank %d.' % p.get_shape().ndims) summed_penalty = math_ops.add_n(penalties, name=scope) ops.add_to_collection(ops.GraphKeys.REGULARIZATION_LOSSES, summed_penalty) diff --git a/easy_rec/python/compat/sok_optimizer.py b/easy_rec/python/compat/sok_optimizer.py index 4c9e3cfb2..a2e3ec361 100644 --- a/easy_rec/python/compat/sok_optimizer.py +++ b/easy_rec/python/compat/sok_optimizer.py @@ -16,20 +16,16 @@ import tensorflow as tf from tensorflow.python.eager import context - # from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from easy_rec.python.compat.dynamic_variable import DynamicVariable + # from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import ( # NOQA - array_ops, - gradients, - resource_variable_ops, - state_ops, + array_ops, gradients, resource_variable_ops, state_ops, ) -from easy_rec.python.compat.dynamic_variable import DynamicVariable - def OptimizerWrapper(optimizer): """Abbreviated as ``sok.experiment.OptimizerWrapper``. @@ -83,10 +79,14 @@ def OptimizerWrapper(optimizer): class OptimizerWrapperV1(object): + def __init__(self, optimizer): self._optimizer = optimizer # slots - unused = tf.Variable([0.0], dtype=tf.float32, name='unused', trainable=False) + unused = tf.Variable([0.0], + dtype=tf.float32, + name='unused', + trainable=False) self._optimizer._create_slots([unused]) names, slots = [], [] for name in self._optimizer.get_slot_names(): @@ -102,12 +102,12 @@ def __init__(self, optimizer): # self._optimizer._prepare() def compute_gradients( - self, - loss, - var_list=None, - aggregation_method=None, - colocate_gradients_with_ops=False, - grad_loss=None, + self, + loss, + var_list=None, + aggregation_method=None, + colocate_gradients_with_ops=False, + grad_loss=None, ): self._loss = loss tmp_grads = gradients.gradients(loss, var_list) @@ -139,22 +139,22 @@ def _create_slots_dynamic(self, var): if var.backend_type == 'hbm': with ops.colocate_with(var): slot = DynamicVariable( - dimension=var.dimension, - initializer=self._initial_vals[slot_name], - name='DynamicSlot', - trainable=False, + dimension=var.dimension, + initializer=self._initial_vals[slot_name], + name='DynamicSlot', + trainable=False, ) else: tmp_config = var.config_dict # tmp_initializer = var.initializer_str with ops.colocate_with(var): slot = DynamicVariable( - dimension=var.dimension, - initializer=self._initial_vals[slot_name], - var_type=var.backend_type, - name='DynamicSlot', - trainable=False, - **tmp_config, + dimension=var.dimension, + initializer=self._initial_vals[slot_name], + var_type=var.backend_type, + name='DynamicSlot', + trainable=False, + **tmp_config, ) self._optimizer._slots[slot_name][key] = slot @@ -173,7 +173,9 @@ def _slots(self): def apply_gradients(self, grads_and_vars, global_step=None, name=None): gradients = grads_and_vars sparse_vars = [x for x in gradients if 'DynamicVariable' in str(type(x[1]))] - dense_vars = [x for x in gradients if 'DynamicVariable' not in str(type(x[1]))] + dense_vars = [ + x for x in gradients if 'DynamicVariable' not in str(type(x[1])) + ] def _dummy_finish(update_ops, name_scope): return update_ops @@ -183,7 +185,8 @@ def _dummy_finish(update_ops, name_scope): with ops.control_dependencies([array_ops.identity(self._loss)]): sparse_grad_updates = self.apply_sparse_gradients(sparse_vars, name=name) - dense_grad_updates = self._optimizer.apply_gradients(dense_vars, global_step=None, name=name) + dense_grad_updates = self._optimizer.apply_gradients( + dense_vars, global_step=None, name=name) if sparse_grad_updates is not None and dense_grad_updates is not None: grad_updates = sparse_grad_updates + dense_grad_updates elif sparse_grad_updates is not None: @@ -198,9 +201,9 @@ def _dummy_finish(update_ops, name_scope): # TODO(apassos): the implicit read in assign_add is slow; consider # making it less so. apply_updates = resource_variable_ops.assign_add_variable_op( - global_step.handle, - ops.convert_to_tensor(1, dtype=global_step.dtype), - name=name, + global_step.handle, + ops.convert_to_tensor(1, dtype=global_step.dtype), + name=name, ) else: apply_updates = state_ops.assign_add(global_step, 1, name=name) @@ -233,22 +236,22 @@ def apply_sparse_gradients(self, grads_and_vars, global_step=None, name=None): if v.backend_type == 'hbm': with ops.colocate_with(v): slot = DynamicVariable( - dimension=v.dimension, - initializer=self._initial_vals[slot_name], - name=tmp_slot_var_name, - trainable=False, + dimension=v.dimension, + initializer=self._initial_vals[slot_name], + name=tmp_slot_var_name, + trainable=False, ) else: tmp_config = v.config_dict # tmp_initializer = v.initializer_str with ops.colocate_with(v): slot = DynamicVariable( - dimension=v.dimension, - initializer=self._initial_vals[slot_name], - var_type=v.backend_type, - name=tmp_slot_var_name, - trainable=False, - **tmp_config, + dimension=v.dimension, + initializer=self._initial_vals[slot_name], + var_type=v.backend_type, + name=tmp_slot_var_name, + trainable=False, + **tmp_config, ) self._optimizer._slots[slot_name][key] = slot @@ -261,7 +264,8 @@ def apply_sparse_gradients(self, grads_and_vars, global_step=None, name=None): # 3. Call tf-optimizer with ops.control_dependencies(to_static_ops): - train_op = self._optimizer.apply_gradients(zip(grad_list, var_list), global_step=global_step, name=name) + train_op = self._optimizer.apply_gradients( + zip(grad_list, var_list), global_step=global_step, name=name) # 5. Write buffer back to dynamic variables to_dynamic_ops = [] @@ -279,11 +283,15 @@ def apply_sparse_gradients(self, grads_and_vars, global_step=None, name=None): class OptimizerWrapperV2(object): + def __init__(self, optimizer): self._optimizer = optimizer # slots if tf.__version__[0] == '1': - unused = tf.Variable([0.0], name='unused', trainable=False, use_resource=True) + unused = tf.Variable([0.0], + name='unused', + trainable=False, + use_resource=True) else: unused = tf.Variable([0.0], name='unused', trainable=False) self._optimizer._create_slots([unused]) @@ -318,21 +326,21 @@ def _create_slots_dynamic(self, var): if slot_name not in self._optimizer._slots[key]: if var.backend_type == 'hbm': slot = DynamicVariable( - dimension=var.dimension, - initializer=self._initial_vals[slot_name], - name='DynamicSlot', - trainable=False, + dimension=var.dimension, + initializer=self._initial_vals[slot_name], + name='DynamicSlot', + trainable=False, ) else: tmp_config = var.config_dict # tmp_initializer = var.initializer_str slot = DynamicVariable( - dimension=var.dimension, - initializer=self._initial_vals[slot_name], - var_type=var.backend_type, - name='DynamicSlot', - trainable=False, - **tmp_config, + dimension=var.dimension, + initializer=self._initial_vals[slot_name], + var_type=var.backend_type, + name='DynamicSlot', + trainable=False, + **tmp_config, ) self._optimizer._slots[key][slot_name] = slot @@ -372,21 +380,21 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): if slot_name not in self._optimizer._slots[key]: if v.backend_type == 'hbm': slot = DynamicVariable( - dimension=v.dimension, - initializer=self._initial_vals[slot_name], - name='DynamicSlot', - trainable=False, + dimension=v.dimension, + initializer=self._initial_vals[slot_name], + name='DynamicSlot', + trainable=False, ) else: tmp_config = v.config_dict # tmp_initializer = v.initializer_str slot = DynamicVariable( - dimension=v.dimension, - initializer=self._initial_vals[slot_name], - var_type=v.backend_type, - name='DynamicSlot', - trainable=False, - **tmp_config, + dimension=v.dimension, + initializer=self._initial_vals[slot_name], + var_type=v.backend_type, + name='DynamicSlot', + trainable=False, + **tmp_config, ) self._optimizer._slots[key][slot_name] = slot @@ -403,7 +411,8 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): # 3. Call tf-optimizer with tf.control_dependencies(to_static_ops): - train_op = self._optimizer.apply_gradients(zip(grad_list, var_list), name=name) + train_op = self._optimizer.apply_gradients( + zip(grad_list, var_list), name=name) # 4. Switch iterations self._optimizer._iterations = iterations @@ -421,6 +430,7 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): class SGD(object): + def __init__(self, lr): self._lr = tf.Variable(lr) @@ -432,6 +442,7 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): train_ops = [] for g, v in grads_and_vars: if g is not None: - scaled_g = ops.IndexedSlices(g.values * self._lr, g.indices, g.dense_shape) + scaled_g = ops.IndexedSlices(g.values * self._lr, g.indices, + g.dense_shape) train_ops.append(v.scatter_sub(scaled_g)) return tf.group(train_ops) diff --git a/easy_rec/python/compat/sync_replicas_optimizer.py b/easy_rec/python/compat/sync_replicas_optimizer.py index d41ad7249..d474165dc 100644 --- a/easy_rec/python/compat/sync_replicas_optimizer.py +++ b/easy_rec/python/compat/sync_replicas_optimizer.py @@ -14,26 +14,23 @@ # ============================================================================== """Synchronize replicas for training.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function from tensorflow.core.framework import types_pb2 -from tensorflow.python.framework import errors_impl, ops +from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import ops +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.util.tf_export import tf_export + from tensorflow.python.ops import ( # NOQA - array_ops, - control_flow_ops, - data_flow_ops, - state_ops, - variable_scope, - variables, + array_ops, control_flow_ops, data_flow_ops, state_ops, variable_scope, + variables, ) -from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import ( # NOQA - optimizer, - queue_runner, - session_manager, - session_run_hook, + optimizer, queue_runner, session_manager, session_run_hook, ) -from tensorflow.python.util.tf_export import tf_export # Please note that the gradients from replicas are averaged instead of summed @@ -140,15 +137,15 @@ class SyncReplicasOptimizer(optimizer.Optimizer): sync_que_id = -1 def __init__( - self, - opt, - replicas_to_aggregate, - total_num_replicas=None, - variable_averages=None, - variables_to_average=None, - use_locking=False, - name='sync_replicas', - **extra_args, + self, + opt, + replicas_to_aggregate, + total_num_replicas=None, + variable_averages=None, + variables_to_average=None, + use_locking=False, + name='sync_replicas', + **extra_args, ): """Construct a sync_replicas optimizer. @@ -176,9 +173,9 @@ def __init__( super(SyncReplicasOptimizer, self).__init__(use_locking, name) logging.info( - 'SyncReplicasV2: replicas_to_aggregate=%s; total_num_replicas=%s', - replicas_to_aggregate, - total_num_replicas, + 'SyncReplicasV2: replicas_to_aggregate=%s; total_num_replicas=%s', + replicas_to_aggregate, + total_num_replicas, ) self._opt = opt self._replicas_to_aggregate = replicas_to_aggregate @@ -258,16 +255,17 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): # Colocating local_step variable prevents it being placed on the PS. with ops.colocate_with(local_anchor): self._local_step = variable_scope.variable( - initial_value=0, - trainable=False, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - dtype=global_step.dtype.base_dtype, - name='sync_rep_local_step', + initial_value=0, + trainable=False, + collections=[ops.GraphKeys.LOCAL_VARIABLES], + dtype=global_step.dtype.base_dtype, + name='sync_rep_local_step', ) self.local_step_init_op = state_ops.assign(self._local_step, global_step) chief_init_ops = [self.local_step_init_op] - self.ready_for_local_init_op = variables.report_uninitialized_variables(variables.global_variables()) + self.ready_for_local_init_op = variables.report_uninitialized_variables( + variables.global_variables()) with ops.name_scope(None, self._name): for grad, var in grads_and_vars: @@ -279,20 +277,25 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): continue elif isinstance(grad, ops.Tensor): grad_accum = data_flow_ops.ConditionalAccumulator( - grad.dtype, - shape=var.get_shape(), - shared_name=var.name + '/grad_accum', + grad.dtype, + shape=var.get_shape(), + shared_name=var.name + '/grad_accum', ) - train_ops.append(grad_accum.apply_grad(grad, local_step=self._local_step)) - aggregated_grad.append(grad_accum.take_grad(self._replicas_to_aggregate)) + train_ops.append( + grad_accum.apply_grad(grad, local_step=self._local_step)) + aggregated_grad.append( + grad_accum.take_grad(self._replicas_to_aggregate)) else: if not isinstance(grad, ops.IndexedSlices): raise ValueError('Unknown grad type!') grad_accum = data_flow_ops.SparseConditionalAccumulator( - grad.dtype, shape=(), shared_name=var.name + '/grad_accum' - ) - train_ops.append(grad_accum.apply_indexed_slices_grad(grad, local_step=self._local_step)) - aggregated_grad.append(grad_accum.take_indexed_slices_grad(self._replicas_to_aggregate)) + grad.dtype, shape=(), shared_name=var.name + '/grad_accum') + train_ops.append( + grad_accum.apply_indexed_slices_grad( + grad, local_step=self._local_step)) + aggregated_grad.append( + grad_accum.take_indexed_slices_grad( + self._replicas_to_aggregate)) self._accumulator_list.append((grad_accum, var.device)) @@ -300,7 +303,8 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): # sync_op will be assigned to the same device as the global step. with ops.device(global_step.device), ops.name_scope(''): - update_op = self._opt.apply_gradients(aggregated_grads_and_vars, global_step) + update_op = self._opt.apply_gradients(aggregated_grads_and_vars, + global_step) def _get_token_qname(): SyncReplicasOptimizer.sync_que_id += 1 @@ -314,25 +318,26 @@ def _get_token_qname(): logging.info('create sync_token_queue[%s]' % token_qname) with ops.device(global_step.device), ops.name_scope(''): sync_token_queue = data_flow_ops.FIFOQueue( - -1, - global_step.dtype.base_dtype, - shapes=(), - name=token_qname, - shared_name=token_qname, + -1, + global_step.dtype.base_dtype, + shapes=(), + name=token_qname, + shared_name=token_qname, ) self._sync_token_queue = sync_token_queue self._is_sync_que_closed = sync_token_queue.is_closed() - self._close_sync_que = sync_token_queue.close(cancel_pending_enqueues=True, name='close_sync_token_queue') + self._close_sync_que = sync_token_queue.close( + cancel_pending_enqueues=True, name='close_sync_token_queue') # dummy_queue is passed to the queue runner. Don't use the real queues # because the queue runner doesn't automatically reopen it once it # closed queues in PS devices. dummy_queue = data_flow_ops.FIFOQueue( - 1, - types_pb2.DT_INT32, - shapes=(), - name='dummy_queue', - shared_name='dummy_queue', + 1, + types_pb2.DT_INT32, + shapes=(), + name='dummy_queue', + shared_name='dummy_queue', ) with ops.device(global_step.device), ops.name_scope(''): @@ -351,11 +356,14 @@ def _get_token_qname(): with ops.control_dependencies([sync_op]), ops.name_scope(''): sync_op = self._variable_averages.apply(self._variables_to_average) - self._chief_queue_runner = queue_runner.QueueRunner(dummy_queue, [sync_op]) - ops.add_to_collection(ops.GraphKeys.QUEUE_RUNNERS, self._chief_queue_runner) + self._chief_queue_runner = queue_runner.QueueRunner( + dummy_queue, [sync_op]) + ops.add_to_collection(ops.GraphKeys.QUEUE_RUNNERS, + self._chief_queue_runner) for accum, dev in self._accumulator_list: with ops.device(dev): - chief_init_ops.append(accum.set_global_step(global_step, name='SetGlobalStep')) + chief_init_ops.append( + accum.set_global_step(global_step, name='SetGlobalStep')) self.chief_init_op = control_flow_ops.group(*(chief_init_ops)) self._gradients_applied = True return train_op @@ -440,15 +448,16 @@ def get_init_tokens_op(self, num_tokens=-1): total_num_replicas. """ if self._gradients_applied is False: - raise ValueError('get_init_tokens_op() should be called after apply_gradients().') + raise ValueError( + 'get_init_tokens_op() should be called after apply_gradients().') tokens_needed = self._replicas_to_aggregate - self._total_num_replicas if num_tokens == -1: num_tokens = self._replicas_to_aggregate elif num_tokens < tokens_needed: raise ValueError( - 'Too few tokens to finish the first step: %d (given) vs %d (needed)' % (num_tokens, tokens_needed) - ) + 'Too few tokens to finish the first step: %d (given) vs %d (needed)' % + (num_tokens, tokens_needed)) if num_tokens > 0: with ops.device(self._global_step.device), ops.name_scope(''): @@ -481,11 +490,14 @@ def __init__(self, sync_optimizer, is_chief, num_tokens): def begin(self): if self._sync_optimizer._gradients_applied is False: # pylint: disable=protected-access - raise ValueError('SyncReplicasOptimizer.apply_gradient should be called before using ' 'the hook.') + raise ValueError( + 'SyncReplicasOptimizer.apply_gradient should be called before using ' + 'the hook.') if self._is_chief: self._local_init_op = self._sync_optimizer.chief_init_op self._ready_for_local_init_op = self._sync_optimizer.ready_for_local_init_op - self._init_tokens_op = self._sync_optimizer.get_init_tokens_op(self._num_tokens) + self._init_tokens_op = self._sync_optimizer.get_init_tokens_op( + self._num_tokens) else: self._local_init_op = self._sync_optimizer.local_step_init_op self._ready_for_local_init_op = self._sync_optimizer.ready_for_local_init_op @@ -494,18 +506,18 @@ def begin(self): def after_create_session(self, session, coord): """Runs SyncReplicasOptimizer initialization ops.""" ( - local_init_success, - msg, + local_init_success, + msg, ) = session_manager._ready( # pylint: disable=protected-access - self._ready_for_local_init_op, - session, - 'Model is not ready for SyncReplicasOptimizer local init.', + self._ready_for_local_init_op, + session, + 'Model is not ready for SyncReplicasOptimizer local init.', ) if not local_init_success: raise RuntimeError( - 'Init operations did not make model ready for SyncReplicasOptimizer ' - 'local_init. Init op: %s, error: %s' % (self._local_init_op.name, msg) - ) + 'Init operations did not make model ready for SyncReplicasOptimizer ' + 'local_init. Init op: %s, error: %s' % + (self._local_init_op.name, msg)) session.run(self._local_init_op) is_closed = session.run(self._sync_optimizer._is_sync_que_closed) assert not is_closed, 'sync_que is closed' diff --git a/easy_rec/python/compat/weight_decay_optimizers.py b/easy_rec/python/compat/weight_decay_optimizers.py index b4e1b96f6..7cff692eb 100755 --- a/easy_rec/python/compat/weight_decay_optimizers.py +++ b/easy_rec/python/compat/weight_decay_optimizers.py @@ -14,20 +14,20 @@ # ============================================================================== """Base class to make optimizers weight decay ready.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function from tensorflow.python.framework import ops -from tensorflow.python.ops import ( # NOQA - array_ops, - control_flow_ops, - resource_variable_ops, - state_ops, -) from tensorflow.python.training import adam from tensorflow.python.training import momentum as momentum_opt from tensorflow.python.training import optimizer from tensorflow.python.util.tf_export import tf_export +from tensorflow.python.ops import ( # NOQA + array_ops, control_flow_ops, resource_variable_ops, state_ops, +) + class DecoupledWeightDecayExtension(object): """This class allows to extend optimizers with decoupled weight decay. @@ -93,16 +93,16 @@ def __init__(self, weight_decay, **kwargs): super(DecoupledWeightDecayExtension, self).__init__(**kwargs) def minimize( - self, - loss, - global_step=None, - var_list=None, - gate_gradients=optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - name=None, - grad_loss=None, - decay_var_list=None, + self, + loss, + global_step=None, + var_list=None, + gate_gradients=optimizer.Optimizer.GATE_OP, + aggregation_method=None, + colocate_gradients_with_ops=False, + name=None, + grad_loss=None, + decay_var_list=None, ): """Add operations to minimize `loss` by updating `var_list` with decay. @@ -135,17 +135,21 @@ def minimize( """ self._decay_var_list = set(decay_var_list) if decay_var_list else False return super(DecoupledWeightDecayExtension, self).minimize( - loss, - global_step=global_step, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - name=name, - grad_loss=grad_loss, + loss, + global_step=global_step, + var_list=var_list, + gate_gradients=gate_gradients, + aggregation_method=aggregation_method, + colocate_gradients_with_ops=colocate_gradients_with_ops, + name=name, + grad_loss=grad_loss, ) - def apply_gradients(self, grads_and_vars, global_step=None, name=None, decay_var_list=None): + def apply_gradients(self, + grads_and_vars, + global_step=None, + name=None, + decay_var_list=None): """Apply gradients to variables and decay the variables. This function is the same as Optimizer.apply_gradients except that it @@ -170,14 +174,14 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None, decay_var """ self._decay_var_list = set(decay_var_list) if decay_var_list else False return super(DecoupledWeightDecayExtension, self).apply_gradients( - grads_and_vars, global_step=global_step, name=name - ) + grads_and_vars, global_step=global_step, name=name) def _prepare(self): weight_decay = self._weight_decay if callable(weight_decay): weight_decay = weight_decay() - self._weight_decay_tensor = ops.convert_to_tensor(weight_decay, name='weight_decay') + self._weight_decay_tensor = ops.convert_to_tensor( + weight_decay, name='weight_decay') # Call the optimizers _prepare function. super(DecoupledWeightDecayExtension, self)._prepare() @@ -200,7 +204,8 @@ def _apply_dense(self, grad, var): def _resource_apply_dense(self, grad, var): with ops.control_dependencies([self._decay_weights_op(var)]): - return super(DecoupledWeightDecayExtension, self)._resource_apply_dense(grad, var) + return super(DecoupledWeightDecayExtension, + self)._resource_apply_dense(grad, var) def _apply_sparse(self, grad, var): scatter_add = state_ops.scatter_add @@ -211,14 +216,16 @@ def _apply_sparse(self, grad, var): def _resource_scatter_add(self, x, i, v, _=None): # last argument allows for one overflow argument, to have the same function # signature as state_ops.scatter_add - with ops.control_dependencies([resource_variable_ops.resource_scatter_add(x.handle, i, v)]): + with ops.control_dependencies( + [resource_variable_ops.resource_scatter_add(x.handle, i, v)]): return x.value() def _resource_apply_sparse(self, grad, var, indices): scatter_add = self._resource_scatter_add decay_op = self._decay_weights_sparse_op(var, indices, scatter_add) with ops.control_dependencies([decay_op]): - return super(DecoupledWeightDecayExtension, self)._resource_apply_sparse(grad, var, indices) + return super(DecoupledWeightDecayExtension, + self)._resource_apply_sparse(grad, var, indices) def extend_with_decoupled_weight_decay(base_optimizer): @@ -259,7 +266,8 @@ def extend_with_decoupled_weight_decay(base_optimizer): and base_optimizer. """ - class OptimizerWithDecoupledWeightDecay(DecoupledWeightDecayExtension, base_optimizer): + class OptimizerWithDecoupledWeightDecay(DecoupledWeightDecayExtension, + base_optimizer): """Base_optimizer with decoupled weight decay. This class computes the update step of `base_optimizer` and @@ -276,14 +284,16 @@ class OptimizerWithDecoupledWeightDecay(DecoupledWeightDecayExtension, base_opti def __init__(self, weight_decay, *args, **kwargs): # super delegation is necessary here # pylint: disable=useless-super-delegation - super(OptimizerWithDecoupledWeightDecay, self).__init__(weight_decay, *args, **kwargs) + super(OptimizerWithDecoupledWeightDecay, + self).__init__(weight_decay, *args, **kwargs) # pylint: enable=useless-super-delegation return OptimizerWithDecoupledWeightDecay @tf_export('contrib.opt.MomentumWOptimizer') -class MomentumWOptimizer(DecoupledWeightDecayExtension, momentum_opt.MomentumOptimizer): +class MomentumWOptimizer(DecoupledWeightDecayExtension, + momentum_opt.MomentumOptimizer): """Optimizer that implements the Momentum algorithm with weight_decay. This is an implementation of the SGDW optimizer described in "Fixing @@ -306,13 +316,13 @@ class MomentumWOptimizer(DecoupledWeightDecayExtension, momentum_opt.MomentumOpt """ def __init__( - self, - weight_decay, - learning_rate, - momentum, - use_locking=False, - name='MomentumW', - use_nesterov=False, + self, + weight_decay, + learning_rate, + momentum, + use_locking=False, + name='MomentumW', + use_nesterov=False, ): """Construct a new MomentumW optimizer. @@ -338,12 +348,12 @@ def __init__( functions. @end_compatibility """ super(MomentumWOptimizer, self).__init__( - weight_decay, - learning_rate=learning_rate, - momentum=momentum, - use_locking=use_locking, - name=name, - use_nesterov=use_nesterov, + weight_decay, + learning_rate=learning_rate, + momentum=momentum, + use_locking=use_locking, + name=name, + use_nesterov=use_nesterov, ) @@ -372,14 +382,14 @@ class AdamWOptimizer(DecoupledWeightDecayExtension, adam.AdamOptimizer): """ def __init__( - self, - weight_decay, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - use_locking=False, - name='AdamW', + self, + weight_decay, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + use_locking=False, + name='AdamW', ): """Construct a new AdamW optimizer. @@ -400,13 +410,13 @@ def __init__( Defaults to "Adam". """ super(AdamWOptimizer, self).__init__( - weight_decay, - learning_rate=learning_rate, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - use_locking=use_locking, - name=name, + weight_decay, + learning_rate=learning_rate, + beta1=beta1, + beta2=beta2, + epsilon=epsilon, + use_locking=use_locking, + name=name, ) @@ -438,14 +448,14 @@ class AdamAsyncWOptimizer(DecoupledWeightDecayExtension, AdamAsyncOptimizer): """ def __init__( - self, - weight_decay, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - use_locking=False, - name='AdamAsyncW', + self, + weight_decay, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + use_locking=False, + name='AdamAsyncW', ): """Construct a new AdamW optimizer. @@ -466,13 +476,13 @@ def __init__( Defaults to "Adam". """ super(AdamAsyncWOptimizer, self).__init__( - weight_decay, - learning_rate=learning_rate, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - use_locking=use_locking, - name=name, + weight_decay, + learning_rate=learning_rate, + beta1=beta1, + beta2=beta2, + epsilon=epsilon, + use_locking=use_locking, + name=name, ) except ImportError: diff --git a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py index 0b9ee1d91..ef6e10f86 100644 --- a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py +++ b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_pai.py @@ -14,23 +14,25 @@ # ============================================================================== """Implementation of tf.metrics module.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes, ops, sparse_tensor -from tensorflow.python.ops import ( # NOQA - array_ops, - check_ops, - confusion_matrix, - control_flow_ops, - math_ops, - nn, - sets, - sparse_ops, - state_ops, - variable_scope, - weights_broadcast_ops, -) +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import check_ops +from tensorflow.python.ops import confusion_matrix +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import sets +from tensorflow.python.ops import sparse_ops +from tensorflow.python.ops import state_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import weights_broadcast_ops from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import distribution_strategy_context from tensorflow.python.util.deprecation import deprecated @@ -74,13 +76,14 @@ def metric_variable(shape, dtype, validate_shape=True, name=None): """ # Note that synchronization "ON_READ" implies trainable=False. return variable_scope.variable( - lambda: array_ops.zeros(shape, dtype), - collections=[ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES], - validate_shape=validate_shape, - synchronization=variable_scope.VariableSynchronization.ON_READ, - aggregation=variable_scope.VariableAggregation.SUM, - name=name, - ) + lambda: array_ops.zeros(shape, dtype), + collections=[ + ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES + ], + validate_shape=validate_shape, + synchronization=variable_scope.VariableSynchronization.ON_READ, + aggregation=variable_scope.VariableAggregation.SUM, + name=name) def _remove_squeezable_dimensions(predictions, labels, weights): @@ -108,7 +111,8 @@ def _remove_squeezable_dimensions(predictions, labels, weights): """ predictions = ops.convert_to_tensor(predictions) if labels is not None: - labels, predictions = confusion_matrix.remove_squeezable_dimensions(labels, predictions) + labels, predictions = confusion_matrix.remove_squeezable_dimensions( + labels, predictions) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if weights is None: @@ -135,31 +139,27 @@ def _remove_squeezable_dimensions(predictions, labels, weights): def _maybe_expand_weights(): return control_flow_ops.cond( - math_ops.equal(rank_diff, -1), - lambda: array_ops.expand_dims(weights, [-1]), - lambda: weights, - ) + math_ops.equal(rank_diff, -1), + lambda: array_ops.expand_dims(weights, [-1]), lambda: weights) # Don't attempt squeeze if it will fail based on static check. - if (weights_rank is not None) and (not weights_shape.dims[-1].is_compatible_with(1)): + if ((weights_rank is not None) and + (not weights_shape.dims[-1].is_compatible_with(1))): maybe_squeeze_weights = lambda: weights # noqa: E731 else: - maybe_squeeze_weights = lambda: array_ops.squeeze(weights, [-1]) # noqa: E731 # noqa: E126 + maybe_squeeze_weights = lambda: array_ops.squeeze( # noqa: E731 + weights, [-1]) # noqa: E126 def _maybe_adjust_weights(): return control_flow_ops.cond( - math_ops.equal(rank_diff, 1), - maybe_squeeze_weights, - _maybe_expand_weights, - ) + math_ops.equal(rank_diff, 1), maybe_squeeze_weights, + _maybe_expand_weights) # If weights are scalar, do nothing. Otherwise, try to add or remove a # dimension to match predictions. weights = control_flow_ops.cond( - math_ops.equal(weights_rank_tensor, 0), - lambda: weights, - _maybe_adjust_weights, - ) + math_ops.equal(weights_rank_tensor, 0), lambda: weights, + _maybe_adjust_weights) return predictions, labels, weights @@ -185,14 +185,14 @@ def _maybe_expand_labels(labels, predictions): # If sparse, expand sparse shape. if isinstance(labels, sparse_tensor.SparseTensor): return control_flow_ops.cond( - math_ops.equal(array_ops.rank(predictions), array_ops.size(labels.dense_shape) + 1), - lambda: sparse_ops.sparse_reshape( # pylint: disable=g-long-lambda - labels, - shape=array_ops.concat((labels.dense_shape, (1,)), 0), - name=scope, - ), - lambda: labels, - ) + math_ops.equal( + array_ops.rank(predictions), + array_ops.size(labels.dense_shape) + 1), + lambda: sparse_ops.sparse_reshape( # pylint: disable=g-long-lambda + labels, + shape=array_ops.concat((labels.dense_shape, (1,)), 0), + name=scope), + lambda: labels) # Otherwise, try to use static shape. labels_rank = labels.get_shape().ndims @@ -204,15 +204,14 @@ def _maybe_expand_labels(labels, predictions): if predictions_rank == labels_rank + 1: return array_ops.expand_dims(labels, -1, name=scope) raise ValueError( - 'Unexpected labels shape %s for predictions shape %s.' % (labels.get_shape(), predictions.get_shape()) - ) + 'Unexpected labels shape %s for predictions shape %s.' % + (labels.get_shape(), predictions.get_shape())) # Otherwise, use dynamic shape. return control_flow_ops.cond( - math_ops.equal(array_ops.rank(predictions), array_ops.rank(labels) + 1), - lambda: array_ops.expand_dims(labels, -1, name=scope), - lambda: labels, - ) + math_ops.equal(array_ops.rank(predictions), + array_ops.rank(labels) + 1), + lambda: array_ops.expand_dims(labels, -1, name=scope), lambda: labels) def _safe_div(numerator, denominator, name): @@ -247,11 +246,11 @@ def _safe_scalar_div(numerator, denominator, name): numerator.get_shape().with_rank_at_most(1) denominator.get_shape().with_rank_at_most(1) return control_flow_ops.cond( - math_ops.equal(array_ops.constant(0.0, dtype=dtypes.float64), denominator), - lambda: array_ops.constant(0.0, dtype=dtypes.float64), - lambda: math_ops.div(numerator, denominator), - name=name, - ) + math_ops.equal( + array_ops.constant(0.0, dtype=dtypes.float64), denominator), + lambda: array_ops.constant(0.0, dtype=dtypes.float64), + lambda: math_ops.div(numerator, denominator), + name=name) def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): @@ -278,7 +277,9 @@ def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): update_op: An operation that increments the confusion matrix. """ # Local variable to accumulate the predictions in the confusion matrix. - total_cm = metric_variable([num_classes, num_classes], dtypes.float64, name='total_confusion_matrix') + total_cm = metric_variable([num_classes, num_classes], + dtypes.float64, + name='total_confusion_matrix') # Cast the type to int64 required by confusion_matrix_ops. predictions = math_ops.to_int64(predictions) @@ -297,8 +298,7 @@ def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): # Accumulate the prediction to current confusion matrix. current_cm = confusion_matrix.confusion_matrix( - labels, predictions, num_classes, weights=weights, dtype=dtypes.float64 - ) + labels, predictions, num_classes, weights=weights, dtype=dtypes.float64) update_op = state_ops.assign_add(total_cm, current_cm, use_locking=True) return total_cm, update_op @@ -339,7 +339,11 @@ def fn(distribution, *a): @tf_export('metrics.mean') -def mean(values, weights=None, metrics_collections=None, updates_collections=None, name=None): +def mean(values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the (weighted) mean of the given values. The `mean` function creates two local variables, `total` and `count` @@ -378,7 +382,8 @@ def mean(values, weights=None, metrics_collections=None, updates_collections=Non RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean is not supported when eager execution ' 'is enabled.') + raise RuntimeError('tf.metrics.mean is not supported when eager execution ' + 'is enabled.') with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.to_float(values) @@ -389,18 +394,23 @@ def mean(values, weights=None, metrics_collections=None, updates_collections=Non if weights is None: num_values = math_ops.to_float(array_ops.size(values)) else: - values, _, weights = _remove_squeezable_dimensions(predictions=values, labels=None, weights=weights) - weights = weights_broadcast_ops.broadcast_weights(math_ops.to_float(weights), values) + values, _, weights = _remove_squeezable_dimensions( + predictions=values, labels=None, weights=weights) + weights = weights_broadcast_ops.broadcast_weights( + math_ops.to_float(weights), values) values = math_ops.multiply(values, weights) num_values = math_ops.reduce_sum(weights) - update_total_op = state_ops.assign_add(total, math_ops.reduce_sum(values), use_locking=True) + update_total_op = state_ops.assign_add( + total, math_ops.reduce_sum(values), use_locking=True) with ops.control_dependencies([values]): - update_count_op = state_ops.assign_add(count, num_values, use_locking=True) + update_count_op = state_ops.assign_add( + count, num_values, use_locking=True) compute_mean = lambda _, t, c: _safe_div(t, c, 'value') # noqa: E731 - mean_t = _aggregate_across_towers(metrics_collections, compute_mean, total, count) + mean_t = _aggregate_across_towers(metrics_collections, compute_mean, total, + count) update_op = _safe_div(update_total_op, update_count_op, 'update_op') if updates_collections: @@ -410,14 +420,12 @@ def mean(values, weights=None, metrics_collections=None, updates_collections=Non @tf_export('metrics.accuracy') -def accuracy( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def accuracy(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Calculates how often `predictions` matches `labels`. The `accuracy` function creates two local variables, `total` and @@ -462,23 +470,24 @@ def accuracy( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.accuracy is not supported when eager ' 'execution is enabled.') + raise RuntimeError('tf.metrics.accuracy is not supported when eager ' + 'execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if labels.dtype != predictions.dtype: predictions = math_ops.cast(predictions, labels.dtype) is_correct = math_ops.to_float(math_ops.equal(predictions, labels)) - return mean( - is_correct, - weights, - metrics_collections, - updates_collections, - name or 'accuracy', - ) + return mean(is_correct, weights, metrics_collections, updates_collections, + name or 'accuracy') -def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=None, includes=None): +def _confusion_matrix_at_thresholds(labels, + predictions, + thresholds, + weights=None, + includes=None): """Computes true_positives, false_negatives, true_negatives, false_positives. This function creates up to four local variables, `true_positives`, @@ -529,31 +538,27 @@ def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=Non if include not in all_includes: raise ValueError('Invalid key: %s.' % include) - with ops.control_dependencies( - [ + with ops.control_dependencies([ check_ops.assert_greater_equal( - predictions, - math_ops.cast(0.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]', - ), + predictions, + math_ops.cast(0.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]'), check_ops.assert_less_equal( - predictions, - math_ops.cast(1.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]', - ), - ] - ): + predictions, + math_ops.cast(1.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]') + ]): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.to_float(predictions), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) + predictions=math_ops.to_float(predictions), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) num_thresholds = len(thresholds) # Reshape predictions and labels. predictions_2d = array_ops.reshape(predictions, [-1, 1]) - labels_2d = array_ops.reshape(math_ops.cast(labels, dtype=dtypes.bool), [1, -1]) + labels_2d = array_ops.reshape( + math_ops.cast(labels, dtype=dtypes.bool), [1, -1]) # Use static shape if known. num_predictions = predictions_2d.get_shape().as_list()[0] @@ -562,15 +567,13 @@ def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=Non if num_predictions is None: num_predictions = array_ops.shape(predictions_2d)[0] thresh_tiled = array_ops.tile( - array_ops.expand_dims(array_ops.constant(thresholds), [1]), - array_ops.stack([1, num_predictions]), - ) + array_ops.expand_dims(array_ops.constant(thresholds), [1]), + array_ops.stack([1, num_predictions])) # Tile the predictions after thresholding them across different thresholds. pred_is_pos = math_ops.greater( - array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]), - thresh_tiled, - ) + array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]), + thresh_tiled) if ('fn' in includes) or ('tn' in includes): pred_is_neg = math_ops.logical_not(pred_is_pos) @@ -580,9 +583,12 @@ def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=Non label_is_neg = math_ops.logical_not(label_is_pos) if weights is not None: - weights = weights_broadcast_ops.broadcast_weights(math_ops.to_float(weights), predictions) - weights_tiled = array_ops.tile(array_ops.reshape(weights, [1, -1]), [num_thresholds, 1]) - thresh_tiled.get_shape().assert_is_compatible_with(weights_tiled.get_shape()) + weights = weights_broadcast_ops.broadcast_weights( + math_ops.to_float(weights), predictions) + weights_tiled = array_ops.tile( + array_ops.reshape(weights, [1, -1]), [num_thresholds, 1]) + thresh_tiled.get_shape().assert_is_compatible_with( + weights_tiled.get_shape()) else: weights_tiled = None @@ -590,35 +596,51 @@ def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=Non update_ops = {} if 'tp' in includes: - true_p = metric_variable([num_thresholds], dtypes.float32, name='true_positives') - is_true_positive = math_ops.cast(math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32) + true_p = metric_variable([num_thresholds], + dtypes.float32, + name='true_positives') + is_true_positive = math_ops.cast( + math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32) if weights_tiled is not None: is_true_positive *= weights_tiled - update_ops['tp'] = state_ops.assign_add(true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True) + update_ops['tp'] = state_ops.assign_add( + true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True) values['tp'] = true_p if 'fn' in includes: - false_n = metric_variable([num_thresholds], dtypes.float32, name='false_negatives') - is_false_negative = math_ops.cast(math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32) + false_n = metric_variable([num_thresholds], + dtypes.float32, + name='false_negatives') + is_false_negative = math_ops.cast( + math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32) if weights_tiled is not None: is_false_negative *= weights_tiled - update_ops['fn'] = state_ops.assign_add(false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True) + update_ops['fn'] = state_ops.assign_add( + false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True) values['fn'] = false_n if 'tn' in includes: - true_n = metric_variable([num_thresholds], dtypes.float32, name='true_negatives') - is_true_negative = math_ops.cast(math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32) + true_n = metric_variable([num_thresholds], + dtypes.float32, + name='true_negatives') + is_true_negative = math_ops.cast( + math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32) if weights_tiled is not None: is_true_negative *= weights_tiled - update_ops['tn'] = state_ops.assign_add(true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True) + update_ops['tn'] = state_ops.assign_add( + true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True) values['tn'] = true_n if 'fp' in includes: - false_p = metric_variable([num_thresholds], dtypes.float32, name='false_positives') - is_false_positive = math_ops.cast(math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32) + false_p = metric_variable([num_thresholds], + dtypes.float32, + name='false_positives') + is_false_positive = math_ops.cast( + math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32) if weights_tiled is not None: is_false_positive *= weights_tiled - update_ops['fp'] = state_ops.assign_add(false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True) + update_ops['fp'] = state_ops.assign_add( + false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True) values['fp'] = false_p return values, update_ops @@ -630,17 +652,15 @@ def _aggregate_variable(v, collections): @tf_export('metrics.auc') -def auc( - labels, - predictions, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - curve='ROC', - name=None, - summation_method='trapezoidal', -): +def auc(labels, + predictions, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + curve='ROC', + name=None, + summation_method='trapezoidal'): """Computes the approximate AUC via a Riemann sum. The `auc` function creates four local variables, `true_positives`, @@ -713,16 +733,21 @@ def auc( print('use_distribute_pai_auc') logging.info('use_distribute_pai_auc') if context.executing_eagerly(): - raise RuntimeError('tf.metrics.auc is not supported when eager execution ' 'is enabled.') + raise RuntimeError('tf.metrics.auc is not supported when eager execution ' + 'is enabled.') - with variable_scope.variable_scope(name, 'auc', (labels, predictions, weights)): + with variable_scope.variable_scope(name, 'auc', + (labels, predictions, weights)): if curve != 'ROC' and curve != 'PR': raise ValueError('curve must be either ROC or PR, %s unknown' % (curve)) kepsilon = 1e-7 # to account for floating point imprecisions - thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] + thresholds = [ + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] - values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) + values, update_ops = _confusion_matrix_at_thresholds( + labels, predictions, thresholds, weights) # Add epsilons to avoid dividing by 0. epsilon = 1.0e-6 @@ -758,32 +783,28 @@ def interpolate_pr_auc(tp, fp, fn): Returns: pr_auc: an approximation of the area under the P-R curve. """ - dtp = tp[: num_thresholds - 1] - tp[1:] + dtp = tp[:num_thresholds - 1] - tp[1:] p = tp + fp - prec_slope = _safe_div(dtp, p[: num_thresholds - 1] - p[1:], 'prec_slope') + prec_slope = _safe_div(dtp, p[:num_thresholds - 1] - p[1:], 'prec_slope') intercept = tp[1:] - math_ops.multiply(prec_slope, p[1:]) safe_p_ratio = array_ops.where( - math_ops.logical_and(p[: num_thresholds - 1] > 0, p[1:] > 0), - _safe_div(p[: num_thresholds - 1], p[1:], 'recall_relative_ratio'), - array_ops.ones_like(p[1:]), - ) + math_ops.logical_and(p[:num_thresholds - 1] > 0, p[1:] > 0), + _safe_div(p[:num_thresholds - 1], p[1:], 'recall_relative_ratio'), + array_ops.ones_like(p[1:])) return math_ops.reduce_sum( - _safe_div( - prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)), - tp[1:] + fn[1:], - name='pr_auc_increment', - ), - name='interpolate_pr_auc', - ) + _safe_div( + prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)), + tp[1:] + fn[1:], + name='pr_auc_increment'), + name='interpolate_pr_auc') def compute_auc(tp, fn, tn, fp, name): """Computes the roc-auc or pr-auc based on confusion counts.""" if curve == 'PR': if summation_method == 'trapezoidal': logging.warning( - 'Trapezoidal rule is known to produce incorrect PR-AUCs; ' - 'please switch to "careful_interpolation" instead.' - ) + 'Trapezoidal rule is known to produce incorrect PR-AUCs; ' + 'please switch to "careful_interpolation" instead.') elif summation_method == 'careful_interpolation': # This one is a bit tricky and is handled separately. return interpolate_pr_auc(tp, fp, fn) @@ -800,43 +821,31 @@ def compute_auc(tp, fn, tn, fp, name): # Note that the case ('PR', 'careful_interpolation') has been handled # above. return math_ops.reduce_sum( - math_ops.multiply( - x[: num_thresholds - 1] - x[1:], - (y[: num_thresholds - 1] + y[1:]) / 2.0, - ), - name=name, - ) + math_ops.multiply(x[:num_thresholds - 1] - x[1:], + (y[:num_thresholds - 1] + y[1:]) / 2.), + name=name) elif summation_method == 'minoring': return math_ops.reduce_sum( - math_ops.multiply( - x[: num_thresholds - 1] - x[1:], - math_ops.minimum(y[: num_thresholds - 1], y[1:]), - ), - name=name, - ) + math_ops.multiply(x[:num_thresholds - 1] - x[1:], + math_ops.minimum(y[:num_thresholds - 1], y[1:])), + name=name) elif summation_method == 'majoring': return math_ops.reduce_sum( - math_ops.multiply( - x[: num_thresholds - 1] - x[1:], - math_ops.maximum(y[: num_thresholds - 1], y[1:]), - ), - name=name, - ) + math_ops.multiply(x[:num_thresholds - 1] - x[1:], + math_ops.maximum(y[:num_thresholds - 1], y[1:])), + name=name) else: raise ValueError('Invalid summation_method: %s' % summation_method) # sum up the areas of all the trapeziums def compute_auc_value(_, values): - return compute_auc(values['tp'], values['fn'], values['tn'], values['fp'], 'value') + return compute_auc(values['tp'], values['fn'], values['tn'], values['fp'], + 'value') - auc_value = _aggregate_across_towers(metrics_collections, compute_auc_value, values) - update_op = compute_auc( - update_ops['tp'], - update_ops['fn'], - update_ops['tn'], - update_ops['fp'], - 'update_op', - ) + auc_value = _aggregate_across_towers(metrics_collections, compute_auc_value, + values) + update_op = compute_auc(update_ops['tp'], update_ops['fn'], + update_ops['tn'], update_ops['fp'], 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -845,14 +854,12 @@ def compute_auc_value(_, values): @tf_export('metrics.mean_absolute_error') -def mean_absolute_error( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_absolute_error(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the mean absolute error between the labels and predictions. The `mean_absolute_error` function creates two local variables, @@ -897,29 +904,24 @@ def mean_absolute_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_absolute_error is not supported ' 'when eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_absolute_error is not supported ' + 'when eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) absolute_errors = math_ops.abs(predictions - labels) - return mean( - absolute_errors, - weights, - metrics_collections, - updates_collections, - name or 'mean_absolute_error', - ) + return mean(absolute_errors, weights, metrics_collections, + updates_collections, name or 'mean_absolute_error') @tf_export('metrics.mean_cosine_distance') -def mean_cosine_distance( - labels, - predictions, - dim, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_cosine_distance(labels, + predictions, + dim, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the cosine distance between the labels and predictions. The `mean_cosine_distance` function creates two local variables, @@ -962,18 +964,18 @@ def mean_cosine_distance( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_cosine_distance is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_cosine_distance is not supported when ' + 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) radial_diffs = math_ops.multiply(predictions, labels) radial_diffs = math_ops.reduce_sum( - radial_diffs, - reduction_indices=[ - dim, - ], - keepdims=True, - ) - mean_distance, update_op = mean(radial_diffs, weights, None, None, name or 'mean_cosine_distance') + radial_diffs, reduction_indices=[ + dim, + ], keepdims=True) + mean_distance, update_op = mean(radial_diffs, weights, None, None, name or + 'mean_cosine_distance') mean_distance = math_ops.subtract(1.0, mean_distance) update_op = math_ops.subtract(1.0, update_op) @@ -987,15 +989,13 @@ def mean_cosine_distance( @tf_export('metrics.mean_per_class_accuracy') -def mean_per_class_accuracy( - labels, - predictions, - num_classes, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_per_class_accuracy(labels, + predictions, + num_classes, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Calculates the mean of the per-class accuracies. Calculates the accuracy for each class, then takes the mean of that. @@ -1037,9 +1037,11 @@ def mean_per_class_accuracy( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_per_class_accuracy is not supported ' 'when eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_per_class_accuracy is not supported ' + 'when eager execution is enabled.') - with variable_scope.variable_scope(name, 'mean_accuracy', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'mean_accuracy', + (predictions, labels, weights)): labels = math_ops.to_int64(labels) # Flatten the input if its rank > 1. @@ -1069,15 +1071,20 @@ def mean_per_class_accuracy( is_correct *= weights ones *= weights - update_total_op = state_ops.scatter_add(total, labels, ones, use_locking=True) - update_count_op = state_ops.scatter_add(count, labels, is_correct, use_locking=True) + update_total_op = state_ops.scatter_add( + total, labels, ones, use_locking=True) + update_count_op = state_ops.scatter_add( + count, labels, is_correct, use_locking=True) def compute_mean_accuracy(_, count, total): per_class_accuracy = _safe_div(count, total, None) - mean_accuracy_v = math_ops.reduce_mean(per_class_accuracy, name='mean_accuracy') + mean_accuracy_v = math_ops.reduce_mean( + per_class_accuracy, name='mean_accuracy') return mean_accuracy_v - mean_accuracy_v = _aggregate_across_towers(metrics_collections, compute_mean_accuracy, count, total) + mean_accuracy_v = _aggregate_across_towers(metrics_collections, + compute_mean_accuracy, count, + total) update_op = _safe_div(update_count_op, update_total_op, name='update_op') if updates_collections: @@ -1087,15 +1094,13 @@ def compute_mean_accuracy(_, count, total): @tf_export('metrics.mean_iou') -def mean_iou( - labels, - predictions, - num_classes, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_iou(labels, + predictions, + num_classes, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Calculate per-step mean Intersection-Over-Union (mIOU). Mean Intersection-Over-Union is a common evaluation metric for @@ -1141,13 +1146,16 @@ def mean_iou( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_iou is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_iou is not supported when ' + 'eager execution is enabled.') - with variable_scope.variable_scope(name, 'mean_iou', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'mean_iou', + (predictions, labels, weights)): # Check if shape is compatible. predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - total_cm, update_op = _streaming_confusion_matrix(labels, predictions, num_classes, weights) + total_cm, update_op = _streaming_confusion_matrix(labels, predictions, + num_classes, weights) def compute_mean_iou(_, total_cm): """Compute the mean intersection-over-union via the confusion matrix.""" @@ -1159,27 +1167,26 @@ def compute_mean_iou(_, total_cm): # The mean is only computed over classes that appear in the # label or prediction tensor. If the denominator is 0, we need to # ignore the class. - num_valid_entries = math_ops.reduce_sum(math_ops.cast(math_ops.not_equal(denominator, 0), dtype=dtypes.float32)) + num_valid_entries = math_ops.reduce_sum( + math_ops.cast( + math_ops.not_equal(denominator, 0), dtype=dtypes.float32)) # If the value of the denominator is 0, set it to 1 to avoid # zero division. denominator = array_ops.where( - math_ops.greater(denominator, 0), - denominator, - array_ops.ones_like(denominator), - ) + math_ops.greater(denominator, 0), denominator, + array_ops.ones_like(denominator)) iou = math_ops.div(cm_diag, denominator) # If the number of valid entries is 0 (no classes) we return 0. result = array_ops.where( - math_ops.greater(num_valid_entries, 0), - math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, - 0, - ) + math_ops.greater(num_valid_entries, 0), + math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, 0) return result # TODO(priyag): Use outside_compilation if in TPU context. - mean_iou_v = _aggregate_across_towers(metrics_collections, compute_mean_iou, total_cm) + mean_iou_v = _aggregate_across_towers(metrics_collections, compute_mean_iou, + total_cm) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1188,15 +1195,13 @@ def compute_mean_iou(_, total_cm): @tf_export('metrics.mean_relative_error') -def mean_relative_error( - labels, - predictions, - normalizer, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_relative_error(labels, + predictions, + normalizer, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the mean relative error by normalizing with the given values. The `mean_relative_error` function creates two local variables, @@ -1242,35 +1247,29 @@ def mean_relative_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_relative_error is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_relative_error is not supported when ' + 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) - predictions, normalizer = confusion_matrix.remove_squeezable_dimensions(predictions, normalizer) + predictions, normalizer = confusion_matrix.remove_squeezable_dimensions( + predictions, normalizer) predictions.get_shape().assert_is_compatible_with(normalizer.get_shape()) relative_errors = array_ops.where( - math_ops.equal(normalizer, 0.0), - array_ops.zeros_like(labels), - math_ops.div(math_ops.abs(labels - predictions), normalizer), - ) - return mean( - relative_errors, - weights, - metrics_collections, - updates_collections, - name or 'mean_relative_error', - ) + math_ops.equal(normalizer, 0.0), array_ops.zeros_like(labels), + math_ops.div(math_ops.abs(labels - predictions), normalizer)) + return mean(relative_errors, weights, metrics_collections, + updates_collections, name or 'mean_relative_error') @tf_export('metrics.mean_squared_error') -def mean_squared_error( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_squared_error(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the mean squared error between the labels and predictions. The `mean_squared_error` function creates two local variables, @@ -1315,21 +1314,22 @@ def mean_squared_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_squared_error is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_squared_error is not supported when ' + 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) squared_error = math_ops.square(labels - predictions) - return mean( - squared_error, - weights, - metrics_collections, - updates_collections, - name or 'mean_squared_error', - ) + return mean(squared_error, weights, metrics_collections, updates_collections, + name or 'mean_squared_error') @tf_export('metrics.mean_tensor') -def mean_tensor(values, weights=None, metrics_collections=None, updates_collections=None, name=None): +def mean_tensor(values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the element-wise (weighted) mean of the given tensors. In contrast to the `mean` function which returns a scalar with the @@ -1372,27 +1372,34 @@ def mean_tensor(values, weights=None, metrics_collections=None, updates_collecti RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_tensor is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_tensor is not supported when ' + 'eager execution is enabled.') with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.to_float(values) - total = metric_variable(values.get_shape(), dtypes.float32, name='total_tensor') - count = metric_variable(values.get_shape(), dtypes.float32, name='count_tensor') + total = metric_variable( + values.get_shape(), dtypes.float32, name='total_tensor') + count = metric_variable( + values.get_shape(), dtypes.float32, name='count_tensor') num_values = array_ops.ones_like(values) if weights is not None: - values, _, weights = _remove_squeezable_dimensions(predictions=values, labels=None, weights=weights) - weights = weights_broadcast_ops.broadcast_weights(math_ops.to_float(weights), values) + values, _, weights = _remove_squeezable_dimensions( + predictions=values, labels=None, weights=weights) + weights = weights_broadcast_ops.broadcast_weights( + math_ops.to_float(weights), values) values = math_ops.multiply(values, weights) num_values = math_ops.multiply(num_values, weights) update_total_op = state_ops.assign_add(total, values, use_locking=True) with ops.control_dependencies([values]): - update_count_op = state_ops.assign_add(count, num_values, use_locking=True) + update_count_op = state_ops.assign_add( + count, num_values, use_locking=True) compute_mean = lambda _, t, c: _safe_div(t, c, 'value') # noqa: E731 - mean_t = _aggregate_across_towers(metrics_collections, compute_mean, total, count) + mean_t = _aggregate_across_towers(metrics_collections, compute_mean, total, + count) update_op = _safe_div(update_total_op, update_count_op, 'update_op') if updates_collections: @@ -1402,14 +1409,12 @@ def mean_tensor(values, weights=None, metrics_collections=None, updates_collecti @tf_export('metrics.percentage_below') -def percentage_below( - values, - threshold, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def percentage_below(values, + threshold, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the percentage of values less than the given threshold. The `percentage_below` function creates two local variables, @@ -1449,19 +1454,18 @@ def percentage_below( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.percentage_below is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.percentage_below is not supported when ' + 'eager execution is enabled.') is_below_threshold = math_ops.to_float(math_ops.less(values, threshold)) - return mean( - is_below_threshold, - weights, - metrics_collections, - updates_collections, - name or 'percentage_below_threshold', - ) + return mean(is_below_threshold, weights, metrics_collections, + updates_collections, name or 'percentage_below_threshold') -def _count_condition(values, weights=None, metrics_collections=None, updates_collections=None): +def _count_condition(values, + weights=None, + metrics_collections=None, + updates_collections=None): """Sums the weights of cases where the given values are True. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1490,13 +1494,15 @@ def _count_condition(values, weights=None, metrics_collections=None, updates_col values = math_ops.to_float(values) if weights is not None: - with ops.control_dependencies((check_ops.assert_rank_in(weights, (0, array_ops.rank(values))),)): + with ops.control_dependencies( + (check_ops.assert_rank_in(weights, (0, array_ops.rank(values))),)): weights = math_ops.to_float(weights) values = math_ops.multiply(values, weights) value_tensor = _aggregate_variable(count, metrics_collections) - update_op = state_ops.assign_add(count, math_ops.reduce_sum(values), use_locking=True) + update_op = state_ops.assign_add( + count, math_ops.reduce_sum(values), use_locking=True) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1504,14 +1510,12 @@ def _count_condition(values, weights=None, metrics_collections=None, updates_col @tf_export('metrics.false_negatives') -def false_negatives( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def false_negatives(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the total number of false negatives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1541,28 +1545,30 @@ def false_negatives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_negatives is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.false_negatives is not supported when ' + 'eager execution is enabled.') + + with variable_scope.variable_scope(name, 'false_negatives', + (predictions, labels, weights)): - with variable_scope.variable_scope(name, 'false_negatives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) - is_false_negative = math_ops.logical_and(math_ops.equal(labels, True), math_ops.equal(predictions, False)) - return _count_condition(is_false_negative, weights, metrics_collections, updates_collections) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) + is_false_negative = math_ops.logical_and( + math_ops.equal(labels, True), math_ops.equal(predictions, False)) + return _count_condition(is_false_negative, weights, metrics_collections, + updates_collections) @tf_export('metrics.false_negatives_at_thresholds') -def false_negatives_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def false_negatives_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes false negatives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1595,12 +1601,13 @@ def false_negatives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_negatives_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.false_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'false_negatives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'false_negatives', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fn',) - ) + labels, predictions, thresholds, weights=weights, includes=('fn',)) fn_value = _aggregate_variable(values['fn'], metrics_collections) @@ -1611,14 +1618,12 @@ def false_negatives_at_thresholds( @tf_export('metrics.false_positives') -def false_positives( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def false_positives(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Sum the weights of false positives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1649,28 +1654,30 @@ def false_positives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_positives is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.false_positives is not supported when ' + 'eager execution is enabled.') + + with variable_scope.variable_scope(name, 'false_positives', + (predictions, labels, weights)): - with variable_scope.variable_scope(name, 'false_positives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) - is_false_positive = math_ops.logical_and(math_ops.equal(labels, False), math_ops.equal(predictions, True)) - return _count_condition(is_false_positive, weights, metrics_collections, updates_collections) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) + is_false_positive = math_ops.logical_and( + math_ops.equal(labels, False), math_ops.equal(predictions, True)) + return _count_condition(is_false_positive, weights, metrics_collections, + updates_collections) @tf_export('metrics.false_positives_at_thresholds') -def false_positives_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def false_positives_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes false positives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1703,12 +1710,13 @@ def false_positives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_positives_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.false_positives_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'false_positives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'false_positives', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fp',) - ) + labels, predictions, thresholds, weights=weights, includes=('fp',)) fp_value = _aggregate_variable(values['fp'], metrics_collections) @@ -1719,14 +1727,12 @@ def false_positives_at_thresholds( @tf_export('metrics.true_negatives') -def true_negatives( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def true_negatives(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Sum the weights of true_negatives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1757,28 +1763,30 @@ def true_negatives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_negatives is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.true_negatives is not ' + 'supported when eager execution is enabled.') + + with variable_scope.variable_scope(name, 'true_negatives', + (predictions, labels, weights)): - with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) - is_true_negative = math_ops.logical_and(math_ops.equal(labels, False), math_ops.equal(predictions, False)) - return _count_condition(is_true_negative, weights, metrics_collections, updates_collections) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) + is_true_negative = math_ops.logical_and( + math_ops.equal(labels, False), math_ops.equal(predictions, False)) + return _count_condition(is_true_negative, weights, metrics_collections, + updates_collections) @tf_export('metrics.true_negatives_at_thresholds') -def true_negatives_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def true_negatives_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes true negatives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1811,12 +1819,13 @@ def true_negatives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_negatives_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.true_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'true_negatives', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tn',) - ) + labels, predictions, thresholds, weights=weights, includes=('tn',)) tn_value = _aggregate_variable(values['tn'], metrics_collections) @@ -1827,14 +1836,12 @@ def true_negatives_at_thresholds( @tf_export('metrics.true_positives') -def true_positives( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def true_positives(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Sum the weights of true_positives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1865,28 +1872,30 @@ def true_positives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_positives is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.true_positives is not ' + 'supported when eager execution is enabled.') + + with variable_scope.variable_scope(name, 'true_positives', + (predictions, labels, weights)): - with variable_scope.variable_scope(name, 'true_positives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) - is_true_positive = math_ops.logical_and(math_ops.equal(labels, True), math_ops.equal(predictions, True)) - return _count_condition(is_true_positive, weights, metrics_collections, updates_collections) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) + is_true_positive = math_ops.logical_and( + math_ops.equal(labels, True), math_ops.equal(predictions, True)) + return _count_condition(is_true_positive, weights, metrics_collections, + updates_collections) @tf_export('metrics.true_positives_at_thresholds') -def true_positives_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def true_positives_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes true positives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1919,12 +1928,13 @@ def true_positives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_positives_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.true_positives_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'true_positives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'true_positives', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tp',) - ) + labels, predictions, thresholds, weights=weights, includes=('tp',)) tp_value = _aggregate_variable(values['tp'], metrics_collections) @@ -1935,14 +1945,12 @@ def true_positives_at_thresholds( @tf_export('metrics.precision') -def precision( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def precision(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the precision of the predictions with respect to the labels. The `precision` function creates two local variables, @@ -1987,41 +1995,44 @@ def precision( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.precision is not ' + 'supported when eager execution is enabled.') + + with variable_scope.variable_scope(name, 'precision', + (predictions, labels, weights)): - with variable_scope.variable_scope(name, 'precision', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) true_p, true_positives_update_op = true_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None, - ) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) false_p, false_positives_update_op = false_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None, - ) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) def compute_precision(tp, fp, name): - return array_ops.where(math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name) + return array_ops.where( + math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name) def once_across_towers(_, true_p, false_p): return compute_precision(true_p, false_p, 'value') - p = _aggregate_across_towers(metrics_collections, once_across_towers, true_p, false_p) + p = _aggregate_across_towers(metrics_collections, once_across_towers, + true_p, false_p) - update_op = compute_precision(true_positives_update_op, false_positives_update_op, 'update_op') + update_op = compute_precision(true_positives_update_op, + false_positives_update_op, 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2029,15 +2040,13 @@ def once_across_towers(_, true_p, false_p): @tf_export('metrics.precision_at_thresholds') -def precision_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def precision_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes precision values for different `thresholds` on `predictions`. The `precision_at_thresholds` function creates four local variables, @@ -2083,12 +2092,13 @@ def precision_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.precision_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'precision_at_thresholds', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'precision_at_thresholds', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights, includes=('tp', 'fp') - ) + labels, predictions, thresholds, weights, includes=('tp', 'fp')) # Avoid division by zero. epsilon = 1e-7 @@ -2099,9 +2109,11 @@ def compute_precision(tp, fp, name): def precision_across_towers(_, values): return compute_precision(values['tp'], values['fp'], 'value') - prec = _aggregate_across_towers(metrics_collections, precision_across_towers, values) + prec = _aggregate_across_towers(metrics_collections, + precision_across_towers, values) - update_op = compute_precision(update_ops['tp'], update_ops['fp'], 'update_op') + update_op = compute_precision(update_ops['tp'], update_ops['fp'], + 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2109,14 +2121,12 @@ def precision_across_towers(_, values): @tf_export('metrics.recall') -def recall( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def recall(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the recall of the predictions with respect to the labels. The `recall` function creates two local variables, `true_positives` @@ -2159,46 +2169,44 @@ def recall( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall is not supported is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.recall is not supported is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'recall', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'recall', + (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) true_p, true_positives_update_op = true_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None, - ) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) false_n, false_negatives_update_op = false_negatives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None, - ) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) def compute_recall(true_p, false_n, name): return array_ops.where( - math_ops.greater(true_p + false_n, 0), - math_ops.div(true_p, true_p + false_n), - 0, - name, - ) + math_ops.greater(true_p + false_n, 0), + math_ops.div(true_p, true_p + false_n), 0, name) def once_across_towers(_, true_p, false_n): return compute_recall(true_p, false_n, 'value') - rec = _aggregate_across_towers(metrics_collections, once_across_towers, true_p, false_n) + rec = _aggregate_across_towers(metrics_collections, once_across_towers, + true_p, false_n) - update_op = compute_recall(true_positives_update_op, false_negatives_update_op, 'update_op') + update_op = compute_recall(true_positives_update_op, + false_negatives_update_op, 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2228,7 +2236,8 @@ def _select_class_id(ids, selected_id): """ ids = sparse_tensor.convert_to_tensor_or_sparse_tensor(ids) if isinstance(ids, sparse_tensor.SparseTensor): - return sparse_ops.sparse_retain(ids, math_ops.equal(ids.values, selected_id)) + return sparse_ops.sparse_retain(ids, math_ops.equal(ids.values, + selected_id)) # TODO(ptucker): Make this more efficient, maybe add a sparse version of # tf.equal and tf.reduce_any? @@ -2236,12 +2245,15 @@ def _select_class_id(ids, selected_id): # Shape of filled IDs is the same as `ids` with the last dim collapsed to 1. ids_shape = array_ops.shape(ids, out_type=dtypes.int64) ids_last_dim = array_ops.size(ids_shape) - 1 - filled_selected_id_shape = math_ops.reduced_shape(ids_shape, array_ops.reshape(ids_last_dim, [1])) + filled_selected_id_shape = math_ops.reduced_shape( + ids_shape, array_ops.reshape(ids_last_dim, [1])) # Intersect `ids` with the selected ID. - filled_selected_id = array_ops.fill(filled_selected_id_shape, math_ops.to_int64(selected_id)) + filled_selected_id = array_ops.fill(filled_selected_id_shape, + math_ops.to_int64(selected_id)) result = sets.set_intersection(filled_selected_id, ids) - return sparse_tensor.SparseTensor(indices=result.indices, values=result.values, dense_shape=ids_shape) + return sparse_tensor.SparseTensor( + indices=result.indices, values=result.values, dense_shape=ids_shape) def _maybe_select_class_id(labels, predictions_idx, selected_id=None): @@ -2263,13 +2275,15 @@ def _maybe_select_class_id(labels, predictions_idx, selected_id=None): """ if selected_id is None: return labels, predictions_idx - return ( - _select_class_id(labels, selected_id), - _select_class_id(predictions_idx, selected_id), - ) + return (_select_class_id(labels, selected_id), + _select_class_id(predictions_idx, selected_id)) -def _sparse_true_positive_at_k(labels, predictions_idx, class_id=None, weights=None, name=None): +def _sparse_true_positive_at_k(labels, + predictions_idx, + class_id=None, + weights=None, + name=None): """Calculates true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2296,18 +2310,26 @@ def _sparse_true_positive_at_k(labels, predictions_idx, class_id=None, weights=N Returns: A [D1, ... DN] `Tensor` of true positive counts. """ - with ops.name_scope(name, 'true_positives', (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) + with ops.name_scope(name, 'true_positives', + (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, + class_id) tp = sets.set_size(sets.set_intersection(predictions_idx, labels)) tp = math_ops.to_double(tp) if weights is not None: - with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, tp),)): + with ops.control_dependencies( + (weights_broadcast_ops.assert_broadcastable(weights, tp),)): weights = math_ops.to_double(weights) tp = math_ops.multiply(tp, weights) return tp -def _streaming_sparse_true_positive_at_k(labels, predictions_idx, k=None, class_id=None, weights=None, name=None): +def _streaming_sparse_true_positive_at_k(labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + name=None): """Calculates weighted per step true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2340,24 +2362,24 @@ def _streaming_sparse_true_positive_at_k(labels, predictions_idx, k=None, class_ Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope( - name, - _at_k_name('true_positive', k, class_id=class_id), - (predictions_idx, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('true_positive', k, class_id=class_id), + (predictions_idx, labels, weights)) as scope: tp = _sparse_true_positive_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights, - ) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights) batch_total_tp = math_ops.to_double(math_ops.reduce_sum(tp)) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add(var, batch_total_tp, name='update', use_locking=True) + return var, state_ops.assign_add( + var, batch_total_tp, name='update', use_locking=True) -def _sparse_false_negative_at_k(labels, predictions_idx, class_id=None, weights=None): +def _sparse_false_negative_at_k(labels, + predictions_idx, + class_id=None, + weights=None): """Calculates false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2383,18 +2405,27 @@ def _sparse_false_negative_at_k(labels, predictions_idx, class_id=None, weights= Returns: A [D1, ... DN] `Tensor` of false negative counts. """ - with ops.name_scope(None, 'false_negatives', (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) - fn = sets.set_size(sets.set_difference(predictions_idx, labels, aminusb=False)) + with ops.name_scope(None, 'false_negatives', + (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, + class_id) + fn = sets.set_size( + sets.set_difference(predictions_idx, labels, aminusb=False)) fn = math_ops.to_double(fn) if weights is not None: - with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, fn),)): + with ops.control_dependencies( + (weights_broadcast_ops.assert_broadcastable(weights, fn),)): weights = math_ops.to_double(weights) fn = math_ops.multiply(fn, weights) return fn -def _streaming_sparse_false_negative_at_k(labels, predictions_idx, k, class_id=None, weights=None, name=None): +def _streaming_sparse_false_negative_at_k(labels, + predictions_idx, + k, + class_id=None, + weights=None, + name=None): """Calculates weighted per step false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2427,34 +2458,29 @@ def _streaming_sparse_false_negative_at_k(labels, predictions_idx, k, class_id=N Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope( - name, - _at_k_name('false_negative', k, class_id=class_id), - (predictions_idx, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('false_negative', k, class_id=class_id), + (predictions_idx, labels, weights)) as scope: fn = _sparse_false_negative_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights, - ) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights) batch_total_fn = math_ops.to_double(math_ops.reduce_sum(fn)) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add(var, batch_total_fn, name='update', use_locking=True) + return var, state_ops.assign_add( + var, batch_total_fn, name='update', use_locking=True) @tf_export('metrics.recall_at_k') -def recall_at_k( - labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def recall_at_k(labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes recall@k of the predictions with respect to sparse labels. If `class_id` is specified, we calculate recall by considering only the @@ -2522,33 +2548,32 @@ def recall_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall_at_k is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.recall_at_k is not ' + 'supported when eager execution is enabled.') - with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), (predictions, labels, weights)) as scope: + with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), + (predictions, labels, weights)) as scope: _, top_k_idx = nn.top_k(predictions, k) return recall_at_top_k( - labels=labels, - predictions_idx=top_k_idx, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope, - ) + labels=labels, + predictions_idx=top_k_idx, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope) @tf_export('metrics.recall_at_top_k') -def recall_at_top_k( - labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def recall_at_top_k(labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes recall@k of top-k predictions with respect to sparse labels. Differs from `recall_at_k` in that predictions must be in the form of top `k` @@ -2594,49 +2619,44 @@ class indices, whereas `recall_at_k` expects logits. Refer to `recall_at_k` `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with ops.name_scope( - name, - _at_k_name('recall', k, class_id=class_id), - (predictions_idx, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), + (predictions_idx, labels, weights)) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.to_int64(predictions_idx) tp, tp_update = _streaming_sparse_true_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights, - ) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights) fn, fn_update = _streaming_sparse_false_negative_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights, - ) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights) def compute_recall(_, tp, fn): return math_ops.div(tp, math_ops.add(tp, fn), name=scope) - metric = _aggregate_across_towers(metrics_collections, compute_recall, tp, fn) + metric = _aggregate_across_towers(metrics_collections, compute_recall, tp, + fn) - update = math_ops.div(tp_update, math_ops.add(tp_update, fn_update), name='update') + update = math_ops.div( + tp_update, math_ops.add(tp_update, fn_update), name='update') if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @tf_export('metrics.recall_at_thresholds') -def recall_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def recall_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes various recall values for different `thresholds` on `predictions`. The `recall_at_thresholds` function creates four local variables, @@ -2680,12 +2700,13 @@ def recall_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.recall_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'recall_at_thresholds', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'recall_at_thresholds', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights, includes=('tp', 'fn') - ) + labels, predictions, thresholds, weights, includes=('tp', 'fn')) # Avoid division by zero. epsilon = 1e-7 @@ -2696,7 +2717,8 @@ def compute_recall(tp, fn, name): def recall_across_towers(_, values): return compute_recall(values['tp'], values['fn'], 'value') - rec = _aggregate_across_towers(metrics_collections, recall_across_towers, values) + rec = _aggregate_across_towers(metrics_collections, recall_across_towers, + values) update_op = compute_recall(update_ops['tp'], update_ops['fn'], 'update_op') if updates_collections: @@ -2706,14 +2728,12 @@ def recall_across_towers(_, values): @tf_export('metrics.root_mean_squared_error') -def root_mean_squared_error( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def root_mean_squared_error(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the root mean squared error between the labels and predictions. The `root_mean_squared_error` function creates two local variables, @@ -2758,10 +2778,14 @@ def root_mean_squared_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.root_mean_squared_error is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.root_mean_squared_error is not ' + 'supported when eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) - mse, update_mse_op = mean_squared_error(labels, predictions, weights, None, None, name or 'root_mean_squared_error') + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) + mse, update_mse_op = mean_squared_error(labels, predictions, weights, None, + None, name or + 'root_mean_squared_error') once_across_towers = lambda _, mse: math_ops.sqrt(mse) # noqa: E731 rmse = _aggregate_across_towers(metrics_collections, once_across_towers, mse) @@ -2774,16 +2798,14 @@ def root_mean_squared_error( @tf_export('metrics.sensitivity_at_specificity') -def sensitivity_at_specificity( - labels, - predictions, - specificity, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None, -): +def sensitivity_at_specificity(labels, + predictions, + specificity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the specificity at a given sensitivity. The `sensitivity_at_specificity` function creates four local @@ -2835,17 +2857,22 @@ def sensitivity_at_specificity( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sensitivity_at_specificity is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.sensitivity_at_specificity is not ' + 'supported when eager execution is enabled.') if specificity < 0 or specificity > 1: raise ValueError('`specificity` must be in the range [0, 1].') - with variable_scope.variable_scope(name, 'sensitivity_at_specificity', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'sensitivity_at_specificity', + (predictions, labels, weights)): kepsilon = 1e-7 # to account for floating point imprecisions - thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] + thresholds = [ + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] - values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) + values, update_ops = _confusion_matrix_at_thresholds( + labels, predictions, thresholds, weights) def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): specificities = math_ops.div(tn, tn + fp + kepsilon) @@ -2853,20 +2880,22 @@ def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the sensitivity: - return math_ops.div(tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, name) + return math_ops.div(tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, + name) def sensitivity_across_towers(_, values): - return compute_sensitivity_at_specificity(values['tp'], values['tn'], values['fp'], values['fn'], 'value') - - sensitivity = _aggregate_across_towers(metrics_collections, sensitivity_across_towers, values) - - update_op = compute_sensitivity_at_specificity( - update_ops['tp'], - update_ops['tn'], - update_ops['fp'], - update_ops['fn'], - 'update_op', - ) + return compute_sensitivity_at_specificity(values['tp'], values['tn'], + values['fp'], values['fn'], + 'value') + + sensitivity = _aggregate_across_towers(metrics_collections, + sensitivity_across_towers, values) + + update_op = compute_sensitivity_at_specificity(update_ops['tp'], + update_ops['tn'], + update_ops['fp'], + update_ops['fn'], + 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2894,34 +2923,37 @@ def _expand_and_tile(tensor, multiple, dim=0, name=None): """ if multiple < 1: raise ValueError('Invalid multiple %s, must be > 0.' % multiple) - with ops.name_scope(name, 'expand_and_tile', (tensor, multiple, dim)) as scope: + with ops.name_scope(name, 'expand_and_tile', + (tensor, multiple, dim)) as scope: # Sparse. tensor = sparse_tensor.convert_to_tensor_or_sparse_tensor(tensor) if isinstance(tensor, sparse_tensor.SparseTensor): if dim < 0: - expand_dims = array_ops.reshape(array_ops.size(tensor.dense_shape) + dim, [1]) + expand_dims = array_ops.reshape( + array_ops.size(tensor.dense_shape) + dim, [1]) else: expand_dims = [dim] expanded_shape = array_ops.concat( - ( - array_ops.slice(tensor.dense_shape, [0], expand_dims), - [1], - array_ops.slice(tensor.dense_shape, expand_dims, [-1]), - ), - 0, - name='expanded_shape', - ) - expanded = sparse_ops.sparse_reshape(tensor, shape=expanded_shape, name='expand') + (array_ops.slice(tensor.dense_shape, [0], expand_dims), [1], + array_ops.slice(tensor.dense_shape, expand_dims, [-1])), + 0, + name='expanded_shape') + expanded = sparse_ops.sparse_reshape( + tensor, shape=expanded_shape, name='expand') if multiple == 1: return expanded - return sparse_ops.sparse_concat(dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope) + return sparse_ops.sparse_concat( + dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope) # Dense. - expanded = array_ops.expand_dims(tensor, dim if (dim >= 0) else (dim - 1), name='expand') + expanded = array_ops.expand_dims( + tensor, dim if (dim >= 0) else (dim - 1), name='expand') if multiple == 1: return expanded ones = array_ops.ones_like(array_ops.shape(tensor)) - tile_multiples = array_ops.concat((ones[:dim], (multiple,), ones[dim:]), 0, name='multiples') + tile_multiples = array_ops.concat((ones[:dim], (multiple,), ones[dim:]), + 0, + name='multiples') return array_ops.tile(expanded, tile_multiples, name=scope) @@ -2993,7 +3025,8 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): Raises: ValueError: if the last dimension of predictions_idx is not set. """ - with ops.name_scope(None, 'average_precision', (predictions_idx, labels)) as scope: + with ops.name_scope(None, 'average_precision', + (predictions_idx, labels)) as scope: predictions_idx = math_ops.to_int64(predictions_idx, name='predictions_idx') if predictions_idx.get_shape().ndims == 0: raise ValueError('The rank of predictions_idx must be at least 1.') @@ -3005,10 +3038,12 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # Expand dims to produce [D1, ... DN, k, 1] tensor. This gives us a separate # prediction for each k, so we can calculate separate true positive values # for each k. - predictions_idx_per_k = array_ops.expand_dims(predictions_idx, -1, name='predictions_idx_per_k') + predictions_idx_per_k = array_ops.expand_dims( + predictions_idx, -1, name='predictions_idx_per_k') # Replicate labels k times to produce [D1, ... DN, k, num_labels] tensor. - labels_per_k = _expand_and_tile(labels, multiple=k, dim=-1, name='labels_per_k') + labels_per_k = _expand_and_tile( + labels, multiple=k, dim=-1, name='labels_per_k') # The following tensors are all of shape [D1, ... DN, k], containing values # per row, per k value. @@ -3022,22 +3057,23 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # term from the formula above. # `relevant_precision_per_k` (float64) - Relevant precisions; i.e., # precisions at all k for which relevance indicator is true. - relevant_per_k = _sparse_true_positive_at_k(labels_per_k, predictions_idx_per_k, name='relevant_per_k') + relevant_per_k = _sparse_true_positive_at_k( + labels_per_k, predictions_idx_per_k, name='relevant_per_k') tp_per_k = math_ops.cumsum(relevant_per_k, axis=-1, name='tp_per_k') - retrieved_per_k = math_ops.cumsum(array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k') + retrieved_per_k = math_ops.cumsum( + array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k') precision_per_k = math_ops.div( - math_ops.to_double(tp_per_k), - math_ops.to_double(retrieved_per_k), - name='precision_per_k', - ) + math_ops.to_double(tp_per_k), + math_ops.to_double(retrieved_per_k), + name='precision_per_k') relevant_precision_per_k = math_ops.multiply( - precision_per_k, - math_ops.to_double(relevant_per_k), - name='relevant_precision_per_k', - ) + precision_per_k, + math_ops.to_double(relevant_per_k), + name='relevant_precision_per_k') # Reduce along k dimension to get the sum, yielding a [D1, ... DN] tensor. - precision_sum = math_ops.reduce_sum(relevant_precision_per_k, reduction_indices=(-1,), name='precision_sum') + precision_sum = math_ops.reduce_sum( + relevant_precision_per_k, reduction_indices=(-1,), name='precision_sum') # Divide by number of relevant items to get average precision. These are # the "num_relevant_items" and "AveP" terms from the formula above. @@ -3045,14 +3081,12 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): return math_ops.div(precision_sum, num_relevant_items, name=scope) -def _streaming_sparse_average_precision_at_top_k( - labels, - predictions_idx, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def _streaming_sparse_average_precision_at_top_k(labels, + predictions_idx, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes average precision@k of predictions with respect to sparse labels. `sparse_average_precision_at_top_k` creates two local variables, @@ -3097,11 +3131,14 @@ def _streaming_sparse_average_precision_at_top_k( update: `Operation` that increments variables appropriately, and whose value matches `metric`. """ - with ops.name_scope(name, 'average_precision_at_top_k', (predictions_idx, labels, weights)) as scope: + with ops.name_scope(name, 'average_precision_at_top_k', + (predictions_idx, labels, weights)) as scope: # Calculate per-example average precision, and apply weights. - average_precision = _sparse_average_precision_at_top_k(predictions_idx=predictions_idx, labels=labels) + average_precision = _sparse_average_precision_at_top_k( + predictions_idx=predictions_idx, labels=labels) if weights is not None: - weights = weights_broadcast_ops.broadcast_weights(math_ops.to_double(weights), average_precision) + weights = weights_broadcast_ops.broadcast_weights( + math_ops.to_double(weights), average_precision) average_precision = math_ops.multiply(average_precision, weights) # Create accumulation variables and update ops for max average precision and @@ -3113,20 +3150,25 @@ def _streaming_sparse_average_precision_at_top_k( # `average_precision` rows. max_var = metric_variable([], dtypes.float64, name=max_scope) if weights is None: - batch_max = math_ops.to_double(array_ops.size(average_precision, name='batch_max')) + batch_max = math_ops.to_double( + array_ops.size(average_precision, name='batch_max')) else: batch_max = math_ops.reduce_sum(weights, name='batch_max') - max_update = state_ops.assign_add(max_var, batch_max, name='update', use_locking=True) + max_update = state_ops.assign_add( + max_var, batch_max, name='update', use_locking=True) with ops.name_scope(None, 'total', (average_precision,)) as total_scope: total_var = metric_variable([], dtypes.float64, name=total_scope) batch_total = math_ops.reduce_sum(average_precision, name='batch_total') - total_update = state_ops.assign_add(total_var, batch_total, name='update', use_locking=True) + total_update = state_ops.assign_add( + total_var, batch_total, name='update', use_locking=True) # Divide total by max to get mean, for both vars and the update ops. def precision_across_towers(_, total_var, max_var): return _safe_scalar_div(total_var, max_var, name='mean') - mean_average_precision = _aggregate_across_towers(metrics_collections, precision_across_towers, total_var, max_var) + mean_average_precision = _aggregate_across_towers(metrics_collections, + precision_across_towers, + total_var, max_var) update = _safe_scalar_div(total_update, max_update, name=scope) if updates_collections: @@ -3137,37 +3179,32 @@ def precision_across_towers(_, total_var, max_var): @tf_export('metrics.sparse_average_precision_at_k') @deprecated(None, 'Use average_precision_at_k instead') -def sparse_average_precision_at_k( - labels, - predictions, - k, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def sparse_average_precision_at_k(labels, + predictions, + k, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Renamed to `average_precision_at_k`, please use that method instead.""" return average_precision_at_k( - labels=labels, - predictions=predictions, - k=k, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name, - ) + labels=labels, + predictions=predictions, + k=k, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) @tf_export('metrics.average_precision_at_k') -def average_precision_at_k( - labels, - predictions, - k, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def average_precision_at_k(labels, + predictions, + k, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes average precision@k of predictions with respect to sparse labels. `average_precision_at_k` creates two local variables, @@ -3221,24 +3258,28 @@ def average_precision_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sparse_average_precision_at_k is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.sparse_average_precision_at_k is not ' + 'supported when eager execution is enabled.') if k < 1: raise ValueError('Invalid k=%s.' % k) - with ops.name_scope(name, _at_k_name('average_precision', k), (predictions, labels, weights)) as scope: + with ops.name_scope(name, _at_k_name('average_precision', k), + (predictions, labels, weights)) as scope: # Calculate top k indices to produce [D1, ... DN, k] tensor. _, predictions_idx = nn.top_k(predictions, k) return _streaming_sparse_average_precision_at_top_k( - labels=labels, - predictions_idx=predictions_idx, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope, - ) - - -def _sparse_false_positive_at_k(labels, predictions_idx, class_id=None, weights=None): + labels=labels, + predictions_idx=predictions_idx, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope) + + +def _sparse_false_positive_at_k(labels, + predictions_idx, + class_id=None, + weights=None): """Calculates false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3264,18 +3305,27 @@ def _sparse_false_positive_at_k(labels, predictions_idx, class_id=None, weights= Returns: A [D1, ... DN] `Tensor` of false positive counts. """ - with ops.name_scope(None, 'false_positives', (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) - fp = sets.set_size(sets.set_difference(predictions_idx, labels, aminusb=True)) + with ops.name_scope(None, 'false_positives', + (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, + class_id) + fp = sets.set_size( + sets.set_difference(predictions_idx, labels, aminusb=True)) fp = math_ops.to_double(fp) if weights is not None: - with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, fp),)): + with ops.control_dependencies( + (weights_broadcast_ops.assert_broadcastable(weights, fp),)): weights = math_ops.to_double(weights) fp = math_ops.multiply(fp, weights) return fp -def _streaming_sparse_false_positive_at_k(labels, predictions_idx, k=None, class_id=None, weights=None, name=None): +def _streaming_sparse_false_positive_at_k(labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + name=None): """Calculates weighted per step false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3308,34 +3358,29 @@ def _streaming_sparse_false_positive_at_k(labels, predictions_idx, k=None, class Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope( - name, - _at_k_name('false_positive', k, class_id=class_id), - (predictions_idx, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('false_positive', k, class_id=class_id), + (predictions_idx, labels, weights)) as scope: fp = _sparse_false_positive_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights, - ) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights) batch_total_fp = math_ops.to_double(math_ops.reduce_sum(fp)) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add(var, batch_total_fp, name='update', use_locking=True) + return var, state_ops.assign_add( + var, batch_total_fp, name='update', use_locking=True) @tf_export('metrics.precision_at_top_k') -def precision_at_top_k( - labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def precision_at_top_k(labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes precision@k of the predictions with respect to sparse labels. Differs from `sparse_precision_at_k` in that predictions must be in the form @@ -3383,36 +3428,34 @@ def precision_at_top_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision_at_top_k is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.precision_at_top_k is not ' + 'supported when eager execution is enabled.') - with ops.name_scope( - name, - _at_k_name('precision', k, class_id=class_id), - (predictions_idx, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), + (predictions_idx, labels, weights)) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.to_int64(predictions_idx) tp, tp_update = _streaming_sparse_true_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights, - ) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights) fp, fp_update = _streaming_sparse_false_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights, - ) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights) def precision_across_towers(_, tp, fp): return math_ops.div(tp, math_ops.add(tp, fp), name=scope) - metric = _aggregate_across_towers(metrics_collections, precision_across_towers, tp, fp) + metric = _aggregate_across_towers(metrics_collections, + precision_across_towers, tp, fp) - update = math_ops.div(tp_update, math_ops.add(tp_update, fp_update), name='update') + update = math_ops.div( + tp_update, math_ops.add(tp_update, fp_update), name='update') if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @@ -3420,40 +3463,35 @@ def precision_across_towers(_, tp, fp): @tf_export('metrics.sparse_precision_at_k') @deprecated(None, 'Use precision_at_k instead') -def sparse_precision_at_k( - labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def sparse_precision_at_k(labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Renamed to `precision_at_k`, please use that method instead.""" return precision_at_k( - labels=labels, - predictions=predictions, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name, - ) + labels=labels, + predictions=predictions, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) @tf_export('metrics.precision_at_k') -def precision_at_k( - labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def precision_at_k(labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes precision@k of the predictions with respect to sparse labels. If `class_id` is specified, we calculate precision by considering only the @@ -3522,37 +3560,32 @@ def precision_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sparse_precision_at_k is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.sparse_precision_at_k is not ' + 'supported when eager execution is enabled.') - with ops.name_scope( - name, - _at_k_name('precision', k, class_id=class_id), - (predictions, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), + (predictions, labels, weights)) as scope: _, top_k_idx = nn.top_k(predictions, k) return precision_at_top_k( - labels=labels, - predictions_idx=top_k_idx, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope, - ) + labels=labels, + predictions_idx=top_k_idx, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope) @tf_export('metrics.specificity_at_sensitivity') -def specificity_at_sensitivity( - labels, - predictions, - sensitivity, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None, -): +def specificity_at_sensitivity(labels, + predictions, + sensitivity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the specificity at a given sensitivity. The `specificity_at_sensitivity` function creates four local @@ -3604,17 +3637,22 @@ def specificity_at_sensitivity( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.specificity_at_sensitivity is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.specificity_at_sensitivity is not ' + 'supported when eager execution is enabled.') if sensitivity < 0 or sensitivity > 1: raise ValueError('`sensitivity` must be in the range [0, 1].') - with variable_scope.variable_scope(name, 'specificity_at_sensitivity', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'specificity_at_sensitivity', + (predictions, labels, weights)): kepsilon = 1e-7 # to account for floating point imprecisions - thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] + thresholds = [ + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 - kepsilon] - values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) + values, update_ops = _confusion_matrix_at_thresholds( + labels, predictions, thresholds, weights) def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): """Computes the specificity at the given sensitivity. @@ -3634,27 +3672,30 @@ def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): # We'll need to use this trick until tf.argmax allows us to specify # whether we should use the first or last index in case of ties. min_val = math_ops.reduce_min(math_ops.abs(sensitivities - sensitivity)) - indices_at_minval = math_ops.equal(math_ops.abs(sensitivities - sensitivity), min_val) + indices_at_minval = math_ops.equal( + math_ops.abs(sensitivities - sensitivity), min_val) indices_at_minval = math_ops.to_int64(indices_at_minval) indices_at_minval = math_ops.cumsum(indices_at_minval) tf_index = math_ops.argmax(indices_at_minval, 0) tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the specificity: - return math_ops.div(tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, name) + return math_ops.div(tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, + name) def specificity_across_towers(_, values): - return compute_specificity_at_sensitivity(values['tp'], values['tn'], values['fp'], values['fn'], 'value') - - specificity = _aggregate_across_towers(metrics_collections, specificity_across_towers, values) - - update_op = compute_specificity_at_sensitivity( - update_ops['tp'], - update_ops['tn'], - update_ops['fp'], - update_ops['fn'], - 'update_op', - ) + return compute_specificity_at_sensitivity(values['tp'], values['tn'], + values['fp'], values['fn'], + 'value') + + specificity = _aggregate_across_towers(metrics_collections, + specificity_across_towers, values) + + update_op = compute_specificity_at_sensitivity(update_ops['tp'], + update_ops['tn'], + update_ops['fp'], + update_ops['fn'], + 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) diff --git a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py index 417d5a481..1756826be 100644 --- a/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py +++ b/easy_rec/python/core/easyrec_metrics/distribute_metrics_impl_tf.py @@ -13,24 +13,26 @@ # ============================================================================== """Implementation of tf.metrics module.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes, ops, sparse_tensor -from tensorflow.python.ops import ( # NOQA - array_ops, - check_ops, - confusion_matrix, - control_flow_ops, - math_ops, - nn, - sets, - sparse_ops, - state_ops, - variable_scope, - weights_broadcast_ops, -) +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import check_ops +from tensorflow.python.ops import confusion_matrix +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import sets +from tensorflow.python.ops import sparse_ops +from tensorflow.python.ops import state_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import weights_broadcast_ops from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.tf_export import tf_export @@ -74,14 +76,15 @@ def metric_variable(shape, dtype, validate_shape=True, name=None): """ # Note that synchronization "ON_READ" implies trainable=False. return variable_scope.variable( - lambda: array_ops.zeros(shape, dtype), - trainable=False, - collections=[ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES], - validate_shape=validate_shape, - synchronization=variable_scope.VariableSynchronization.ON_READ, - aggregation=variable_scope.VariableAggregation.SUM, - name=name, - ) + lambda: array_ops.zeros(shape, dtype), + trainable=False, + collections=[ + ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.METRIC_VARIABLES + ], + validate_shape=validate_shape, + synchronization=variable_scope.VariableSynchronization.ON_READ, + aggregation=variable_scope.VariableAggregation.SUM, + name=name) def _remove_squeezable_dimensions(predictions, labels, weights): @@ -109,7 +112,8 @@ def _remove_squeezable_dimensions(predictions, labels, weights): """ predictions = ops.convert_to_tensor(predictions) if labels is not None: - labels, predictions = confusion_matrix.remove_squeezable_dimensions(labels, predictions) + labels, predictions = confusion_matrix.remove_squeezable_dimensions( + labels, predictions) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if weights is None: @@ -136,31 +140,27 @@ def _remove_squeezable_dimensions(predictions, labels, weights): def _maybe_expand_weights(): return control_flow_ops.cond( - math_ops.equal(rank_diff, -1), - lambda: array_ops.expand_dims(weights, [-1]), - lambda: weights, - ) + math_ops.equal(rank_diff, -1), + lambda: array_ops.expand_dims(weights, [-1]), lambda: weights) # Don't attempt squeeze if it will fail based on static check. - if (weights_rank is not None) and (not weights_shape.dims[-1].is_compatible_with(1)): + if ((weights_rank is not None) and + (not weights_shape.dims[-1].is_compatible_with(1))): maybe_squeeze_weights = lambda: weights # noqa: E731 else: - maybe_squeeze_weights = lambda: array_ops.squeeze(weights, [-1]) # noqa: E731 + maybe_squeeze_weights = lambda: array_ops.squeeze( # noqa: E731 + weights, [-1]) def _maybe_adjust_weights(): return control_flow_ops.cond( - math_ops.equal(rank_diff, 1), - maybe_squeeze_weights, - _maybe_expand_weights, - ) + math_ops.equal(rank_diff, 1), maybe_squeeze_weights, + _maybe_expand_weights) # If weights are scalar, do nothing. Otherwise, try to add or remove a # dimension to match predictions. weights = control_flow_ops.cond( - math_ops.equal(weights_rank_tensor, 0), - lambda: weights, - _maybe_adjust_weights, - ) + math_ops.equal(weights_rank_tensor, 0), lambda: weights, + _maybe_adjust_weights) return predictions, labels, weights @@ -186,14 +186,14 @@ def _maybe_expand_labels(labels, predictions): # If sparse, expand sparse shape. if isinstance(labels, sparse_tensor.SparseTensor): return control_flow_ops.cond( - math_ops.equal(array_ops.rank(predictions), array_ops.size(labels.dense_shape) + 1), - lambda: sparse_ops.sparse_reshape( # pylint: disable=g-long-lambda - labels, - shape=array_ops.concat((labels.dense_shape, (1,)), 0), - name=scope, - ), - lambda: labels, - ) + math_ops.equal( + array_ops.rank(predictions), + array_ops.size(labels.dense_shape) + 1), + lambda: sparse_ops.sparse_reshape( # pylint: disable=g-long-lambda + labels, + shape=array_ops.concat((labels.dense_shape, (1,)), 0), + name=scope), + lambda: labels) # Otherwise, try to use static shape. labels_rank = labels.get_shape().ndims @@ -205,15 +205,14 @@ def _maybe_expand_labels(labels, predictions): if predictions_rank == labels_rank + 1: return array_ops.expand_dims(labels, -1, name=scope) raise ValueError( - 'Unexpected labels shape %s for predictions shape %s.' % (labels.get_shape(), predictions.get_shape()) - ) + 'Unexpected labels shape %s for predictions shape %s.' % + (labels.get_shape(), predictions.get_shape())) # Otherwise, use dynamic shape. return control_flow_ops.cond( - math_ops.equal(array_ops.rank(predictions), array_ops.rank(labels) + 1), - lambda: array_ops.expand_dims(labels, -1, name=scope), - lambda: labels, - ) + math_ops.equal(array_ops.rank(predictions), + array_ops.rank(labels) + 1), + lambda: array_ops.expand_dims(labels, -1, name=scope), lambda: labels) def _safe_scalar_div(numerator, denominator, name): @@ -256,7 +255,9 @@ def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): update_op: An operation that increments the confusion matrix. """ # Local variable to accumulate the predictions in the confusion matrix. - total_cm = metric_variable([num_classes, num_classes], dtypes.float64, name='total_confusion_matrix') + total_cm = metric_variable([num_classes, num_classes], + dtypes.float64, + name='total_confusion_matrix') # Cast the type to int64 required by confusion_matrix_ops. predictions = math_ops.cast(predictions, dtypes.int64) @@ -275,8 +276,7 @@ def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): # Accumulate the prediction to current confusion matrix. current_cm = confusion_matrix.confusion_matrix( - labels, predictions, num_classes, weights=weights, dtype=dtypes.float64 - ) + labels, predictions, num_classes, weights=weights, dtype=dtypes.float64) update_op = state_ops.assign_add(total_cm, current_cm, use_locking=True) return total_cm, update_op @@ -313,11 +313,16 @@ def fn(distribution, *a): ops.add_to_collections(metrics_collections, metric_value) return metric_value - return distribution_strategy_context.get_replica_context().merge_call(fn, args=args) + return distribution_strategy_context.get_replica_context().merge_call( + fn, args=args) @tf_export(v1=['metrics.mean']) -def mean(values, weights=None, metrics_collections=None, updates_collections=None, name=None): +def mean(values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the (weighted) mean of the given values. The `mean` function creates two local variables, `total` and `count` @@ -356,7 +361,8 @@ def mean(values, weights=None, metrics_collections=None, updates_collections=Non RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean is not supported when eager execution ' 'is enabled.') + raise RuntimeError('tf.metrics.mean is not supported when eager execution ' + 'is enabled.') with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.cast(values, dtypes.float32) @@ -367,20 +373,26 @@ def mean(values, weights=None, metrics_collections=None, updates_collections=Non if weights is None: num_values = math_ops.cast(array_ops.size(values), dtypes.float32) else: - values, _, weights = _remove_squeezable_dimensions(predictions=values, labels=None, weights=weights) - weights = weights_broadcast_ops.broadcast_weights(math_ops.cast(weights, dtypes.float32), values) + values, _, weights = _remove_squeezable_dimensions( + predictions=values, labels=None, weights=weights) + weights = weights_broadcast_ops.broadcast_weights( + math_ops.cast(weights, dtypes.float32), values) values = math_ops.multiply(values, weights) num_values = math_ops.reduce_sum(weights) - update_total_op = state_ops.assign_add(total, math_ops.reduce_sum(values), use_locking=True) + update_total_op = state_ops.assign_add( + total, math_ops.reduce_sum(values), use_locking=True) with ops.control_dependencies([values]): - update_count_op = state_ops.assign_add(count, num_values, use_locking=True) + update_count_op = state_ops.assign_add( + count, num_values, use_locking=True) def compute_mean(_, t, c): return math_ops.div_no_nan(t, math_ops.maximum(c, 0), name='value') - mean_t = _aggregate_across_replicas(metrics_collections, compute_mean, total, count) - update_op = math_ops.div_no_nan(update_total_op, math_ops.maximum(update_count_op, 0), name='update_op') + mean_t = _aggregate_across_replicas(metrics_collections, compute_mean, + total, count) + update_op = math_ops.div_no_nan( + update_total_op, math_ops.maximum(update_count_op, 0), name='update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -389,14 +401,12 @@ def compute_mean(_, t, c): @tf_export(v1=['metrics.accuracy']) -def accuracy( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def accuracy(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Calculates how often `predictions` matches `labels`. The `accuracy` function creates two local variables, `total` and @@ -441,23 +451,25 @@ def accuracy( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.accuracy is not supported when eager ' 'execution is enabled.') + raise RuntimeError('tf.metrics.accuracy is not supported when eager ' + 'execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if labels.dtype != predictions.dtype: predictions = math_ops.cast(predictions, labels.dtype) - is_correct = math_ops.cast(math_ops.equal(predictions, labels), dtypes.float32) - return mean( - is_correct, - weights, - metrics_collections, - updates_collections, - name or 'accuracy', - ) + is_correct = math_ops.cast( + math_ops.equal(predictions, labels), dtypes.float32) + return mean(is_correct, weights, metrics_collections, updates_collections, + name or 'accuracy') -def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=None, includes=None): +def _confusion_matrix_at_thresholds(labels, + predictions, + thresholds, + weights=None, + includes=None): """Computes true_positives, false_negatives, true_negatives, false_positives. This function creates up to four local variables, `true_positives`, @@ -508,31 +520,27 @@ def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=Non if include not in all_includes: raise ValueError('Invalid key: %s.' % include) - with ops.control_dependencies( - [ + with ops.control_dependencies([ check_ops.assert_greater_equal( - predictions, - math_ops.cast(0.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]', - ), + predictions, + math_ops.cast(0.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]'), check_ops.assert_less_equal( - predictions, - math_ops.cast(1.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]', - ), - ] - ): + predictions, + math_ops.cast(1.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]') + ]): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtypes.float32), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) + predictions=math_ops.cast(predictions, dtypes.float32), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) num_thresholds = len(thresholds) # Reshape predictions and labels. predictions_2d = array_ops.reshape(predictions, [-1, 1]) - labels_2d = array_ops.reshape(math_ops.cast(labels, dtype=dtypes.bool), [1, -1]) + labels_2d = array_ops.reshape( + math_ops.cast(labels, dtype=dtypes.bool), [1, -1]) # Use static shape if known. num_predictions = predictions_2d.get_shape().as_list()[0] @@ -541,15 +549,13 @@ def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=Non if num_predictions is None: num_predictions = array_ops.shape(predictions_2d)[0] thresh_tiled = array_ops.tile( - array_ops.expand_dims(array_ops.constant(thresholds), [1]), - array_ops.stack([1, num_predictions]), - ) + array_ops.expand_dims(array_ops.constant(thresholds), [1]), + array_ops.stack([1, num_predictions])) # Tile the predictions after thresholding them across different thresholds. pred_is_pos = math_ops.greater( - array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]), - thresh_tiled, - ) + array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]), + thresh_tiled) if ('fn' in includes) or ('tn' in includes): pred_is_neg = math_ops.logical_not(pred_is_pos) @@ -559,9 +565,12 @@ def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=Non label_is_neg = math_ops.logical_not(label_is_pos) if weights is not None: - weights = weights_broadcast_ops.broadcast_weights(math_ops.cast(weights, dtypes.float32), predictions) - weights_tiled = array_ops.tile(array_ops.reshape(weights, [1, -1]), [num_thresholds, 1]) - thresh_tiled.get_shape().assert_is_compatible_with(weights_tiled.get_shape()) + weights = weights_broadcast_ops.broadcast_weights( + math_ops.cast(weights, dtypes.float32), predictions) + weights_tiled = array_ops.tile( + array_ops.reshape(weights, [1, -1]), [num_thresholds, 1]) + thresh_tiled.get_shape().assert_is_compatible_with( + weights_tiled.get_shape()) else: weights_tiled = None @@ -569,58 +578,73 @@ def _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights=Non update_ops = {} if 'tp' in includes: - true_p = metric_variable([num_thresholds], dtypes.float32, name='true_positives') - is_true_positive = math_ops.cast(math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32) + true_p = metric_variable([num_thresholds], + dtypes.float32, + name='true_positives') + is_true_positive = math_ops.cast( + math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32) if weights_tiled is not None: is_true_positive *= weights_tiled - update_ops['tp'] = state_ops.assign_add(true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True) + update_ops['tp'] = state_ops.assign_add( + true_p, math_ops.reduce_sum(is_true_positive, 1), use_locking=True) values['tp'] = true_p if 'fn' in includes: - false_n = metric_variable([num_thresholds], dtypes.float32, name='false_negatives') - is_false_negative = math_ops.cast(math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32) + false_n = metric_variable([num_thresholds], + dtypes.float32, + name='false_negatives') + is_false_negative = math_ops.cast( + math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32) if weights_tiled is not None: is_false_negative *= weights_tiled - update_ops['fn'] = state_ops.assign_add(false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True) + update_ops['fn'] = state_ops.assign_add( + false_n, math_ops.reduce_sum(is_false_negative, 1), use_locking=True) values['fn'] = false_n if 'tn' in includes: - true_n = metric_variable([num_thresholds], dtypes.float32, name='true_negatives') - is_true_negative = math_ops.cast(math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32) + true_n = metric_variable([num_thresholds], + dtypes.float32, + name='true_negatives') + is_true_negative = math_ops.cast( + math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32) if weights_tiled is not None: is_true_negative *= weights_tiled - update_ops['tn'] = state_ops.assign_add(true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True) + update_ops['tn'] = state_ops.assign_add( + true_n, math_ops.reduce_sum(is_true_negative, 1), use_locking=True) values['tn'] = true_n if 'fp' in includes: - false_p = metric_variable([num_thresholds], dtypes.float32, name='false_positives') - is_false_positive = math_ops.cast(math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32) + false_p = metric_variable([num_thresholds], + dtypes.float32, + name='false_positives') + is_false_positive = math_ops.cast( + math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32) if weights_tiled is not None: is_false_positive *= weights_tiled - update_ops['fp'] = state_ops.assign_add(false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True) + update_ops['fp'] = state_ops.assign_add( + false_p, math_ops.reduce_sum(is_false_positive, 1), use_locking=True) values['fp'] = false_p return values, update_ops def _aggregate_variable(v, collections): - f = lambda distribution, value: distribution.extended.read_var(value) # noqa: E731 + f = lambda distribution, value: distribution.extended.read_var( # noqa: E731 + value) return _aggregate_across_replicas(collections, f, v) @tf_export(v1=['metrics.auc']) -def auc( - labels, - predictions, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - curve='ROC', - name=None, - summation_method='trapezoidal', - thresholds=None, -): +def auc(labels, + predictions, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + curve='ROC', + name=None, + summation_method='trapezoidal', + thresholds=None): """Computes the approximate AUC via a Riemann sum. The `auc` function creates four local variables, `true_positives`, @@ -700,9 +724,11 @@ def auc( """ print('use_tf_auc') if context.executing_eagerly(): - raise RuntimeError('tf.metrics.auc is not supported when eager execution ' 'is enabled.') + raise RuntimeError('tf.metrics.auc is not supported when eager execution ' + 'is enabled.') - with variable_scope.variable_scope(name, 'auc', (labels, predictions, weights)): + with variable_scope.variable_scope(name, 'auc', + (labels, predictions, weights)): if curve != 'ROC' and curve != 'PR': raise ValueError('curve must be either ROC or PR, %s unknown' % (curve)) @@ -714,13 +740,15 @@ def auc( else: # Otherwise, linearly interpolate (num_thresholds - 2) thresholds in # (0, 1). - thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] + thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) + for i in range(num_thresholds - 2)] # Add an endpoint "threshold" below zero and above one for either threshold # method. thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] - values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) + values, update_ops = _confusion_matrix_at_thresholds( + labels, predictions, thresholds, weights) # Add epsilons to avoid dividing by 0. epsilon = 1.0e-6 @@ -756,40 +784,33 @@ def interpolate_pr_auc(tp, fp, fn): Returns: pr_auc: an approximation of the area under the P-R curve. """ - dtp = tp[: num_thresholds - 1] - tp[1:] + dtp = tp[:num_thresholds - 1] - tp[1:] p = tp + fp prec_slope = math_ops.div_no_nan( - dtp, - math_ops.maximum(p[: num_thresholds - 1] - p[1:], 0), - name='prec_slope', - ) + dtp, + math_ops.maximum(p[:num_thresholds - 1] - p[1:], 0), + name='prec_slope') intercept = tp[1:] - math_ops.multiply(prec_slope, p[1:]) safe_p_ratio = array_ops.where( - math_ops.logical_and(p[: num_thresholds - 1] > 0, p[1:] > 0), - math_ops.div_no_nan( - p[: num_thresholds - 1], - math_ops.maximum(p[1:], 0), - name='recall_relative_ratio', - ), - array_ops.ones_like(p[1:]), - ) + math_ops.logical_and(p[:num_thresholds - 1] > 0, p[1:] > 0), + math_ops.div_no_nan( + p[:num_thresholds - 1], + math_ops.maximum(p[1:], 0), + name='recall_relative_ratio'), array_ops.ones_like(p[1:])) return math_ops.reduce_sum( - math_ops.div_no_nan( - prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)), - math_ops.maximum(tp[1:] + fn[1:], 0), - name='pr_auc_increment', - ), - name='interpolate_pr_auc', - ) + math_ops.div_no_nan( + prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)), + math_ops.maximum(tp[1:] + fn[1:], 0), + name='pr_auc_increment'), + name='interpolate_pr_auc') def compute_auc(tp, fn, tn, fp, name): """Computes the roc-auc or pr-auc based on confusion counts.""" if curve == 'PR': if summation_method == 'trapezoidal': logging.warning( - 'Trapezoidal rule is known to produce incorrect PR-AUCs; ' - 'please switch to "careful_interpolation" instead.' - ) + 'Trapezoidal rule is known to produce incorrect PR-AUCs; ' + 'please switch to "careful_interpolation" instead.') elif summation_method == 'careful_interpolation': # This one is a bit tricky and is handled separately. return interpolate_pr_auc(tp, fp, fn) @@ -806,43 +827,31 @@ def compute_auc(tp, fn, tn, fp, name): # Note that the case ('PR', 'careful_interpolation') has been handled # above. return math_ops.reduce_sum( - math_ops.multiply( - x[: num_thresholds - 1] - x[1:], - (y[: num_thresholds - 1] + y[1:]) / 2.0, - ), - name=name, - ) + math_ops.multiply(x[:num_thresholds - 1] - x[1:], + (y[:num_thresholds - 1] + y[1:]) / 2.), + name=name) elif summation_method == 'minoring': return math_ops.reduce_sum( - math_ops.multiply( - x[: num_thresholds - 1] - x[1:], - math_ops.minimum(y[: num_thresholds - 1], y[1:]), - ), - name=name, - ) + math_ops.multiply(x[:num_thresholds - 1] - x[1:], + math_ops.minimum(y[:num_thresholds - 1], y[1:])), + name=name) elif summation_method == 'majoring': return math_ops.reduce_sum( - math_ops.multiply( - x[: num_thresholds - 1] - x[1:], - math_ops.maximum(y[: num_thresholds - 1], y[1:]), - ), - name=name, - ) + math_ops.multiply(x[:num_thresholds - 1] - x[1:], + math_ops.maximum(y[:num_thresholds - 1], y[1:])), + name=name) else: raise ValueError('Invalid summation_method: %s' % summation_method) # sum up the areas of all the trapeziums def compute_auc_value(_, values): - return compute_auc(values['tp'], values['fn'], values['tn'], values['fp'], 'value') + return compute_auc(values['tp'], values['fn'], values['tn'], values['fp'], + 'value') - auc_value = _aggregate_across_replicas(metrics_collections, compute_auc_value, values) - update_op = compute_auc( - update_ops['tp'], - update_ops['fn'], - update_ops['tn'], - update_ops['fp'], - 'update_op', - ) + auc_value = _aggregate_across_replicas(metrics_collections, + compute_auc_value, values) + update_op = compute_auc(update_ops['tp'], update_ops['fn'], + update_ops['tn'], update_ops['fp'], 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -851,14 +860,12 @@ def compute_auc_value(_, values): @tf_export(v1=['metrics.mean_absolute_error']) -def mean_absolute_error( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_absolute_error(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the mean absolute error between the labels and predictions. The `mean_absolute_error` function creates two local variables, @@ -903,29 +910,24 @@ def mean_absolute_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_absolute_error is not supported ' 'when eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_absolute_error is not supported ' + 'when eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) absolute_errors = math_ops.abs(predictions - labels) - return mean( - absolute_errors, - weights, - metrics_collections, - updates_collections, - name or 'mean_absolute_error', - ) + return mean(absolute_errors, weights, metrics_collections, + updates_collections, name or 'mean_absolute_error') @tf_export(v1=['metrics.mean_cosine_distance']) -def mean_cosine_distance( - labels, - predictions, - dim, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_cosine_distance(labels, + predictions, + dim, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the cosine distance between the labels and predictions. The `mean_cosine_distance` function creates two local variables, @@ -968,18 +970,18 @@ def mean_cosine_distance( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_cosine_distance is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_cosine_distance is not supported when ' + 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) radial_diffs = math_ops.multiply(predictions, labels) radial_diffs = math_ops.reduce_sum( - radial_diffs, - axis=[ - dim, - ], - keepdims=True, - ) - mean_distance, update_op = mean(radial_diffs, weights, None, None, name or 'mean_cosine_distance') + radial_diffs, axis=[ + dim, + ], keepdims=True) + mean_distance, update_op = mean(radial_diffs, weights, None, None, name or + 'mean_cosine_distance') mean_distance = math_ops.subtract(1.0, mean_distance) update_op = math_ops.subtract(1.0, update_op) @@ -993,15 +995,13 @@ def mean_cosine_distance( @tf_export(v1=['metrics.mean_per_class_accuracy']) -def mean_per_class_accuracy( - labels, - predictions, - num_classes, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_per_class_accuracy(labels, + predictions, + num_classes, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Calculates the mean of the per-class accuracies. Calculates the accuracy for each class, then takes the mean of that. @@ -1043,9 +1043,11 @@ def mean_per_class_accuracy( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_per_class_accuracy is not supported ' 'when eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_per_class_accuracy is not supported ' + 'when eager execution is enabled.') - with variable_scope.variable_scope(name, 'mean_accuracy', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'mean_accuracy', + (predictions, labels, weights)): labels = math_ops.cast(labels, dtypes.int64) # Flatten the input if its rank > 1. @@ -1065,7 +1067,8 @@ def mean_per_class_accuracy( if labels.dtype != predictions.dtype: predictions = math_ops.cast(predictions, labels.dtype) - is_correct = math_ops.cast(math_ops.equal(predictions, labels), dtypes.float32) + is_correct = math_ops.cast( + math_ops.equal(predictions, labels), dtypes.float32) if weights is not None: if weights.get_shape().ndims > 1: @@ -1079,13 +1082,18 @@ def mean_per_class_accuracy( update_count_op = state_ops.scatter_add(count, labels, is_correct) def compute_mean_accuracy(_, count, total): - per_class_accuracy = math_ops.div_no_nan(count, math_ops.maximum(total, 0), name=None) - mean_accuracy_v = math_ops.reduce_mean(per_class_accuracy, name='mean_accuracy') + per_class_accuracy = math_ops.div_no_nan( + count, math_ops.maximum(total, 0), name=None) + mean_accuracy_v = math_ops.reduce_mean( + per_class_accuracy, name='mean_accuracy') return mean_accuracy_v - mean_accuracy_v = _aggregate_across_replicas(metrics_collections, compute_mean_accuracy, count, total) + mean_accuracy_v = _aggregate_across_replicas(metrics_collections, + compute_mean_accuracy, count, + total) - update_op = math_ops.div_no_nan(update_count_op, math_ops.maximum(update_total_op, 0), name='update_op') + update_op = math_ops.div_no_nan( + update_count_op, math_ops.maximum(update_total_op, 0), name='update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1093,15 +1101,13 @@ def compute_mean_accuracy(_, count, total): @tf_export(v1=['metrics.mean_iou']) -def mean_iou( - labels, - predictions, - num_classes, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_iou(labels, + predictions, + num_classes, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Calculate per-step mean Intersection-Over-Union (mIOU). Mean Intersection-Over-Union is a common evaluation metric for @@ -1147,45 +1153,49 @@ def mean_iou( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_iou is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_iou is not supported when ' + 'eager execution is enabled.') - with variable_scope.variable_scope(name, 'mean_iou', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'mean_iou', + (predictions, labels, weights)): # Check if shape is compatible. predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - total_cm, update_op = _streaming_confusion_matrix(labels, predictions, num_classes, weights) + total_cm, update_op = _streaming_confusion_matrix(labels, predictions, + num_classes, weights) def compute_mean_iou(_, total_cm): """Compute the mean intersection-over-union via the confusion matrix.""" - sum_over_row = math_ops.cast(math_ops.reduce_sum(total_cm, 0), dtypes.float32) - sum_over_col = math_ops.cast(math_ops.reduce_sum(total_cm, 1), dtypes.float32) + sum_over_row = math_ops.cast( + math_ops.reduce_sum(total_cm, 0), dtypes.float32) + sum_over_col = math_ops.cast( + math_ops.reduce_sum(total_cm, 1), dtypes.float32) cm_diag = math_ops.cast(array_ops.diag_part(total_cm), dtypes.float32) denominator = sum_over_row + sum_over_col - cm_diag # The mean is only computed over classes that appear in the # label or prediction tensor. If the denominator is 0, we need to # ignore the class. - num_valid_entries = math_ops.reduce_sum(math_ops.cast(math_ops.not_equal(denominator, 0), dtype=dtypes.float32)) + num_valid_entries = math_ops.reduce_sum( + math_ops.cast( + math_ops.not_equal(denominator, 0), dtype=dtypes.float32)) # If the value of the denominator is 0, set it to 1 to avoid # zero division. denominator = array_ops.where( - math_ops.greater(denominator, 0), - denominator, - array_ops.ones_like(denominator), - ) + math_ops.greater(denominator, 0), denominator, + array_ops.ones_like(denominator)) iou = math_ops.div(cm_diag, denominator) # If the number of valid entries is 0 (no classes) we return 0. result = array_ops.where( - math_ops.greater(num_valid_entries, 0), - math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, - 0, - ) + math_ops.greater(num_valid_entries, 0), + math_ops.reduce_sum(iou, name='mean_iou') / num_valid_entries, 0) return result # TODO(priyag): Use outside_compilation if in TPU context. - mean_iou_v = _aggregate_across_replicas(metrics_collections, compute_mean_iou, total_cm) + mean_iou_v = _aggregate_across_replicas(metrics_collections, + compute_mean_iou, total_cm) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1194,15 +1204,13 @@ def compute_mean_iou(_, total_cm): @tf_export(v1=['metrics.mean_relative_error']) -def mean_relative_error( - labels, - predictions, - normalizer, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_relative_error(labels, + predictions, + normalizer, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the mean relative error by normalizing with the given values. The `mean_relative_error` function creates two local variables, @@ -1248,35 +1256,29 @@ def mean_relative_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_relative_error is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_relative_error is not supported when ' + 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) - predictions, normalizer = confusion_matrix.remove_squeezable_dimensions(predictions, normalizer) + predictions, normalizer = confusion_matrix.remove_squeezable_dimensions( + predictions, normalizer) predictions.get_shape().assert_is_compatible_with(normalizer.get_shape()) relative_errors = array_ops.where( - math_ops.equal(normalizer, 0.0), - array_ops.zeros_like(labels), - math_ops.div(math_ops.abs(labels - predictions), normalizer), - ) - return mean( - relative_errors, - weights, - metrics_collections, - updates_collections, - name or 'mean_relative_error', - ) + math_ops.equal(normalizer, 0.0), array_ops.zeros_like(labels), + math_ops.div(math_ops.abs(labels - predictions), normalizer)) + return mean(relative_errors, weights, metrics_collections, + updates_collections, name or 'mean_relative_error') @tf_export(v1=['metrics.mean_squared_error']) -def mean_squared_error( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def mean_squared_error(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the mean squared error between the labels and predictions. The `mean_squared_error` function creates two local variables, @@ -1321,21 +1323,22 @@ def mean_squared_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_squared_error is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_squared_error is not supported when ' + 'eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) squared_error = math_ops.squared_difference(labels, predictions) - return mean( - squared_error, - weights, - metrics_collections, - updates_collections, - name or 'mean_squared_error', - ) + return mean(squared_error, weights, metrics_collections, updates_collections, + name or 'mean_squared_error') @tf_export(v1=['metrics.mean_tensor']) -def mean_tensor(values, weights=None, metrics_collections=None, updates_collections=None, name=None): +def mean_tensor(values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the element-wise (weighted) mean of the given tensors. In contrast to the `mean` function which returns a scalar with the @@ -1378,29 +1381,38 @@ def mean_tensor(values, weights=None, metrics_collections=None, updates_collecti RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.mean_tensor is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.mean_tensor is not supported when ' + 'eager execution is enabled.') with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.cast(values, dtypes.float32) - total = metric_variable(values.get_shape(), dtypes.float32, name='total_tensor') - count = metric_variable(values.get_shape(), dtypes.float32, name='count_tensor') + total = metric_variable( + values.get_shape(), dtypes.float32, name='total_tensor') + count = metric_variable( + values.get_shape(), dtypes.float32, name='count_tensor') num_values = array_ops.ones_like(values) if weights is not None: - values, _, weights = _remove_squeezable_dimensions(predictions=values, labels=None, weights=weights) - weights = weights_broadcast_ops.broadcast_weights(math_ops.cast(weights, dtypes.float32), values) + values, _, weights = _remove_squeezable_dimensions( + predictions=values, labels=None, weights=weights) + weights = weights_broadcast_ops.broadcast_weights( + math_ops.cast(weights, dtypes.float32), values) values = math_ops.multiply(values, weights) num_values = math_ops.multiply(num_values, weights) update_total_op = state_ops.assign_add(total, values, use_locking=True) with ops.control_dependencies([values]): - update_count_op = state_ops.assign_add(count, num_values, use_locking=True) + update_count_op = state_ops.assign_add( + count, num_values, use_locking=True) - compute_mean = lambda _, t, c: math_ops.div_no_nan(t, math_ops.maximum(c, 0), name='value') # noqa: E731 + compute_mean = lambda _, t, c: math_ops.div_no_nan( # noqa: E731 + t, math_ops.maximum(c, 0), name='value') - mean_t = _aggregate_across_replicas(metrics_collections, compute_mean, total, count) + mean_t = _aggregate_across_replicas(metrics_collections, compute_mean, + total, count) - update_op = math_ops.div_no_nan(update_total_op, math_ops.maximum(update_count_op, 0), name='update_op') + update_op = math_ops.div_no_nan( + update_total_op, math_ops.maximum(update_count_op, 0), name='update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1408,14 +1420,12 @@ def mean_tensor(values, weights=None, metrics_collections=None, updates_collecti @tf_export(v1=['metrics.percentage_below']) -def percentage_below( - values, - threshold, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def percentage_below(values, + threshold, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the percentage of values less than the given threshold. The `percentage_below` function creates two local variables, @@ -1455,19 +1465,19 @@ def percentage_below( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.percentage_below is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.percentage_below is not supported when ' + 'eager execution is enabled.') - is_below_threshold = math_ops.cast(math_ops.less(values, threshold), dtypes.float32) - return mean( - is_below_threshold, - weights, - metrics_collections, - updates_collections, - name or 'percentage_below_threshold', - ) + is_below_threshold = math_ops.cast( + math_ops.less(values, threshold), dtypes.float32) + return mean(is_below_threshold, weights, metrics_collections, + updates_collections, name or 'percentage_below_threshold') -def _count_condition(values, weights=None, metrics_collections=None, updates_collections=None): +def _count_condition(values, + weights=None, + metrics_collections=None, + updates_collections=None): """Sums the weights of cases where the given values are True. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1496,13 +1506,15 @@ def _count_condition(values, weights=None, metrics_collections=None, updates_col values = math_ops.cast(values, dtypes.float32) if weights is not None: - with ops.control_dependencies((check_ops.assert_rank_in(weights, (0, array_ops.rank(values))),)): + with ops.control_dependencies( + (check_ops.assert_rank_in(weights, (0, array_ops.rank(values))),)): weights = math_ops.cast(weights, dtypes.float32) values = math_ops.multiply(values, weights) value_tensor = _aggregate_variable(count, metrics_collections) - update_op = state_ops.assign_add(count, math_ops.reduce_sum(values), use_locking=True) + update_op = state_ops.assign_add( + count, math_ops.reduce_sum(values), use_locking=True) if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -1510,14 +1522,12 @@ def _count_condition(values, weights=None, metrics_collections=None, updates_col @tf_export(v1=['metrics.false_negatives']) -def false_negatives( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def false_negatives(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the total number of false negatives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1547,28 +1557,30 @@ def false_negatives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_negatives is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.false_negatives is not supported when ' + 'eager execution is enabled.') + + with variable_scope.variable_scope(name, 'false_negatives', + (predictions, labels, weights)): - with variable_scope.variable_scope(name, 'false_negatives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) - is_false_negative = math_ops.logical_and(math_ops.equal(labels, True), math_ops.equal(predictions, False)) - return _count_condition(is_false_negative, weights, metrics_collections, updates_collections) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) + is_false_negative = math_ops.logical_and( + math_ops.equal(labels, True), math_ops.equal(predictions, False)) + return _count_condition(is_false_negative, weights, metrics_collections, + updates_collections) @tf_export(v1=['metrics.false_negatives_at_thresholds']) -def false_negatives_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def false_negatives_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes false negatives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1601,12 +1613,13 @@ def false_negatives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_negatives_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.false_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'false_negatives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'false_negatives', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fn',) - ) + labels, predictions, thresholds, weights=weights, includes=('fn',)) fn_value = _aggregate_variable(values['fn'], metrics_collections) @@ -1617,14 +1630,12 @@ def false_negatives_at_thresholds( @tf_export(v1=['metrics.false_positives']) -def false_positives( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def false_positives(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Sum the weights of false positives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1655,28 +1666,30 @@ def false_positives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_positives is not supported when ' 'eager execution is enabled.') + raise RuntimeError('tf.metrics.false_positives is not supported when ' + 'eager execution is enabled.') + + with variable_scope.variable_scope(name, 'false_positives', + (predictions, labels, weights)): - with variable_scope.variable_scope(name, 'false_positives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) - is_false_positive = math_ops.logical_and(math_ops.equal(labels, False), math_ops.equal(predictions, True)) - return _count_condition(is_false_positive, weights, metrics_collections, updates_collections) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) + is_false_positive = math_ops.logical_and( + math_ops.equal(labels, False), math_ops.equal(predictions, True)) + return _count_condition(is_false_positive, weights, metrics_collections, + updates_collections) @tf_export(v1=['metrics.false_positives_at_thresholds']) -def false_positives_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def false_positives_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes false positives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1709,12 +1722,13 @@ def false_positives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.false_positives_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.false_positives_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'false_positives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'false_positives', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('fp',) - ) + labels, predictions, thresholds, weights=weights, includes=('fp',)) fp_value = _aggregate_variable(values['fp'], metrics_collections) @@ -1725,14 +1739,12 @@ def false_positives_at_thresholds( @tf_export(v1=['metrics.true_negatives']) -def true_negatives( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def true_negatives(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Sum the weights of true_negatives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1763,28 +1775,30 @@ def true_negatives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_negatives is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.true_negatives is not ' + 'supported when eager execution is enabled.') + + with variable_scope.variable_scope(name, 'true_negatives', + (predictions, labels, weights)): - with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) - is_true_negative = math_ops.logical_and(math_ops.equal(labels, False), math_ops.equal(predictions, False)) - return _count_condition(is_true_negative, weights, metrics_collections, updates_collections) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) + is_true_negative = math_ops.logical_and( + math_ops.equal(labels, False), math_ops.equal(predictions, False)) + return _count_condition(is_true_negative, weights, metrics_collections, + updates_collections) @tf_export(v1=['metrics.true_negatives_at_thresholds']) -def true_negatives_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def true_negatives_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes true negatives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1817,12 +1831,13 @@ def true_negatives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_negatives_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.true_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'true_negatives', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tn',) - ) + labels, predictions, thresholds, weights=weights, includes=('tn',)) tn_value = _aggregate_variable(values['tn'], metrics_collections) @@ -1833,14 +1848,12 @@ def true_negatives_at_thresholds( @tf_export(v1=['metrics.true_positives']) -def true_positives( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def true_positives(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Sum the weights of true_positives. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1871,28 +1884,30 @@ def true_positives( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_positives is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.true_positives is not ' + 'supported when eager execution is enabled.') + + with variable_scope.variable_scope(name, 'true_positives', + (predictions, labels, weights)): - with variable_scope.variable_scope(name, 'true_positives', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) - is_true_positive = math_ops.logical_and(math_ops.equal(labels, True), math_ops.equal(predictions, True)) - return _count_condition(is_true_positive, weights, metrics_collections, updates_collections) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) + is_true_positive = math_ops.logical_and( + math_ops.equal(labels, True), math_ops.equal(predictions, True)) + return _count_condition(is_true_positive, weights, metrics_collections, + updates_collections) @tf_export(v1=['metrics.true_positives_at_thresholds']) -def true_positives_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def true_positives_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes true positives at provided threshold values. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. @@ -1925,12 +1940,13 @@ def true_positives_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.true_positives_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.true_positives_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'true_positives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'true_positives', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights=weights, includes=('tp',) - ) + labels, predictions, thresholds, weights=weights, includes=('tp',)) tp_value = _aggregate_variable(values['tp'], metrics_collections) @@ -1941,14 +1957,12 @@ def true_positives_at_thresholds( @tf_export(v1=['metrics.precision']) -def precision( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def precision(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the precision of the predictions with respect to the labels. The `precision` function creates two local variables, @@ -1993,41 +2007,44 @@ def precision( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.precision is not ' + 'supported when eager execution is enabled.') + + with variable_scope.variable_scope(name, 'precision', + (predictions, labels, weights)): - with variable_scope.variable_scope(name, 'precision', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) true_p, true_positives_update_op = true_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None, - ) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) false_p, false_positives_update_op = false_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None, - ) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) def compute_precision(tp, fp, name): - return array_ops.where(math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name) + return array_ops.where( + math_ops.greater(tp + fp, 0), math_ops.div(tp, tp + fp), 0, name) def once_across_replicas(_, true_p, false_p): return compute_precision(true_p, false_p, 'value') - p = _aggregate_across_replicas(metrics_collections, once_across_replicas, true_p, false_p) + p = _aggregate_across_replicas(metrics_collections, once_across_replicas, + true_p, false_p) - update_op = compute_precision(true_positives_update_op, false_positives_update_op, 'update_op') + update_op = compute_precision(true_positives_update_op, + false_positives_update_op, 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2035,15 +2052,13 @@ def once_across_replicas(_, true_p, false_p): @tf_export(v1=['metrics.precision_at_thresholds']) -def precision_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def precision_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes precision values for different `thresholds` on `predictions`. The `precision_at_thresholds` function creates four local variables, @@ -2089,12 +2104,13 @@ def precision_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.precision_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'precision_at_thresholds', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'precision_at_thresholds', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights, includes=('tp', 'fp') - ) + labels, predictions, thresholds, weights, includes=('tp', 'fp')) # Avoid division by zero. epsilon = 1e-7 @@ -2105,9 +2121,11 @@ def compute_precision(tp, fp, name): def precision_across_replicas(_, values): return compute_precision(values['tp'], values['fp'], 'value') - prec = _aggregate_across_replicas(metrics_collections, precision_across_replicas, values) + prec = _aggregate_across_replicas(metrics_collections, + precision_across_replicas, values) - update_op = compute_precision(update_ops['tp'], update_ops['fp'], 'update_op') + update_op = compute_precision(update_ops['tp'], update_ops['fp'], + 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2115,14 +2133,12 @@ def precision_across_replicas(_, values): @tf_export(v1=['metrics.recall']) -def recall( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def recall(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the recall of the predictions with respect to the labels. The `recall` function creates two local variables, `true_positives` @@ -2165,46 +2181,44 @@ def recall( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall is not supported is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.recall is not supported is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'recall', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'recall', + (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights, - ) + predictions=math_ops.cast(predictions, dtype=dtypes.bool), + labels=math_ops.cast(labels, dtype=dtypes.bool), + weights=weights) true_p, true_positives_update_op = true_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None, - ) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) false_n, false_negatives_update_op = false_negatives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None, - ) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) def compute_recall(true_p, false_n, name): return array_ops.where( - math_ops.greater(true_p + false_n, 0), - math_ops.div(true_p, true_p + false_n), - 0, - name, - ) + math_ops.greater(true_p + false_n, 0), + math_ops.div(true_p, true_p + false_n), 0, name) def once_across_replicas(_, true_p, false_n): return compute_recall(true_p, false_n, 'value') - rec = _aggregate_across_replicas(metrics_collections, once_across_replicas, true_p, false_n) + rec = _aggregate_across_replicas(metrics_collections, once_across_replicas, + true_p, false_n) - update_op = compute_recall(true_positives_update_op, false_negatives_update_op, 'update_op') + update_op = compute_recall(true_positives_update_op, + false_negatives_update_op, 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2234,7 +2248,8 @@ def _select_class_id(ids, selected_id): """ ids = sparse_tensor.convert_to_tensor_or_sparse_tensor(ids) if isinstance(ids, sparse_tensor.SparseTensor): - return sparse_ops.sparse_retain(ids, math_ops.equal(ids.values, selected_id)) + return sparse_ops.sparse_retain(ids, math_ops.equal(ids.values, + selected_id)) # TODO(ptucker): Make this more efficient, maybe add a sparse version of # tf.equal and tf.reduce_any? @@ -2242,12 +2257,15 @@ def _select_class_id(ids, selected_id): # Shape of filled IDs is the same as `ids` with the last dim collapsed to 1. ids_shape = array_ops.shape(ids, out_type=dtypes.int64) ids_last_dim = array_ops.size(ids_shape) - 1 - filled_selected_id_shape = math_ops.reduced_shape(ids_shape, array_ops.reshape(ids_last_dim, [1])) + filled_selected_id_shape = math_ops.reduced_shape( + ids_shape, array_ops.reshape(ids_last_dim, [1])) # Intersect `ids` with the selected ID. - filled_selected_id = array_ops.fill(filled_selected_id_shape, math_ops.cast(selected_id, dtypes.int64)) + filled_selected_id = array_ops.fill(filled_selected_id_shape, + math_ops.cast(selected_id, dtypes.int64)) result = sets.set_intersection(filled_selected_id, ids) - return sparse_tensor.SparseTensor(indices=result.indices, values=result.values, dense_shape=ids_shape) + return sparse_tensor.SparseTensor( + indices=result.indices, values=result.values, dense_shape=ids_shape) def _maybe_select_class_id(labels, predictions_idx, selected_id=None): @@ -2269,13 +2287,15 @@ def _maybe_select_class_id(labels, predictions_idx, selected_id=None): """ if selected_id is None: return labels, predictions_idx - return ( - _select_class_id(labels, selected_id), - _select_class_id(predictions_idx, selected_id), - ) + return (_select_class_id(labels, selected_id), + _select_class_id(predictions_idx, selected_id)) -def _sparse_true_positive_at_k(labels, predictions_idx, class_id=None, weights=None, name=None): +def _sparse_true_positive_at_k(labels, + predictions_idx, + class_id=None, + weights=None, + name=None): """Calculates true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2302,18 +2322,26 @@ def _sparse_true_positive_at_k(labels, predictions_idx, class_id=None, weights=N Returns: A [D1, ... DN] `Tensor` of true positive counts. """ - with ops.name_scope(name, 'true_positives', (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) + with ops.name_scope(name, 'true_positives', + (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, + class_id) tp = sets.set_size(sets.set_intersection(predictions_idx, labels)) tp = math_ops.cast(tp, dtypes.float64) if weights is not None: - with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, tp),)): + with ops.control_dependencies( + (weights_broadcast_ops.assert_broadcastable(weights, tp),)): weights = math_ops.cast(weights, dtypes.float64) tp = math_ops.multiply(tp, weights) return tp -def _streaming_sparse_true_positive_at_k(labels, predictions_idx, k=None, class_id=None, weights=None, name=None): +def _streaming_sparse_true_positive_at_k(labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + name=None): """Calculates weighted per step true positives for recall@k and precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2346,24 +2374,24 @@ def _streaming_sparse_true_positive_at_k(labels, predictions_idx, k=None, class_ Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope( - name, - _at_k_name('true_positive', k, class_id=class_id), - (predictions_idx, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('true_positive', k, class_id=class_id), + (predictions_idx, labels, weights)) as scope: tp = _sparse_true_positive_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights, - ) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights) batch_total_tp = math_ops.cast(math_ops.reduce_sum(tp), dtypes.float64) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add(var, batch_total_tp, name='update', use_locking=True) + return var, state_ops.assign_add( + var, batch_total_tp, name='update', use_locking=True) -def _sparse_false_negative_at_k(labels, predictions_idx, class_id=None, weights=None): +def _sparse_false_negative_at_k(labels, + predictions_idx, + class_id=None, + weights=None): """Calculates false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2389,18 +2417,27 @@ def _sparse_false_negative_at_k(labels, predictions_idx, class_id=None, weights= Returns: A [D1, ... DN] `Tensor` of false negative counts. """ - with ops.name_scope(None, 'false_negatives', (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) - fn = sets.set_size(sets.set_difference(predictions_idx, labels, aminusb=False)) + with ops.name_scope(None, 'false_negatives', + (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, + class_id) + fn = sets.set_size( + sets.set_difference(predictions_idx, labels, aminusb=False)) fn = math_ops.cast(fn, dtypes.float64) if weights is not None: - with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, fn),)): + with ops.control_dependencies( + (weights_broadcast_ops.assert_broadcastable(weights, fn),)): weights = math_ops.cast(weights, dtypes.float64) fn = math_ops.multiply(fn, weights) return fn -def _streaming_sparse_false_negative_at_k(labels, predictions_idx, k, class_id=None, weights=None, name=None): +def _streaming_sparse_false_negative_at_k(labels, + predictions_idx, + k, + class_id=None, + weights=None, + name=None): """Calculates weighted per step false negatives for recall@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -2433,34 +2470,29 @@ def _streaming_sparse_false_negative_at_k(labels, predictions_idx, k, class_id=N Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope( - name, - _at_k_name('false_negative', k, class_id=class_id), - (predictions_idx, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('false_negative', k, class_id=class_id), + (predictions_idx, labels, weights)) as scope: fn = _sparse_false_negative_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights, - ) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights) batch_total_fn = math_ops.cast(math_ops.reduce_sum(fn), dtypes.float64) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add(var, batch_total_fn, name='update', use_locking=True) + return var, state_ops.assign_add( + var, batch_total_fn, name='update', use_locking=True) @tf_export(v1=['metrics.recall_at_k']) -def recall_at_k( - labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def recall_at_k(labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes recall@k of the predictions with respect to sparse labels. If `class_id` is specified, we calculate recall by considering only the @@ -2528,33 +2560,32 @@ def recall_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall_at_k is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.recall_at_k is not ' + 'supported when eager execution is enabled.') - with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), (predictions, labels, weights)) as scope: + with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), + (predictions, labels, weights)) as scope: _, top_k_idx = nn.top_k(predictions, k) return recall_at_top_k( - labels=labels, - predictions_idx=top_k_idx, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope, - ) + labels=labels, + predictions_idx=top_k_idx, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope) @tf_export(v1=['metrics.recall_at_top_k']) -def recall_at_top_k( - labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def recall_at_top_k(labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes recall@k of top-k predictions with respect to sparse labels. Differs from `recall_at_k` in that predictions must be in the form of top `k` @@ -2600,49 +2631,44 @@ class indices, whereas `recall_at_k` expects logits. Refer to `recall_at_k` `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with ops.name_scope( - name, - _at_k_name('recall', k, class_id=class_id), - (predictions_idx, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('recall', k, class_id=class_id), + (predictions_idx, labels, weights)) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.cast(predictions_idx, dtypes.int64) tp, tp_update = _streaming_sparse_true_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights, - ) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights) fn, fn_update = _streaming_sparse_false_negative_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights, - ) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights) def compute_recall(_, tp, fn): return math_ops.div(tp, math_ops.add(tp, fn), name=scope) - metric = _aggregate_across_replicas(metrics_collections, compute_recall, tp, fn) + metric = _aggregate_across_replicas(metrics_collections, compute_recall, tp, + fn) - update = math_ops.div(tp_update, math_ops.add(tp_update, fn_update), name='update') + update = math_ops.div( + tp_update, math_ops.add(tp_update, fn_update), name='update') if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @tf_export(v1=['metrics.recall_at_thresholds']) -def recall_at_thresholds( - labels, - predictions, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def recall_at_thresholds(labels, + predictions, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes various recall values for different `thresholds` on `predictions`. The `recall_at_thresholds` function creates four local variables, @@ -2686,12 +2712,13 @@ def recall_at_thresholds( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.recall_at_thresholds is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.recall_at_thresholds is not ' + 'supported when eager execution is enabled.') - with variable_scope.variable_scope(name, 'recall_at_thresholds', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'recall_at_thresholds', + (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( - labels, predictions, thresholds, weights, includes=('tp', 'fn') - ) + labels, predictions, thresholds, weights, includes=('tp', 'fn')) # Avoid division by zero. epsilon = 1e-7 @@ -2702,7 +2729,8 @@ def compute_recall(tp, fn, name): def recall_across_replicas(_, values): return compute_recall(values['tp'], values['fn'], 'value') - rec = _aggregate_across_replicas(metrics_collections, recall_across_replicas, values) + rec = _aggregate_across_replicas(metrics_collections, + recall_across_replicas, values) update_op = compute_recall(update_ops['tp'], update_ops['fn'], 'update_op') if updates_collections: @@ -2712,14 +2740,12 @@ def recall_across_replicas(_, values): @tf_export(v1=['metrics.root_mean_squared_error']) -def root_mean_squared_error( - labels, - predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def root_mean_squared_error(labels, + predictions, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the root mean squared error between the labels and predictions. The `root_mean_squared_error` function creates two local variables, @@ -2764,13 +2790,18 @@ def root_mean_squared_error( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.root_mean_squared_error is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.root_mean_squared_error is not ' + 'supported when eager execution is enabled.') - predictions, labels, weights = _remove_squeezable_dimensions(predictions=predictions, labels=labels, weights=weights) - mse, update_mse_op = mean_squared_error(labels, predictions, weights, None, None, name or 'root_mean_squared_error') + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) + mse, update_mse_op = mean_squared_error(labels, predictions, weights, None, + None, name or + 'root_mean_squared_error') once_across_replicas = lambda _, mse: math_ops.sqrt(mse) # noqa: E731 - rmse = _aggregate_across_replicas(metrics_collections, once_across_replicas, mse) + rmse = _aggregate_across_replicas(metrics_collections, once_across_replicas, + mse) update_rmse_op = math_ops.sqrt(update_mse_op) if updates_collections: @@ -2780,16 +2811,14 @@ def root_mean_squared_error( @tf_export(v1=['metrics.sensitivity_at_specificity']) -def sensitivity_at_specificity( - labels, - predictions, - specificity, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None, -): +def sensitivity_at_specificity(labels, + predictions, + specificity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the specificity at a given sensitivity. The `sensitivity_at_specificity` function creates four local @@ -2841,17 +2870,22 @@ def sensitivity_at_specificity( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sensitivity_at_specificity is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.sensitivity_at_specificity is not ' + 'supported when eager execution is enabled.') if specificity < 0 or specificity > 1: raise ValueError('`specificity` must be in the range [0, 1].') - with variable_scope.variable_scope(name, 'sensitivity_at_specificity', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'sensitivity_at_specificity', + (predictions, labels, weights)): kepsilon = 1e-7 # to account for floating point imprecisions - thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] + thresholds = [ + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] - values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) + values, update_ops = _confusion_matrix_at_thresholds( + labels, predictions, thresholds, weights) def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): specificities = math_ops.div(tn, tn + fp + kepsilon) @@ -2859,20 +2893,23 @@ def compute_sensitivity_at_specificity(tp, tn, fp, fn, name): tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the sensitivity: - return math_ops.div(tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, name) + return math_ops.div(tp[tf_index], tp[tf_index] + fn[tf_index] + kepsilon, + name) def sensitivity_across_replicas(_, values): - return compute_sensitivity_at_specificity(values['tp'], values['tn'], values['fp'], values['fn'], 'value') - - sensitivity = _aggregate_across_replicas(metrics_collections, sensitivity_across_replicas, values) - - update_op = compute_sensitivity_at_specificity( - update_ops['tp'], - update_ops['tn'], - update_ops['fp'], - update_ops['fn'], - 'update_op', - ) + return compute_sensitivity_at_specificity(values['tp'], values['tn'], + values['fp'], values['fn'], + 'value') + + sensitivity = _aggregate_across_replicas(metrics_collections, + sensitivity_across_replicas, + values) + + update_op = compute_sensitivity_at_specificity(update_ops['tp'], + update_ops['tn'], + update_ops['fp'], + update_ops['fn'], + 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -2900,34 +2937,37 @@ def _expand_and_tile(tensor, multiple, dim=0, name=None): """ if multiple < 1: raise ValueError('Invalid multiple %s, must be > 0.' % multiple) - with ops.name_scope(name, 'expand_and_tile', (tensor, multiple, dim)) as scope: + with ops.name_scope(name, 'expand_and_tile', + (tensor, multiple, dim)) as scope: # Sparse. tensor = sparse_tensor.convert_to_tensor_or_sparse_tensor(tensor) if isinstance(tensor, sparse_tensor.SparseTensor): if dim < 0: - expand_dims = array_ops.reshape(array_ops.size(tensor.dense_shape) + dim, [1]) + expand_dims = array_ops.reshape( + array_ops.size(tensor.dense_shape) + dim, [1]) else: expand_dims = [dim] expanded_shape = array_ops.concat( - ( - array_ops.slice(tensor.dense_shape, [0], expand_dims), - [1], - array_ops.slice(tensor.dense_shape, expand_dims, [-1]), - ), - 0, - name='expanded_shape', - ) - expanded = sparse_ops.sparse_reshape(tensor, shape=expanded_shape, name='expand') + (array_ops.slice(tensor.dense_shape, [0], expand_dims), [1], + array_ops.slice(tensor.dense_shape, expand_dims, [-1])), + 0, + name='expanded_shape') + expanded = sparse_ops.sparse_reshape( + tensor, shape=expanded_shape, name='expand') if multiple == 1: return expanded - return sparse_ops.sparse_concat(dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope) + return sparse_ops.sparse_concat( + dim - 1 if dim < 0 else dim, [expanded] * multiple, name=scope) # Dense. - expanded = array_ops.expand_dims(tensor, dim if (dim >= 0) else (dim - 1), name='expand') + expanded = array_ops.expand_dims( + tensor, dim if (dim >= 0) else (dim - 1), name='expand') if multiple == 1: return expanded ones = array_ops.ones_like(array_ops.shape(tensor)) - tile_multiples = array_ops.concat((ones[:dim], (multiple,), ones[dim:]), 0, name='multiples') + tile_multiples = array_ops.concat((ones[:dim], (multiple,), ones[dim:]), + 0, + name='multiples') return array_ops.tile(expanded, tile_multiples, name=scope) @@ -2962,13 +3002,10 @@ def _num_relevant(labels, k): # The relevant values for each (d1, ... dN) is the minimum of k and the # number of labels along the last dimension that are non-negative. num_labels = math_ops.reduce_sum( - array_ops.where_v2( - math_ops.greater_equal(labels, 0), - array_ops.ones_like(labels), - array_ops.zeros_like(labels), - ), - axis=-1, - ) + array_ops.where_v2( + math_ops.greater_equal(labels, 0), array_ops.ones_like(labels), + array_ops.zeros_like(labels)), + axis=-1) return math_ops.minimum(num_labels, k, name=scope) @@ -3004,8 +3041,10 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): Raises: ValueError: if the last dimension of predictions_idx is not set. """ - with ops.name_scope(None, 'average_precision', (predictions_idx, labels)) as scope: - predictions_idx = math_ops.cast(predictions_idx, dtypes.int64, name='predictions_idx') + with ops.name_scope(None, 'average_precision', + (predictions_idx, labels)) as scope: + predictions_idx = math_ops.cast( + predictions_idx, dtypes.int64, name='predictions_idx') if predictions_idx.get_shape().ndims == 0: raise ValueError('The rank of predictions_idx must be at least 1.') k = predictions_idx.get_shape().as_list()[-1] @@ -3016,10 +3055,12 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # Expand dims to produce [D1, ... DN, k, 1] tensor. This gives us a separate # prediction for each k, so we can calculate separate true positive values # for each k. - predictions_idx_per_k = array_ops.expand_dims(predictions_idx, -1, name='predictions_idx_per_k') + predictions_idx_per_k = array_ops.expand_dims( + predictions_idx, -1, name='predictions_idx_per_k') # Replicate labels k times to produce [D1, ... DN, k, num_labels] tensor. - labels_per_k = _expand_and_tile(labels, multiple=k, dim=-1, name='labels_per_k') + labels_per_k = _expand_and_tile( + labels, multiple=k, dim=-1, name='labels_per_k') # The following tensors are all of shape [D1, ... DN, k], containing values # per row, per k value. @@ -3033,22 +3074,23 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # term from the formula above. # `relevant_precision_per_k` (float64) - Relevant precisions; i.e., # precisions at all k for which relevance indicator is true. - relevant_per_k = _sparse_true_positive_at_k(labels_per_k, predictions_idx_per_k, name='relevant_per_k') + relevant_per_k = _sparse_true_positive_at_k( + labels_per_k, predictions_idx_per_k, name='relevant_per_k') tp_per_k = math_ops.cumsum(relevant_per_k, axis=-1, name='tp_per_k') - retrieved_per_k = math_ops.cumsum(array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k') + retrieved_per_k = math_ops.cumsum( + array_ops.ones_like(relevant_per_k), axis=-1, name='retrieved_per_k') precision_per_k = math_ops.div( - math_ops.cast(tp_per_k, dtypes.float64), - math_ops.cast(retrieved_per_k, dtypes.float64), - name='precision_per_k', - ) + math_ops.cast(tp_per_k, dtypes.float64), + math_ops.cast(retrieved_per_k, dtypes.float64), + name='precision_per_k') relevant_precision_per_k = math_ops.multiply( - precision_per_k, - math_ops.cast(relevant_per_k, dtypes.float64), - name='relevant_precision_per_k', - ) + precision_per_k, + math_ops.cast(relevant_per_k, dtypes.float64), + name='relevant_precision_per_k') # Reduce along k dimension to get the sum, yielding a [D1, ... DN] tensor. - precision_sum = math_ops.reduce_sum(relevant_precision_per_k, axis=(-1,), name='precision_sum') + precision_sum = math_ops.reduce_sum( + relevant_precision_per_k, axis=(-1,), name='precision_sum') # Divide by number of relevant items to get average precision. These are # the "num_relevant_items" and "AveP" terms from the formula above. @@ -3056,14 +3098,12 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): return math_ops.div(precision_sum, num_relevant_items, name=scope) -def _streaming_sparse_average_precision_at_top_k( - labels, - predictions_idx, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def _streaming_sparse_average_precision_at_top_k(labels, + predictions_idx, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes average precision@k of predictions with respect to sparse labels. `sparse_average_precision_at_top_k` creates two local variables, @@ -3108,11 +3148,14 @@ def _streaming_sparse_average_precision_at_top_k( update: `Operation` that increments variables appropriately, and whose value matches `metric`. """ - with ops.name_scope(name, 'average_precision_at_top_k', (predictions_idx, labels, weights)) as scope: + with ops.name_scope(name, 'average_precision_at_top_k', + (predictions_idx, labels, weights)) as scope: # Calculate per-example average precision, and apply weights. - average_precision = _sparse_average_precision_at_top_k(predictions_idx=predictions_idx, labels=labels) + average_precision = _sparse_average_precision_at_top_k( + predictions_idx=predictions_idx, labels=labels) if weights is not None: - weights = weights_broadcast_ops.broadcast_weights(math_ops.cast(weights, dtypes.float64), average_precision) + weights = weights_broadcast_ops.broadcast_weights( + math_ops.cast(weights, dtypes.float64), average_precision) average_precision = math_ops.multiply(average_precision, weights) # Create accumulation variables and update ops for max average precision and @@ -3124,22 +3167,24 @@ def _streaming_sparse_average_precision_at_top_k( # `average_precision` rows. max_var = metric_variable([], dtypes.float64, name=max_scope) if weights is None: - batch_max = math_ops.cast(array_ops.size(average_precision, name='batch_max'), dtypes.float64) + batch_max = math_ops.cast( + array_ops.size(average_precision, name='batch_max'), dtypes.float64) else: batch_max = math_ops.reduce_sum(weights, name='batch_max') - max_update = state_ops.assign_add(max_var, batch_max, name='update', use_locking=True) + max_update = state_ops.assign_add( + max_var, batch_max, name='update', use_locking=True) with ops.name_scope(None, 'total', (average_precision,)) as total_scope: total_var = metric_variable([], dtypes.float64, name=total_scope) batch_total = math_ops.reduce_sum(average_precision, name='batch_total') - total_update = state_ops.assign_add(total_var, batch_total, name='update', use_locking=True) + total_update = state_ops.assign_add( + total_var, batch_total, name='update', use_locking=True) # Divide total by max to get mean, for both vars and the update ops. def precision_across_replicas(_, total_var, max_var): return _safe_scalar_div(total_var, max_var, name='mean') mean_average_precision = _aggregate_across_replicas( - metrics_collections, precision_across_replicas, total_var, max_var - ) + metrics_collections, precision_across_replicas, total_var, max_var) update = _safe_scalar_div(total_update, max_update, name=scope) if updates_collections: @@ -3165,68 +3210,60 @@ def _clean_out_of_range_indices(labels, num_classes): def _labels_is_sparse(): """Returns true is `labels` is a sparse tensor.""" - return isinstance(labels, (sparse_tensor.SparseTensor, sparse_tensor.SparseTensorValue)) + return isinstance( + labels, (sparse_tensor.SparseTensor, sparse_tensor.SparseTensorValue)) def _clean_out_of_range(values): """Replaces by -1 any large out-of-range `values`.""" return array_ops.where_v2( - math_ops.greater_equal(values, num_classes), - -1 * array_ops.ones_like(values), - values, - ) + math_ops.greater_equal(values, num_classes), + -1 * array_ops.ones_like(values), values) def _clean_labels_out_of_range(): """Replaces by -1 ane large out-of-range values in `labels`.""" if _labels_is_sparse(): return type(labels)( - indices=labels.indices, - values=_clean_out_of_range(labels.values), - dense_shape=labels.dense_shape, - ) + indices=labels.indices, + values=_clean_out_of_range(labels.values), + dense_shape=labels.dense_shape) else: return _clean_out_of_range(labels) - max_labels = math_ops.reduce_max(labels.values if _labels_is_sparse() else labels) + max_labels = math_ops.reduce_max( + labels.values if _labels_is_sparse() else labels) return control_flow_ops.cond( - math_ops.greater_equal(max_labels, num_classes), - _clean_labels_out_of_range, - lambda: labels, - ) + math_ops.greater_equal(max_labels, num_classes), + _clean_labels_out_of_range, lambda: labels) @tf_export(v1=['metrics.sparse_average_precision_at_k']) @deprecated(None, 'Use average_precision_at_k instead') -def sparse_average_precision_at_k( - labels, - predictions, - k, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def sparse_average_precision_at_k(labels, + predictions, + k, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Renamed to `average_precision_at_k`, please use that method instead.""" return average_precision_at_k( - labels=labels, - predictions=predictions, - k=k, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name, - ) + labels=labels, + predictions=predictions, + k=k, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) @tf_export(v1=['metrics.average_precision_at_k']) -def average_precision_at_k( - labels, - predictions, - k, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def average_precision_at_k(labels, + predictions, + k, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes average precision@k of predictions with respect to sparse labels. `average_precision_at_k` creates two local variables, @@ -3280,29 +3317,34 @@ def average_precision_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sparse_average_precision_at_k is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.sparse_average_precision_at_k is not ' + 'supported when eager execution is enabled.') if k < 1: raise ValueError('Invalid k=%s.' % k) - with ops.name_scope(name, _at_k_name('average_precision', k), (predictions, labels, weights)) as scope: + with ops.name_scope(name, _at_k_name('average_precision', k), + (predictions, labels, weights)) as scope: # Calculate top k indices to produce [D1, ... DN, k] tensor. _, predictions_idx = nn.top_k(predictions, k) # The documentation states that labels should be in [0, ..., num_classes), # but num_classes is lost when predictions_idx replaces predictions. # For conformity with the documentation, any label >= num_classes, which is # ignored, is replaced by -1. - labels = _clean_out_of_range_indices(labels, math_ops.cast(array_ops.shape(predictions)[-1], dtypes.int64)) + labels = _clean_out_of_range_indices( + labels, math_ops.cast(array_ops.shape(predictions)[-1], dtypes.int64)) return _streaming_sparse_average_precision_at_top_k( - labels=labels, - predictions_idx=predictions_idx, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope, - ) - - -def _sparse_false_positive_at_k(labels, predictions_idx, class_id=None, weights=None): + labels=labels, + predictions_idx=predictions_idx, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope) + + +def _sparse_false_positive_at_k(labels, + predictions_idx, + class_id=None, + weights=None): """Calculates false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3328,18 +3370,27 @@ def _sparse_false_positive_at_k(labels, predictions_idx, class_id=None, weights= Returns: A [D1, ... DN] `Tensor` of false positive counts. """ - with ops.name_scope(None, 'false_positives', (predictions_idx, labels, weights)): - labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, class_id) - fp = sets.set_size(sets.set_difference(predictions_idx, labels, aminusb=True)) + with ops.name_scope(None, 'false_positives', + (predictions_idx, labels, weights)): + labels, predictions_idx = _maybe_select_class_id(labels, predictions_idx, + class_id) + fp = sets.set_size( + sets.set_difference(predictions_idx, labels, aminusb=True)) fp = math_ops.cast(fp, dtypes.float64) if weights is not None: - with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, fp),)): + with ops.control_dependencies( + (weights_broadcast_ops.assert_broadcastable(weights, fp),)): weights = math_ops.cast(weights, dtypes.float64) fp = math_ops.multiply(fp, weights) return fp -def _streaming_sparse_false_positive_at_k(labels, predictions_idx, k=None, class_id=None, weights=None, name=None): +def _streaming_sparse_false_positive_at_k(labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + name=None): """Calculates weighted per step false positives for precision@k. If `class_id` is specified, calculate binary true positives for `class_id` @@ -3372,34 +3423,29 @@ def _streaming_sparse_false_positive_at_k(labels, predictions_idx, k=None, class Raises: ValueError: If `weights` is not `None` and has an incompatible shape. """ - with ops.name_scope( - name, - _at_k_name('false_positive', k, class_id=class_id), - (predictions_idx, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('false_positive', k, class_id=class_id), + (predictions_idx, labels, weights)) as scope: fp = _sparse_false_positive_at_k( - predictions_idx=predictions_idx, - labels=labels, - class_id=class_id, - weights=weights, - ) + predictions_idx=predictions_idx, + labels=labels, + class_id=class_id, + weights=weights) batch_total_fp = math_ops.cast(math_ops.reduce_sum(fp), dtypes.float64) var = metric_variable([], dtypes.float64, name=scope) - return var, state_ops.assign_add(var, batch_total_fp, name='update', use_locking=True) + return var, state_ops.assign_add( + var, batch_total_fp, name='update', use_locking=True) @tf_export(v1=['metrics.precision_at_top_k']) -def precision_at_top_k( - labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def precision_at_top_k(labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes precision@k of the predictions with respect to sparse labels. Differs from `sparse_precision_at_k` in that predictions must be in the form @@ -3447,36 +3493,34 @@ def precision_at_top_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision_at_top_k is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.precision_at_top_k is not ' + 'supported when eager execution is enabled.') - with ops.name_scope( - name, - _at_k_name('precision', k, class_id=class_id), - (predictions_idx, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), + (predictions_idx, labels, weights)) as scope: labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.cast(predictions_idx, dtypes.int64) tp, tp_update = _streaming_sparse_true_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights, - ) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights) fp, fp_update = _streaming_sparse_false_positive_at_k( - predictions_idx=top_k_idx, - labels=labels, - k=k, - class_id=class_id, - weights=weights, - ) + predictions_idx=top_k_idx, + labels=labels, + k=k, + class_id=class_id, + weights=weights) def precision_across_replicas(_, tp, fp): return math_ops.div(tp, math_ops.add(tp, fp), name=scope) - metric = _aggregate_across_replicas(metrics_collections, precision_across_replicas, tp, fp) + metric = _aggregate_across_replicas(metrics_collections, + precision_across_replicas, tp, fp) - update = math_ops.div(tp_update, math_ops.add(tp_update, fp_update), name='update') + update = math_ops.div( + tp_update, math_ops.add(tp_update, fp_update), name='update') if updates_collections: ops.add_to_collections(updates_collections, update) return metric, update @@ -3484,40 +3528,35 @@ def precision_across_replicas(_, tp, fp): @tf_export(v1=['metrics.sparse_precision_at_k']) @deprecated(None, 'Use precision_at_k instead') -def sparse_precision_at_k( - labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def sparse_precision_at_k(labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Renamed to `precision_at_k`, please use that method instead.""" return precision_at_k( - labels=labels, - predictions=predictions, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name, - ) + labels=labels, + predictions=predictions, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) @tf_export(v1=['metrics.precision_at_k']) -def precision_at_k( - labels, - predictions, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None, -): +def precision_at_k(labels, + predictions, + k, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes precision@k of the predictions with respect to sparse labels. If `class_id` is specified, we calculate precision by considering only the @@ -3586,37 +3625,32 @@ def precision_at_k( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.sparse_precision_at_k is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.sparse_precision_at_k is not ' + 'supported when eager execution is enabled.') - with ops.name_scope( - name, - _at_k_name('precision', k, class_id=class_id), - (predictions, labels, weights), - ) as scope: + with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), + (predictions, labels, weights)) as scope: _, top_k_idx = nn.top_k(predictions, k) return precision_at_top_k( - labels=labels, - predictions_idx=top_k_idx, - k=k, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=scope, - ) + labels=labels, + predictions_idx=top_k_idx, + k=k, + class_id=class_id, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=scope) @tf_export(v1=['metrics.specificity_at_sensitivity']) -def specificity_at_sensitivity( - labels, - predictions, - sensitivity, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None, -): +def specificity_at_sensitivity(labels, + predictions, + sensitivity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the specificity at a given sensitivity. The `specificity_at_sensitivity` function creates four local @@ -3668,17 +3702,22 @@ def specificity_at_sensitivity( RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): - raise RuntimeError('tf.metrics.specificity_at_sensitivity is not ' 'supported when eager execution is enabled.') + raise RuntimeError('tf.metrics.specificity_at_sensitivity is not ' + 'supported when eager execution is enabled.') if sensitivity < 0 or sensitivity > 1: raise ValueError('`sensitivity` must be in the range [0, 1].') - with variable_scope.variable_scope(name, 'specificity_at_sensitivity', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'specificity_at_sensitivity', + (predictions, labels, weights)): kepsilon = 1e-7 # to account for floating point imprecisions - thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] + thresholds = [ + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 - kepsilon] - values, update_ops = _confusion_matrix_at_thresholds(labels, predictions, thresholds, weights) + values, update_ops = _confusion_matrix_at_thresholds( + labels, predictions, thresholds, weights) def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): """Computes the specificity at the given sensitivity. @@ -3698,27 +3737,31 @@ def compute_specificity_at_sensitivity(tp, tn, fp, fn, name): # We'll need to use this trick until tf.argmax allows us to specify # whether we should use the first or last index in case of ties. min_val = math_ops.reduce_min(math_ops.abs(sensitivities - sensitivity)) - indices_at_minval = math_ops.equal(math_ops.abs(sensitivities - sensitivity), min_val) + indices_at_minval = math_ops.equal( + math_ops.abs(sensitivities - sensitivity), min_val) indices_at_minval = math_ops.cast(indices_at_minval, dtypes.int64) indices_at_minval = math_ops.cumsum(indices_at_minval) tf_index = math_ops.argmax(indices_at_minval, 0) tf_index = math_ops.cast(tf_index, dtypes.int32) # Now, we have the implicit threshold, so compute the specificity: - return math_ops.div(tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, name) + return math_ops.div(tn[tf_index], tn[tf_index] + fp[tf_index] + kepsilon, + name) def specificity_across_replicas(_, values): - return compute_specificity_at_sensitivity(values['tp'], values['tn'], values['fp'], values['fn'], 'value') - - specificity = _aggregate_across_replicas(metrics_collections, specificity_across_replicas, values) - - update_op = compute_specificity_at_sensitivity( - update_ops['tp'], - update_ops['tn'], - update_ops['fp'], - update_ops['fn'], - 'update_op', - ) + return compute_specificity_at_sensitivity(values['tp'], values['tn'], + values['fp'], values['fn'], + 'value') + + specificity = _aggregate_across_replicas(metrics_collections, + specificity_across_replicas, + values) + + update_op = compute_specificity_at_sensitivity(update_ops['tp'], + update_ops['tn'], + update_ops['fp'], + update_ops['fn'], + 'update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) diff --git a/easy_rec/python/core/learning_schedules.py b/easy_rec/python/core/learning_schedules.py index d1b1fa2e1..506aea3af 100644 --- a/easy_rec/python/core/learning_schedules.py +++ b/easy_rec/python/core/learning_schedules.py @@ -23,14 +23,14 @@ def exponential_decay_with_burnin( - global_step, - learning_rate_base, - learning_rate_decay_steps, - learning_rate_decay_factor, - burnin_learning_rate=0.0, - burnin_steps=0, - min_learning_rate=0.0, - staircase=True, + global_step, + learning_rate_base, + learning_rate_decay_steps, + learning_rate_decay_factor, + burnin_learning_rate=0.0, + burnin_steps=0, + min_learning_rate=0.0, + staircase=True, ): """Exponential decay schedule with burn-in period. @@ -59,32 +59,33 @@ def exponential_decay_with_burnin( burnin_rate = learning_rate_base else: slope = (learning_rate_base - burnin_learning_rate) / burnin_steps - burnin_rate = slope * tf.cast(global_step, tf.float32) + burnin_learning_rate + burnin_rate = slope * tf.cast(global_step, + tf.float32) + burnin_learning_rate post_burnin_learning_rate = tf.train.exponential_decay( - learning_rate_base, - global_step - burnin_steps, - learning_rate_decay_steps, - learning_rate_decay_factor, - staircase=staircase, + learning_rate_base, + global_step - burnin_steps, + learning_rate_decay_steps, + learning_rate_decay_factor, + staircase=staircase, ) return tf.maximum( - tf.where( - tf.less(tf.cast(global_step, tf.int32), tf.constant(burnin_steps)), - burnin_rate, - post_burnin_learning_rate, - ), - min_learning_rate, - name='learning_rate', + tf.where( + tf.less(tf.cast(global_step, tf.int32), tf.constant(burnin_steps)), + burnin_rate, + post_burnin_learning_rate, + ), + min_learning_rate, + name='learning_rate', ) def cosine_decay_with_warmup( - global_step, - learning_rate_base, - total_steps, - warmup_learning_rate=0.0, - warmup_steps=0, - hold_base_rate_steps=0, + global_step, + learning_rate_base, + total_steps, + warmup_learning_rate=0.0, + warmup_steps=0, + hold_base_rate_steps=0, ): """Cosine decay schedule with warm up period. @@ -112,32 +113,28 @@ def cosine_decay_with_warmup( or if warmup_steps is larger than total_steps. """ if learning_rate_base < warmup_learning_rate: - raise ValueError('learning_rate_base must be larger ' 'or equal to warmup_learning_rate.') + raise ValueError('learning_rate_base must be larger ' + 'or equal to warmup_learning_rate.') if total_steps < warmup_steps: raise ValueError('total_steps must be larger or equal to ' 'warmup_steps.') - learning_rate = ( - 0.5 - * learning_rate_base - * ( - 1 - + tf.cos( - np.pi - * (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps) - / float(total_steps - warmup_steps - hold_base_rate_steps) - ) - ) - ) + learning_rate = (0.5 * learning_rate_base * (1 + tf.cos( + np.pi * + (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps) / + float(total_steps - warmup_steps - hold_base_rate_steps)))) if hold_base_rate_steps > 0: learning_rate = tf.where( - global_step > warmup_steps + hold_base_rate_steps, - learning_rate, - learning_rate_base, + global_step > warmup_steps + hold_base_rate_steps, + learning_rate, + learning_rate_base, ) if warmup_steps > 0: slope = (learning_rate_base - warmup_learning_rate) / warmup_steps - warmup_rate = slope * tf.cast(global_step, tf.float32) + warmup_learning_rate - learning_rate = tf.where(global_step < warmup_steps, warmup_rate, learning_rate) - return tf.where(global_step > total_steps, 0.0, learning_rate, name='learning_rate') + warmup_rate = slope * tf.cast(global_step, + tf.float32) + warmup_learning_rate + learning_rate = tf.where(global_step < warmup_steps, warmup_rate, + learning_rate) + return tf.where( + global_step > total_steps, 0.0, learning_rate, name='learning_rate') def manual_stepping(global_step, boundaries, rates, warmup=False): @@ -168,14 +165,16 @@ def manual_stepping(global_step, boundaries, rates, warmup=False): 2. len(rates) == len(boundaries) + 1 3. boundaries[0] != 0 """ - if any([b < 0 for b in boundaries]) or any([not isinstance(b, int) for b in boundaries]): + if any([b < 0 for b in boundaries]) or any( + [not isinstance(b, int) for b in boundaries]): raise ValueError('boundaries must be a list of positive integers') if any([bnext <= b for bnext, b in zip(boundaries[1:], boundaries[:-1])]): raise ValueError('Entries in boundaries must be strictly increasing.') if any([not isinstance(r, float) for r in rates]): raise ValueError('Learning rates must be floats') if len(rates) != len(boundaries) + 1: - raise ValueError('Number of provided learning rates must exceed ' 'number of boundary points by exactly 1.') + raise ValueError('Number of provided learning rates must exceed ' + 'number of boundary points by exactly 1.') if boundaries and boundaries[0] == 0: raise ValueError('First step cannot be zero.') @@ -190,24 +189,25 @@ def manual_stepping(global_step, boundaries, rates, warmup=False): boundaries = [0] + boundaries num_boundaries = len(boundaries) rate_index = tf.reduce_max( - tf.where( - tf.greater_equal(global_step, boundaries), - list(range(num_boundaries)), - [0] * num_boundaries, - ) - ) - return tf.reduce_sum(rates * tf.one_hot(rate_index, depth=num_boundaries), name='learning_rate') + tf.where( + tf.greater_equal(global_step, boundaries), + list(range(num_boundaries)), + [0] * num_boundaries, + )) + return tf.reduce_sum( + rates * tf.one_hot(rate_index, depth=num_boundaries), + name='learning_rate') def transformer_policy( - global_step, - learning_rate, - d_model, - warmup_steps, - step_scaling_rate=1.0, - max_lr=None, - coefficient=1.0, - dtype=tf.float32, + global_step, + learning_rate, + d_model, + warmup_steps, + step_scaling_rate=1.0, + max_lr=None, + coefficient=1.0, + dtype=tf.float32, ): """Transformer's learning rate schedule. @@ -234,7 +234,8 @@ def transformer_policy( step_num *= step_scaling_rate ws *= step_scaling_rate - decay = coefficient * d_model**-0.5 * tf.minimum((step_num + 1) * ws**-1.5, (step_num + 1) ** -0.5) + decay = coefficient * d_model**-0.5 * tf.minimum((step_num + 1) * ws**-1.5, + (step_num + 1)**-0.5) new_lr = decay * learning_rate if max_lr is not None: diff --git a/easy_rec/python/core/metrics.py b/easy_rec/python/core/metrics.py index 57a2b5945..6209e807e 100644 --- a/easy_rec/python/core/metrics.py +++ b/easy_rec/python/core/metrics.py @@ -8,14 +8,15 @@ import numpy as np import tensorflow as tf from sklearn import metrics as sklearn_metrics -from tensorflow.python.ops import array_ops, math_ops, state_ops, variable_scope # NOQA from easy_rec.python.utils.estimator_utils import get_task_index_and_num +from easy_rec.python.utils.shape_utils import get_shape_list + +from tensorflow.python.ops import array_ops, math_ops, state_ops, variable_scope # NOQA + from easy_rec.python.utils.io_util import ( # NOQA - read_data_from_json_path, - save_data_to_json_path, + read_data_from_json_path, save_data_to_json_path, ) -from easy_rec.python.utils.shape_utils import get_shape_list if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -32,7 +33,9 @@ def max_f1(label, predictions): num_thresholds = 200 kepsilon = 1e-7 - thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)] + thresholds = [ + (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) + ] thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] f1_scores = [] @@ -41,9 +44,9 @@ def max_f1(label, predictions): for threshold in thresholds: pred = predictions > threshold precision, precision_update_op = metrics_tf.precision( - labels=label, predictions=pred, name='precision_%s' % threshold - ) - recall, recall_update_op = metrics_tf.recall(labels=label, predictions=pred, name='recall_%s' % threshold) + labels=label, predictions=pred, name='precision_%s' % threshold) + recall, recall_update_op = metrics_tf.recall( + labels=label, predictions=pred, name='recall_%s' % threshold) f1_score = (2 * precision * recall) / (precision + recall + 1e-12) precision_update_ops.append(precision_update_op) recall_update_ops.append(recall_update_op) @@ -69,9 +72,9 @@ def _separated_auc_impl(labels, predictions, keys, reduction='mean'): * "mean_by_positive_num": weighted mean with positive sample num of different keys """ assert reduction in [ - 'mean', - 'mean_by_sample_num', - 'mean_by_positive_num', + 'mean', + 'mean_by_sample_num', + 'mean_by_positive_num', ], 'reduction method must in mean | mean_by_sample_num | mean_by_positive_num' separated_label = defaultdict(list) separated_prediction = defaultdict(list) @@ -122,45 +125,49 @@ def value_pyfunc(pos_neg_arr, total_pos_neg): auc += (total_pos - partial_sum_pos) * pos_neg_arr[0][i] * 2 auc += pos_neg_arr[0][i] * pos_neg_arr[1][i] auc = np.double(auc) / np.double(total_pos * total_neg * 2) - logging.info( - 'fast_auc[%s]: total_pos=%d total_neg=%d total=%d' % (name, total_pos, total_neg, total_pos + total_neg) - ) + logging.info('fast_auc[%s]: total_pos=%d total_neg=%d total=%d' % + (name, total_pos, total_neg, total_pos + total_neg)) return np.float32(auc) with variable_scope.variable_scope(name_or_scope=name), tf.name_scope(name): neg_pos_var = variable_scope.get_variable( - name='neg_pos_cnt', - shape=[2, num_thresholds + 1], - trainable=False, - collections=[tf.GraphKeys.METRIC_VARIABLES], - initializer=tf.zeros_initializer(), - dtype=tf.int64, + name='neg_pos_cnt', + shape=[2, num_thresholds + 1], + trainable=False, + collections=[tf.GraphKeys.METRIC_VARIABLES], + initializer=tf.zeros_initializer(), + dtype=tf.int64, ) total_var = variable_scope.get_variable( - name='total_cnt', - shape=[2], - trainable=False, - collections=[tf.GraphKeys.METRIC_VARIABLES], - initializer=tf.zeros_initializer(), - dtype=tf.int64, + name='total_cnt', + shape=[2], + trainable=False, + collections=[tf.GraphKeys.METRIC_VARIABLES], + initializer=tf.zeros_initializer(), + dtype=tf.int64, ) pred_bins = math_ops.cast(predictions * num_thresholds, dtype=tf.int32) labels = math_ops.cast(labels, dtype=tf.int32) labels = array_ops.reshape(labels, [-1, 1]) pred_bins = array_ops.reshape(pred_bins, [-1, 1]) update_op0 = state_ops.scatter_nd_add( - neg_pos_var, - tf.concat([labels, pred_bins], axis=1), - array_ops.ones(tf.shape(labels)[0], dtype=tf.int64), + neg_pos_var, + tf.concat([labels, pred_bins], axis=1), + array_ops.ones(tf.shape(labels)[0], dtype=tf.int64), ) total_pos = math_ops.reduce_sum(labels) total_neg = array_ops.shape(labels)[0] - total_pos total_add = math_ops.cast(tf.stack([total_neg, total_pos]), dtype=tf.int64) update_op1 = state_ops.assign_add(total_var, total_add) - return tf.py_func(value_pyfunc, [neg_pos_var, total_var], tf.float32), tf.group([update_op0, update_op1]) + return tf.py_func(value_pyfunc, [neg_pos_var, total_var], + tf.float32), tf.group([update_op0, update_op1]) -def _distribute_separated_auc_impl(labels, predictions, keys, reduction='mean', metric_name='sepatated_auc'): +def _distribute_separated_auc_impl(labels, + predictions, + keys, + reduction='mean', + metric_name='sepatated_auc'): """Computes the AUC group by the key separately. Args: @@ -176,9 +183,9 @@ def _distribute_separated_auc_impl(labels, predictions, keys, reduction='mean', * "mean_by_positive_num": weighted mean with positive sample num of different keys """ assert reduction in [ - 'mean', - 'mean_by_sample_num', - 'mean_by_positive_num', + 'mean', + 'mean_by_sample_num', + 'mean_by_positive_num', ], 'reduction method must in mean | mean_by_sample_num | mean_by_positive_num' separated_label = defaultdict(list) separated_prediction = defaultdict(list) @@ -188,7 +195,8 @@ def _distribute_separated_auc_impl(labels, predictions, keys, reduction='mean', cur_task_index, task_num = get_task_index_and_num() cur_work_device = 'job_' + cur_job_name + '__' + 'task_' + str(cur_task_index) eval_tmp_results_dir = os.environ['eval_tmp_results_dir'] - assert tf.gfile.IsDirectory(eval_tmp_results_dir), 'eval_tmp_results_dir not exists' + assert tf.gfile.IsDirectory( + eval_tmp_results_dir), 'eval_tmp_results_dir not exists' def update_pyfunc(labels, predictions, keys): for label, prediction, key in zip(labels, predictions, keys): @@ -202,8 +210,8 @@ def update_pyfunc(labels, predictions, keys): elif reduction == 'mean_by_positive_num': separated_weights[key] += label.item() for name, data in zip( - ['separated_label', 'separated_prediction', 'separated_weights'], - [separated_label, separated_prediction, separated_weights], + ['separated_label', 'separated_prediction', 'separated_weights'], + [separated_label, separated_prediction, separated_weights], ): cur_json_name = metric_name + '__' + cur_work_device + '__' + name + '.json' cur_json_path = os.path.join(eval_tmp_results_dir, cur_json_name) @@ -213,37 +221,34 @@ def value_pyfunc(): for task_i in range(1, task_num): work_device_i = 'job_worker__task_' + str(task_i) for name in [ - 'separated_label', - 'separated_prediction', - 'separated_weights', + 'separated_label', + 'separated_prediction', + 'separated_weights', ]: json_name_i = metric_name + '__' + work_device_i + '__' + name + '.json' json_path_i = os.path.join(eval_tmp_results_dir, json_name_i) data_i = read_data_from_json_path(json_path_i) if name == 'separated_label': - separated_label.update( - { + separated_label.update({ key: separated_label.get(key, []) + data_i.get(key, []) - for key in set(list(separated_label.keys()) + list(data_i.keys())) - } - ) + for key in set( + list(separated_label.keys()) + list(data_i.keys())) + }) elif name == 'separated_prediction': - separated_prediction.update( - { + separated_prediction.update({ key: separated_prediction.get(key, []) + data_i.get(key, []) - for key in set(list(separated_prediction.keys()) + list(data_i.keys())) - } - ) + for key in set( + list(separated_prediction.keys()) + list(data_i.keys())) + }) elif name == 'separated_weights': if reduction == 'mean': separated_weights.update(data_i) else: - separated_weights.update( - { + separated_weights.update({ key: separated_weights.get(key, 0) + data_i.get(key, 0) - for key in set(list(separated_weights.keys()) + list(data_i.keys())) - } - ) + for key in set( + list(separated_weights.keys()) + list(data_i.keys())) + }) else: assert False, 'Not supported name {}'.format(name) metrics = [] @@ -281,7 +286,8 @@ def gauc(labels, predictions, uids, reduction='mean'): * "mean_by_positive_num": weighted mean with positive sample num of different users """ if os.environ.get('distribute_eval') == 'True': - return _distribute_separated_auc_impl(labels, predictions, uids, reduction, metric_name='gauc') + return _distribute_separated_auc_impl( + labels, predictions, uids, reduction, metric_name='gauc') return _separated_auc_impl(labels, predictions, uids, reduction) @@ -300,11 +306,16 @@ def session_auc(labels, predictions, session_ids, reduction='mean'): * "mean_by_positive_num": weighted mean with positive sample num of different sessions """ if os.environ.get('distribute_eval') == 'True': - return _distribute_separated_auc_impl(labels, predictions, session_ids, reduction, metric_name='session_auc') + return _distribute_separated_auc_impl( + labels, predictions, session_ids, reduction, metric_name='session_auc') return _separated_auc_impl(labels, predictions, session_ids, reduction) -def metric_learning_recall_at_k(k, embeddings, labels, session_ids=None, embed_normed=False): +def metric_learning_recall_at_k(k, + embeddings, + labels, + session_ids=None, + embed_normed=False): """Computes the recall_at_k metric for metric learning. Args: @@ -327,13 +338,17 @@ def metric_learning_recall_at_k(k, embeddings, labels, session_ids=None, embed_n # Uses broadcasting where the 1st argument has shape (1, batch_size) and the 2nd (batch_size, 1) labels_equal = tf.equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1)) if session_ids is not None and session_ids is not labels: - sessions_equal = tf.equal(tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1)) + sessions_equal = tf.equal( + tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1)) labels_equal = tf.logical_and(sessions_equal, labels_equal) mask = tf.logical_and(indices_not_equal, labels_equal) - mask_pos = tf.where(mask, sim_mat, -array_ops.ones_like(sim_mat)) # shape: (batch_size, batch_size) + mask_pos = tf.where( + mask, sim_mat, + -array_ops.ones_like(sim_mat)) # shape: (batch_size, batch_size) if isinstance(k, int): _, pos_top_k_idx = tf.nn.top_k(mask_pos, k) # shape: (batch_size, k) - return metrics_tf.recall_at_k(labels=tf.to_int64(pos_top_k_idx), predictions=sim_mat, k=k) + return metrics_tf.recall_at_k( + labels=tf.to_int64(pos_top_k_idx), predictions=sim_mat, k=k) if any((isinstance(k, list), isinstance(k, tuple), isinstance(k, set))): metrics = {} for kk in k: @@ -341,14 +356,17 @@ def metric_learning_recall_at_k(k, embeddings, labels, session_ids=None, embed_n continue _, pos_top_k_idx = tf.nn.top_k(mask_pos, kk) metrics['recall@' + str(kk)] = metrics_tf.recall_at_k( - labels=tf.to_int64(pos_top_k_idx), predictions=sim_mat, k=kk - ) + labels=tf.to_int64(pos_top_k_idx), predictions=sim_mat, k=kk) return metrics else: raise ValueError('k should be a `int` or a list/tuple/set of int.') -def metric_learning_average_precision_at_k(k, embeddings, labels, session_ids=None, embed_normed=False): +def metric_learning_average_precision_at_k(k, + embeddings, + labels, + session_ids=None, + embed_normed=False): from easy_rec.python.core.easyrec_metrics import metrics_tf # make sure embedding should be l2-normalized @@ -360,7 +378,8 @@ def metric_learning_average_precision_at_k(k, embeddings, labels, session_ids=No sim_mat = sim_mat - tf.eye(batch_size) * 2.0 mask = tf.equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1)) if session_ids is not None and session_ids is not labels: - sessions_equal = tf.equal(tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1)) + sessions_equal = tf.equal( + tf.expand_dims(session_ids, 0), tf.expand_dims(session_ids, 1)) mask = tf.logical_and(sessions_equal, mask) label_indices = _get_matrix_mask_indices(mask) if isinstance(k, int): @@ -370,7 +389,8 @@ def metric_learning_average_precision_at_k(k, embeddings, labels, session_ids=No for kk in k: if kk < 1: continue - metrics['MAP@' + str(kk)] = metrics_tf.average_precision_at_k(label_indices, sim_mat, kk) + metrics['MAP@' + str(kk)] = metrics_tf.average_precision_at_k( + label_indices, sim_mat, kk) return metrics else: raise ValueError('k should be a `int` or a list/tuple/set of int.') @@ -381,7 +401,8 @@ def _get_matrix_mask_indices(matrix, num_rows=None): num_rows = get_shape_list(matrix)[0] indices = tf.where(matrix) num_indices = tf.shape(indices)[0] - elem_per_row = tf.bincount(tf.cast(indices[:, 0], tf.int32), minlength=num_rows) + elem_per_row = tf.bincount( + tf.cast(indices[:, 0], tf.int32), minlength=num_rows) max_elem_per_row = tf.reduce_max(elem_per_row) row_start = tf.concat([[0], tf.cumsum(elem_per_row[:-1])], axis=0) r = tf.range(max_elem_per_row) @@ -389,7 +410,8 @@ def _get_matrix_mask_indices(matrix, num_rows=None): idx = tf.minimum(idx, num_indices - 1) result = tf.gather(indices[:, 1], idx) # replace invalid elements with -1 - result = tf.where(tf.expand_dims(elem_per_row, 1) > r, result, -array_ops.ones_like(result)) + result = tf.where( + tf.expand_dims(elem_per_row, 1) > r, result, -array_ops.ones_like(result)) max_index_per_row = tf.reduce_max(result, axis=1, keepdims=True) max_index_per_row = tf.tile(max_index_per_row, [1, max_elem_per_row]) result = tf.where(result >= 0, result, max_index_per_row) diff --git a/easy_rec/python/core/sampler.py b/easy_rec/python/core/sampler.py index 890857fbc..dcdcd44f0 100644 --- a/easy_rec/python/core/sampler.py +++ b/easy_rec/python/core/sampler.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import division, print_function +from __future__ import division +from __future__ import print_function import json import logging @@ -42,11 +43,10 @@ def string_attrs(self, string_attrs): # NOQA try: import graphlearn as gl from graphlearn.python.data.values import Values - Values.string_attrs = string_attrs except Exception: logging.info( - 'GraphLearn is not installed. You can install it by "pip install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-0.7-cp27-cp27mu-linux_x86_64.whl"' # noqa: E501 + 'GraphLearn is not installed. You can install it by "pip install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-0.7-cp27-cp27mu-linux_x86_64.whl"' # noqa: E501 ) if tf.__version__ >= '2.0': @@ -55,12 +55,12 @@ def string_attrs(self, string_attrs): # NOQA def _get_gl_type(field_type): type_map = { - DatasetConfig.INT32: 'int', - DatasetConfig.INT64: 'int', - DatasetConfig.STRING: 'string', - DatasetConfig.BOOL: 'int', - DatasetConfig.FLOAT: 'float', - DatasetConfig.DOUBLE: 'float', + DatasetConfig.INT32: 'int', + DatasetConfig.INT64: 'int', + DatasetConfig.STRING: 'string', + DatasetConfig.BOOL: 'int', + DatasetConfig.FLOAT: 'float', + DatasetConfig.DOUBLE: 'float' } assert field_type in type_map, 'invalid type: %s' % field_type return type_map[field_type] @@ -68,12 +68,12 @@ def _get_gl_type(field_type): def _get_np_type(field_type): type_map = { - DatasetConfig.INT32: np.int32, - DatasetConfig.INT64: np.int64, - DatasetConfig.STRING: str, - DatasetConfig.BOOL: bool, - DatasetConfig.FLOAT: np.float32, - DatasetConfig.DOUBLE: np.double, + DatasetConfig.INT32: np.int32, + DatasetConfig.INT64: np.int64, + DatasetConfig.STRING: str, + DatasetConfig.BOOL: bool, + DatasetConfig.FLOAT: np.float32, + DatasetConfig.DOUBLE: np.double } assert field_type in type_map, 'invalid type: %s' % field_type return type_map[field_type] @@ -92,7 +92,8 @@ def __init__(self, fields, num_sample, num_eval_sample=None): self._is_on_ds = ds_util.is_on_ds() def set_eval_num_sample(self): - print('set_eval_num_sample: %d %d' % (self._num_sample, self._num_eval_sample)) + print('set_eval_num_sample: %d %d' % + (self._num_sample, self._num_eval_sample)) self._num_sample = self._num_eval_sample def _init_graph(self): @@ -107,10 +108,13 @@ def _init_graph(self): task_count = 2 if self._is_on_ds: gl.set_tracker_mode(0) - server_hosts = [host.split(':')[0] + ':888' + str(i) for i, host in enumerate(tf_config['cluster']['ps'])] + server_hosts = [ + host.split(':')[0] + ':888' + str(i) + for i, host in enumerate(tf_config['cluster']['ps']) + ] cluster = { - 'server': ','.join(server_hosts), - 'client_count': task_count, + 'server': ','.join(server_hosts), + 'client_count': task_count } else: ps_count = len(tf_config['cluster']['ps']) @@ -119,25 +123,22 @@ def _init_graph(self): self._g.init(cluster=cluster, job_name='client', task_index=0) elif tf_config['task']['type'] == 'worker': self._g.init( - cluster=cluster, - job_name='client', - task_index=tf_config['task']['index'] + 2, - ) + cluster=cluster, + job_name='client', + task_index=tf_config['task']['index'] + 2) # TODO(hongsheng.jhs): check cluster has evaluator or not? elif tf_config['task']['type'] == 'evaluator': self._g.init( - cluster=cluster, - job_name='client', - task_index=tf_config['task']['index'] + 1, - ) + cluster=cluster, + job_name='client', + task_index=tf_config['task']['index'] + 1) if self._num_eval_sample is not None and self._num_eval_sample > 0: self._num_sample = self._num_eval_sample elif tf_config['task']['type'] == 'ps': self._g.init( - cluster=cluster, - job_name='server', - task_index=tf_config['task']['index'], - ) + cluster=cluster, + job_name='server', + task_index=tf_config['task']['index']) else: # worker mode task_count = len(tf_config['cluster']['worker']) + 1 @@ -146,31 +147,31 @@ def _init_graph(self): self._g.init(task_index=0, task_count=task_count) elif tf_config['task']['type'] == 'worker': self._g.init( - task_index=tf_config['task']['index'] + 1, - task_count=task_count, - ) + task_index=tf_config['task']['index'] + 1, + task_count=task_count) else: gl.set_tracker_mode(0) if tf_config['cluster'].get('chief', ''): - chief_host = tf_config['cluster']['chief'][0].split(':')[0] + ':8880' + chief_host = tf_config['cluster']['chief'][0].split( + ':')[0] + ':8880' else: - chief_host = tf_config['cluster']['master'][0].split(':')[0] + ':8880' + chief_host = tf_config['cluster']['master'][0].split( + ':')[0] + ':8880' worker_hosts = chief_host + [ - host.split(':')[0] + ':888' + str(i) for i, host in enumerate(tf_config['cluster']['worker']) + host.split(':')[0] + ':888' + str(i) + for i, host in enumerate(tf_config['cluster']['worker']) ] if tf_config['task']['type'] in ['chief', 'master']: self._g.init( - task_index=0, - task_count=task_count, - hosts=','.join(worker_hosts), - ) + task_index=0, + task_count=task_count, + hosts=','.join(worker_hosts)) elif tf_config['task']['type'] == 'worker': self._g.init( - task_index=tf_config['task']['index'] + 1, - task_count=task_count, - hosts=worker_hosts, - ) + task_index=tf_config['task']['index'] + 1, + task_count=task_count, + hosts=worker_hosts) # TODO(hongsheng.jhs): check cluster has evaluator or not? else: @@ -203,15 +204,15 @@ def __del__(self): def _parse_nodes(self, nodes): if self._log_first_n > 0: - logging.info( - 'num_example=%d num_eval_example=%d node_num=%d' % (self._num_sample, self._num_eval_sample, len(nodes.ids)) - ) + logging.info('num_example=%d num_eval_example=%d node_num=%d' % + (self._num_sample, self._num_eval_sample, len(nodes.ids))) self._log_first_n -= 1 features = [] int_idx = 0 float_idx = 0 string_idx = 0 - for attr_gl_type, attr_np_type in zip(self._attr_gl_types, self._attr_np_types): + for attr_gl_type, attr_np_type in zip(self._attr_gl_types, + self._attr_np_types): if attr_gl_type == 'int': feature = nodes.int_attrs[:, :, int_idx] int_idx += 1 @@ -225,7 +226,8 @@ def _parse_nodes(self, nodes): string_idx += 1 else: raise ValueError('Unknown attr type %s' % attr_gl_type) - feature = np.reshape(feature, [-1])[: self._num_sample].astype(attr_np_type) + feature = np.reshape(feature, + [-1])[:self._num_sample].astype(attr_np_type) if attr_gl_type == 'string': feature = feature.tolist() features.append(feature) @@ -236,7 +238,8 @@ def _parse_sparse_nodes(self, nodes): int_idx = 0 float_idx = 0 string_idx = 0 - for attr_gl_type, attr_np_type in zip(self._attr_gl_types, self._attr_np_types): + for attr_gl_type, attr_np_type in zip(self._attr_gl_types, + self._attr_np_types): if attr_gl_type == 'int': feature = nodes.int_attrs[:, int_idx] int_idx += 1 @@ -269,30 +272,27 @@ class NegativeSampler(BaseSampler): num_eval_sample: number of negative samples for evaluator. """ - def __init__( - self, - data_path, - fields, - num_sample, - batch_size, - attr_delimiter=':', - num_eval_sample=None, - ): + def __init__(self, + data_path, + fields, + num_sample, + batch_size, + attr_delimiter=':', + num_eval_sample=None): super(NegativeSampler, self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size self._g = gl.Graph().node( - tf.compat.as_str(data_path), - node_type='item', - decoder=gl.Decoder( - attr_types=self._attr_gl_types, - weighted=True, - attr_delimiter=attr_delimiter, - ), - ) + tf.compat.as_str(data_path), + node_type='item', + decoder=gl.Decoder( + attr_types=self._attr_gl_types, + weighted=True, + attr_delimiter=attr_delimiter)) self._init_graph() expand_factor = int(math.ceil(self._num_sample / batch_size)) - self._sampler = self._g.negative_sampler('item', expand_factor, strategy='node_weight') + self._sampler = self._g.negative_sampler( + 'item', expand_factor, strategy='node_weight') def _get_impl(self, ids): ids = np.array(ids, dtype=np.int64) @@ -332,22 +332,21 @@ class NegativeSamplerInMemory(BaseSampler): num_eval_sample: number of negative samples for evaluator. """ - def __init__( - self, - data_path, - fields, - num_sample, - batch_size, - attr_delimiter=':', - num_eval_sample=None, - ): - super(NegativeSamplerInMemory, self).__init__(fields, num_sample, num_eval_sample) + def __init__(self, + data_path, + fields, + num_sample, + batch_size, + attr_delimiter=':', + num_eval_sample=None): + super(NegativeSamplerInMemory, self).__init__(fields, num_sample, + num_eval_sample) self._batch_size = batch_size self._item_ids = [] self._cols = [[] for x in fields] - if six.PY2 and isinstance(attr_delimiter, type('')): + if six.PY2 and isinstance(attr_delimiter, type(u'')): attr_delimiter = attr_delimiter.encode('utf-8') if data_path.startswith('odps://'): self._load_table(data_path, attr_delimiter) @@ -361,11 +360,11 @@ def __init__( if np_type != str: self._cols[col_id] = np.array(self._cols[col_id], dtype=np_type) else: - self._cols[col_id] = np.asarray(self._cols[col_id], order='C', dtype=object) + self._cols[col_id] = np.asarray( + self._cols[col_id], order='C', dtype=object) def _load_table(self, data_path, attr_delimiter): import common_io - reader = common_io.table.TableReader(data_path) schema = reader.get_schema() item_id_col = 0 @@ -378,7 +377,8 @@ def _load_table(self, data_path, attr_delimiter): if schema[tid][0].startswith('id'): item_id_col = tid break - print('NegativeSamplerInMemory: feature_id_col = %d, item_id_col = %d' % (fea_id_col, item_id_col)) + print('NegativeSamplerInMemory: feature_id_col = %d, item_id_col = %d' % + (fea_id_col, item_id_col)) while True: try: row_arr = reader.read(num_records=1024, allow_smaller_final_batch=True) @@ -386,12 +386,9 @@ def _load_table(self, data_path, attr_delimiter): # item_id, weight, feature self._item_ids.append(int(row[item_id_col])) col_vals = row[fea_id_col].split(attr_delimiter) - assert len(col_vals) == len(self._cols), 'invalid row[%d %d]: %s %s' % ( - len(col_vals), - len(self._cols), - row[item_id_col], - row[fea_id_col], - ) + assert len(col_vals) == len( + self._cols), 'invalid row[%d %d]: %s %s' % (len( + col_vals), len(self._cols), row[item_id_col], row[fea_id_col]) for col_id in range(len(col_vals)): self._cols[col_id].append(col_vals[col_id]) except common_io.exception.OutOfRangeException: @@ -413,17 +410,15 @@ def _load_data(self, data_path, attr_delimiter): item_id_col = tid if schema[tid][0].startswith('feature'): fea_id_col = tid - print('feature_id_col = %d, item_id_col = %d' % (fea_id_col, item_id_col)) + print('feature_id_col = %d, item_id_col = %d' % + (fea_id_col, item_id_col)) else: self._item_ids.append(int(cols[item_id_col])) fea_vals = cols[fea_id_col].split(attr_delimiter) - assert len(fea_vals) == len(self._cols), 'invalid row[%d][%d %d]:%s %s' % ( - line_id, - len(fea_vals), - len(self._cols), - cols[item_id_col], - cols[fea_id_col], - ) + assert len(fea_vals) == len( + self._cols), 'invalid row[%d][%d %d]:%s %s' % ( + line_id, len(fea_vals), len( + self._cols), cols[item_id_col], cols[fea_id_col]) for col_id in range(len(fea_vals)): self._cols[col_id].append(fea_vals[col_id]) @@ -433,7 +428,10 @@ def _get_impl(self, ids): ids = [int(x) for x in ids] assert self._num_sample > 0, 'invalid num_sample: %d' % self._num_sample - indices = np.random.choice(len(self._item_ids), size=self._num_sample + self._batch_size, replace=False) + indices = np.random.choice( + len(self._item_ids), + size=self._num_sample + self._batch_size, + replace=False) sel_ids = [] for tid in indices: @@ -451,7 +449,8 @@ def _get_impl(self, ids): sel_feas = tmp_col[sel_ids] features.append(sel_feas) else: - features.append(np.asarray([tmp_col[x] for x in sel_ids], order='C', dtype=object)) + features.append( + np.asarray([tmp_col[x] for x in sel_ids], order='C', dtype=object)) return features def get(self, ids): @@ -489,45 +488,35 @@ class NegativeSamplerV2(BaseSampler): num_eval_sample: number of negative samples for evaluator. """ - def __init__( - self, - user_data_path, - item_data_path, - edge_data_path, - fields, - num_sample, - batch_size, - attr_delimiter=':', - num_eval_sample=None, - ): + def __init__(self, + user_data_path, + item_data_path, + edge_data_path, + fields, + num_sample, + batch_size, + attr_delimiter=':', + num_eval_sample=None): super(NegativeSamplerV2, self).__init__(fields, num_sample, num_eval_sample) self._batch_size = batch_size - self._g = ( - gl.Graph() - .node( - tf.compat.as_str(user_data_path), - node_type='user', - decoder=gl.Decoder(weighted=True), - ) - .node( - tf.compat.as_str(item_data_path), - node_type='item', - decoder=gl.Decoder( - attr_types=self._attr_gl_types, - weighted=True, - attr_delimiter=attr_delimiter, - ), - ) - .edge( - tf.compat.as_str(edge_data_path), - edge_type=('user', 'item', 'edge'), - decoder=gl.Decoder(weighted=True), - ) - ) + self._g = gl.Graph() \ + .node(tf.compat.as_str(user_data_path), + node_type='user', + decoder=gl.Decoder(weighted=True)) \ + .node(tf.compat.as_str(item_data_path), + node_type='item', + decoder=gl.Decoder( + attr_types=self._attr_gl_types, + weighted=True, + attr_delimiter=attr_delimiter)) \ + .edge(tf.compat.as_str(edge_data_path), + edge_type=('user', 'item', 'edge'), + decoder=gl.Decoder(weighted=True)) self._init_graph() expand_factor = int(math.ceil(self._num_sample / batch_size)) - self._sampler = self._g.negative_sampler('edge', expand_factor, strategy='random', conditional=True) + self._sampler = self._g.negative_sampler( + 'edge', expand_factor, strategy='random', conditional=True) def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) @@ -548,7 +537,8 @@ def get(self, src_ids, dst_ids): Returns: Negative sampled feature dict. """ - sampled_values = tf.py_func(self._get_impl, [src_ids, dst_ids], self._attr_tf_types) + sampled_values = tf.py_func(self._get_impl, [src_ids, dst_ids], + self._attr_tf_types) result_dict = {} for k, t, v in zip(self._attr_names, self._attr_tf_types, sampled_values): v.set_shape([self._num_sample]) @@ -574,47 +564,40 @@ class HardNegativeSampler(BaseSampler): num_eval_sample: number of negative samples for evaluator. """ - def __init__( - self, - user_data_path, - item_data_path, - hard_neg_edge_data_path, - fields, - num_sample, - num_hard_sample, - batch_size, - attr_delimiter=':', - num_eval_sample=None, - ): - super(HardNegativeSampler, self).__init__(fields, num_sample, num_eval_sample) + def __init__(self, + user_data_path, + item_data_path, + hard_neg_edge_data_path, + fields, + num_sample, + num_hard_sample, + batch_size, + attr_delimiter=':', + num_eval_sample=None): + super(HardNegativeSampler, self).__init__(fields, num_sample, + num_eval_sample) self._batch_size = batch_size - self._g = ( - gl.Graph() - .node( - tf.compat.as_str(user_data_path), - node_type='user', - decoder=gl.Decoder(weighted=True), - ) - .node( - tf.compat.as_str(item_data_path), - node_type='item', - decoder=gl.Decoder( - attr_types=self._attr_gl_types, - weighted=True, - attr_delimiter=attr_delimiter, - ), - ) - .edge( - tf.compat.as_str(hard_neg_edge_data_path), - edge_type=('user', 'item', 'hard_neg_edge'), - decoder=gl.Decoder(weighted=True), - ) - ) + self._g = gl.Graph() \ + .node(tf.compat.as_str(user_data_path), + node_type='user', + decoder=gl.Decoder(weighted=True)) \ + .node(tf.compat.as_str(item_data_path), + node_type='item', + decoder=gl.Decoder( + attr_types=self._attr_gl_types, + weighted=True, + attr_delimiter=attr_delimiter)) \ + .edge(tf.compat.as_str(hard_neg_edge_data_path), + edge_type=('user', 'item', 'hard_neg_edge'), + decoder=gl.Decoder(weighted=True)) self._init_graph() expand_factor = int(math.ceil(self._num_sample / batch_size)) - self._neg_sampler = self._g.negative_sampler('item', expand_factor, strategy='node_weight') - self._hard_neg_sampler = self._g.neighbor_sampler(['hard_neg_edge'], num_hard_sample, strategy='full') + self._neg_sampler = self._g.negative_sampler( + 'item', expand_factor, strategy='node_weight') + self._hard_neg_sampler = self._g.neighbor_sampler(['hard_neg_edge'], + num_hard_sample, + strategy='full') def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) @@ -647,7 +630,8 @@ def get(self, src_ids, dst_ids): output_types = self._attr_tf_types + [tf.int64] output_values = tf.py_func(self._get_impl, [src_ids, dst_ids], output_types) result_dict = {} - for k, t, v in zip(self._attr_names, self._attr_tf_types, output_values[:-1]): + for k, t, v in zip(self._attr_names, self._attr_tf_types, + output_values[:-1]): v.set_shape([None]) result_dict[k] = v @@ -676,57 +660,49 @@ class HardNegativeSamplerV2(BaseSampler): num_eval_sample: number of negative samples for evaluator. """ - def __init__( - self, - user_data_path, - item_data_path, - edge_data_path, - hard_neg_edge_data_path, - fields, - num_sample, - num_hard_sample, - batch_size, - attr_delimiter=':', - num_eval_sample=None, - ): - super(HardNegativeSamplerV2, self).__init__(fields, num_sample, num_eval_sample) + def __init__(self, + user_data_path, + item_data_path, + edge_data_path, + hard_neg_edge_data_path, + fields, + num_sample, + num_hard_sample, + batch_size, + attr_delimiter=':', + num_eval_sample=None): + super(HardNegativeSamplerV2, self).__init__(fields, num_sample, + num_eval_sample) self._batch_size = batch_size - self._g = ( - gl.Graph() - .node( - tf.compat.as_str(user_data_path), - node_type='user', - decoder=gl.Decoder(weighted=True), - ) - .node( - tf.compat.as_str(item_data_path), - node_type='item', - decoder=gl.Decoder( - attr_types=self._attr_gl_types, - weighted=True, - attr_delimiter=attr_delimiter, - ), - ) - .edge( - tf.compat.as_str(edge_data_path), - edge_type=('user', 'item', 'edge'), - decoder=gl.Decoder(weighted=True), - ) - .edge( - tf.compat.as_str(hard_neg_edge_data_path), - edge_type=('user', 'item', 'hard_neg_edge'), - decoder=gl.Decoder(weighted=True), - ) - ) + self._g = gl.Graph() \ + .node(tf.compat.as_str(user_data_path), + node_type='user', + decoder=gl.Decoder(weighted=True)) \ + .node(tf.compat.as_str(item_data_path), + node_type='item', + decoder=gl.Decoder( + attr_types=self._attr_gl_types, + weighted=True, + attr_delimiter=attr_delimiter)) \ + .edge(tf.compat.as_str(edge_data_path), + edge_type=('user', 'item', 'edge'), + decoder=gl.Decoder(weighted=True)) \ + .edge(tf.compat.as_str(hard_neg_edge_data_path), + edge_type=('user', 'item', 'hard_neg_edge'), + decoder=gl.Decoder(weighted=True)) self._init_graph() expand_factor = int(math.ceil(self._num_sample / batch_size)) - self._neg_sampler = self._g.negative_sampler('edge', expand_factor, strategy='random', conditional=True) - self._hard_neg_sampler = self._g.neighbor_sampler(['hard_neg_edge'], num_hard_sample, strategy='full') + self._neg_sampler = self._g.negative_sampler( + 'edge', expand_factor, strategy='random', conditional=True) + self._hard_neg_sampler = self._g.neighbor_sampler(['hard_neg_edge'], + num_hard_sample, + strategy='full') def _get_impl(self, src_ids, dst_ids): src_ids = np.array(src_ids, dtype=np.int64) - src_ids_padded = np.pad(src_ids, (0, self._batch_size - len(src_ids)), 'edge') + src_ids_padded = np.pad(src_ids, (0, self._batch_size - len(src_ids)), + 'edge') dst_ids = np.array(dst_ids, dtype=np.int64) dst_ids = np.pad(dst_ids, (0, self._batch_size - len(dst_ids)), 'edge') nodes = self._neg_sampler.get(src_ids_padded, dst_ids) @@ -756,7 +732,8 @@ def get(self, src_ids, dst_ids): output_types = self._attr_tf_types + [tf.int64] output_values = tf.py_func(self._get_impl, [src_ids, dst_ids], output_types) result_dict = {} - for k, t, v in zip(self._attr_names, self._attr_tf_types, output_values[:-1]): + for k, t, v in zip(self._attr_names, self._attr_tf_types, + output_values[:-1]): v.set_shape([None]) result_dict[k] = v @@ -767,6 +744,7 @@ def get(self, src_ids, dst_ids): def build(data_config): + if not data_config.HasField('sampler'): return None sampler_type = data_config.WhichOneof('sampler') @@ -782,80 +760,85 @@ def build(data_config): input_path = process_multi_file_input_path(sampler_config.input_path) return NegativeSampler.instance( - data_path=input_path, - fields=attr_fields, - num_sample=sampler_config.num_sample, - batch_size=data_config.batch_size, - attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample, - ) + data_path=input_path, + fields=attr_fields, + num_sample=sampler_config.num_sample, + batch_size=data_config.batch_size, + attr_delimiter=sampler_config.attr_delimiter, + num_eval_sample=sampler_config.num_eval_sample) elif sampler_type == 'negative_sampler_in_memory': input_fields = {f.input_name: f for f in data_config.input_fields} attr_fields = [input_fields[name] for name in sampler_config.attr_fields] input_path = process_multi_file_input_path(sampler_config.input_path) return NegativeSamplerInMemory.instance( - data_path=input_path, - fields=attr_fields, - num_sample=sampler_config.num_sample, - batch_size=data_config.batch_size, - attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample, - ) + data_path=input_path, + fields=attr_fields, + num_sample=sampler_config.num_sample, + batch_size=data_config.batch_size, + attr_delimiter=sampler_config.attr_delimiter, + num_eval_sample=sampler_config.num_eval_sample) elif sampler_type == 'negative_sampler_v2': input_fields = {f.input_name: f for f in data_config.input_fields} attr_fields = [input_fields[name] for name in sampler_config.attr_fields] - user_input_path = process_multi_file_input_path(sampler_config.user_input_path) - item_input_path = process_multi_file_input_path(sampler_config.item_input_path) - pos_edge_input_path = process_multi_file_input_path(sampler_config.pos_edge_input_path) + user_input_path = process_multi_file_input_path( + sampler_config.user_input_path) + item_input_path = process_multi_file_input_path( + sampler_config.item_input_path) + pos_edge_input_path = process_multi_file_input_path( + sampler_config.pos_edge_input_path) return NegativeSamplerV2.instance( - user_data_path=user_input_path, - item_data_path=item_input_path, - edge_data_path=pos_edge_input_path, - fields=attr_fields, - num_sample=sampler_config.num_sample, - batch_size=data_config.batch_size, - attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample, - ) + user_data_path=user_input_path, + item_data_path=item_input_path, + edge_data_path=pos_edge_input_path, + fields=attr_fields, + num_sample=sampler_config.num_sample, + batch_size=data_config.batch_size, + attr_delimiter=sampler_config.attr_delimiter, + num_eval_sample=sampler_config.num_eval_sample) elif sampler_type == 'hard_negative_sampler': input_fields = {f.input_name: f for f in data_config.input_fields} attr_fields = [input_fields[name] for name in sampler_config.attr_fields] - user_input_path = process_multi_file_input_path(sampler_config.user_input_path) - item_input_path = process_multi_file_input_path(sampler_config.item_input_path) - hard_neg_edge_input_path = process_multi_file_input_path(sampler_config.hard_neg_edge_input_path) + user_input_path = process_multi_file_input_path( + sampler_config.user_input_path) + item_input_path = process_multi_file_input_path( + sampler_config.item_input_path) + hard_neg_edge_input_path = process_multi_file_input_path( + sampler_config.hard_neg_edge_input_path) return HardNegativeSampler.instance( - user_data_path=user_input_path, - item_data_path=item_input_path, - hard_neg_edge_data_path=hard_neg_edge_input_path, - fields=attr_fields, - num_sample=sampler_config.num_sample, - num_hard_sample=sampler_config.num_hard_sample, - batch_size=data_config.batch_size, - attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample, - ) + user_data_path=user_input_path, + item_data_path=item_input_path, + hard_neg_edge_data_path=hard_neg_edge_input_path, + fields=attr_fields, + num_sample=sampler_config.num_sample, + num_hard_sample=sampler_config.num_hard_sample, + batch_size=data_config.batch_size, + attr_delimiter=sampler_config.attr_delimiter, + num_eval_sample=sampler_config.num_eval_sample) elif sampler_type == 'hard_negative_sampler_v2': input_fields = {f.input_name: f for f in data_config.input_fields} attr_fields = [input_fields[name] for name in sampler_config.attr_fields] - user_input_path = process_multi_file_input_path(sampler_config.user_input_path) - item_input_path = process_multi_file_input_path(sampler_config.item_input_path) - pos_edge_input_path = process_multi_file_input_path(sampler_config.pos_edge_input_path) - hard_neg_edge_input_path = process_multi_file_input_path(sampler_config.hard_neg_edge_input_path) + user_input_path = process_multi_file_input_path( + sampler_config.user_input_path) + item_input_path = process_multi_file_input_path( + sampler_config.item_input_path) + pos_edge_input_path = process_multi_file_input_path( + sampler_config.pos_edge_input_path) + hard_neg_edge_input_path = process_multi_file_input_path( + sampler_config.hard_neg_edge_input_path) return HardNegativeSamplerV2.instance( - user_data_path=user_input_path, - item_data_path=item_input_path, - edge_data_path=pos_edge_input_path, - hard_neg_edge_data_path=hard_neg_edge_input_path, - fields=attr_fields, - num_sample=sampler_config.num_sample, - num_hard_sample=sampler_config.num_hard_sample, - batch_size=data_config.batch_size, - attr_delimiter=sampler_config.attr_delimiter, - num_eval_sample=sampler_config.num_eval_sample, - ) + user_data_path=user_input_path, + item_data_path=item_input_path, + edge_data_path=pos_edge_input_path, + hard_neg_edge_data_path=hard_neg_edge_input_path, + fields=attr_fields, + num_sample=sampler_config.num_sample, + num_hard_sample=sampler_config.num_hard_sample, + batch_size=data_config.batch_size, + attr_delimiter=sampler_config.attr_delimiter, + num_eval_sample=sampler_config.num_eval_sample) else: raise ValueError('Unknown sampler %s' % sampler_type) diff --git a/easy_rec/python/eval.py b/easy_rec/python/eval.py index c7c1124dd..bb4205764 100644 --- a/easy_rec/python/eval.py +++ b/easy_rec/python/eval.py @@ -8,36 +8,46 @@ import tensorflow as tf from tensorflow.python.lib.io import file_io -from easy_rec.python.main import distribute_evaluate, evaluate +from easy_rec.python.main import distribute_evaluate +from easy_rec.python.main import evaluate from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.utils import config_util, ds_util, estimator_utils +from easy_rec.python.utils import config_util +from easy_rec.python.utils import ds_util +from easy_rec.python.utils import estimator_utils + from easy_rec.python.utils.distribution_utils import ( # NOQA - set_tf_config_and_get_distribute_eval_worker_num_on_ds, -) + set_tf_config_and_get_distribute_eval_worker_num_on_ds,) if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) -tf.app.flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') +tf.app.flags.DEFINE_string('pipeline_config_path', None, + 'Path to pipeline config ' + 'file.') tf.app.flags.DEFINE_string( - 'checkpoint_path', - None, - 'checkpoint to be evaled ' ' if not specified, use the latest checkpoint in ' 'train_config.model_dir', + 'checkpoint_path', + None, + 'checkpoint to be evaled ' + ' if not specified, use the latest checkpoint in ' + 'train_config.model_dir', ) tf.app.flags.DEFINE_multi_string( - 'eval_input_path', - None, - 'eval data path, if specified will ' 'override pipeline_config.eval_input_path', + 'eval_input_path', + None, + 'eval data path, if specified will ' + 'override pipeline_config.eval_input_path', ) tf.app.flags.DEFINE_string('model_dir', None, help='will update the model_dir') tf.app.flags.DEFINE_string('odps_config', None, help='odps config path') -tf.app.flags.DEFINE_string('eval_result_path', 'eval_result.txt', 'eval result metric file') -tf.app.flags.DEFINE_bool('distribute_eval', False, 'use distribute parameter server for train and eval.') +tf.app.flags.DEFINE_string('eval_result_path', 'eval_result.txt', + 'eval result metric file') +tf.app.flags.DEFINE_bool('distribute_eval', False, + 'use distribute parameter server for train and eval.') tf.app.flags.DEFINE_bool('is_on_ds', False, help='is on ds') FLAGS = tf.app.flags.FLAGS @@ -61,17 +71,18 @@ def main(argv): else: pipeline_config_path = FLAGS.pipeline_config_path - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path) if FLAGS.model_dir: pipeline_config.model_dir = FLAGS.model_dir if pipeline_config.train_config.train_distribute in [ - DistributionStrategy.HorovodStrategy, + DistributionStrategy.HorovodStrategy, ]: estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ - DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy, + DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy, ]: estimator_utils.init_hvd() estimator_utils.init_sok() @@ -79,18 +90,18 @@ def main(argv): if FLAGS.distribute_eval: os.environ['distribute_eval'] = 'True' eval_result = distribute_evaluate( - pipeline_config, - FLAGS.checkpoint_path, - FLAGS.eval_input_path, - FLAGS.eval_result_path, + pipeline_config, + FLAGS.checkpoint_path, + FLAGS.eval_input_path, + FLAGS.eval_result_path, ) else: os.environ['distribute_eval'] = 'False' eval_result = evaluate( - pipeline_config, - FLAGS.checkpoint_path, - FLAGS.eval_input_path, - FLAGS.eval_result_path, + pipeline_config, + FLAGS.checkpoint_path, + FLAGS.eval_input_path, + FLAGS.eval_result_path, ) if eval_result is not None: # when distribute evaluate, only master has eval_result. diff --git a/easy_rec/python/export.py b/easy_rec/python/export.py index 5df4a6f19..ac59d50dc 100644 --- a/easy_rec/python/export.py +++ b/easy_rec/python/export.py @@ -8,7 +8,8 @@ from easy_rec.python.main import export from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.utils import config_util, estimator_utils +from easy_rec.python.utils import config_util +from easy_rec.python.utils import estimator_utils if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile @@ -19,30 +20,42 @@ tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) -tf.app.flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') +tf.app.flags.DEFINE_string('pipeline_config_path', None, + 'Path to pipeline config ' + 'file.') tf.app.flags.DEFINE_string('checkpoint_path', '', 'checkpoint to be exported') -tf.app.flags.DEFINE_string('export_dir', None, 'directory where model should be exported to') +tf.app.flags.DEFINE_string('export_dir', None, + 'directory where model should be exported to') tf.app.flags.DEFINE_string('redis_url', None, 'export to redis url, host:port') tf.app.flags.DEFINE_string('redis_passwd', None, 'export to redis passwd') tf.app.flags.DEFINE_integer('redis_threads', 0, 'export to redis threads') -tf.app.flags.DEFINE_integer('redis_batch_size', 256, 'export to redis batch_size') -tf.app.flags.DEFINE_integer('redis_timeout', 600, 'export to redis time_out in seconds') -tf.app.flags.DEFINE_integer('redis_expire', 24, 'export to redis expire time in hour') -tf.app.flags.DEFINE_string('redis_embedding_version', '', 'redis embedding version') -tf.app.flags.DEFINE_integer('redis_write_kv', 1, 'whether to write embedding to redis') - -tf.app.flags.DEFINE_string('oss_path', None, 'write embed objects to oss folder, oss://bucket/folder') +tf.app.flags.DEFINE_integer('redis_batch_size', 256, + 'export to redis batch_size') +tf.app.flags.DEFINE_integer('redis_timeout', 600, + 'export to redis time_out in seconds') +tf.app.flags.DEFINE_integer('redis_expire', 24, + 'export to redis expire time in hour') +tf.app.flags.DEFINE_string('redis_embedding_version', '', + 'redis embedding version') +tf.app.flags.DEFINE_integer('redis_write_kv', 1, + 'whether to write embedding to redis') + +tf.app.flags.DEFINE_string( + 'oss_path', None, 'write embed objects to oss folder, oss://bucket/folder') tf.app.flags.DEFINE_string('oss_endpoint', None, 'oss endpoint') tf.app.flags.DEFINE_string('oss_ak', None, 'oss ak') tf.app.flags.DEFINE_string('oss_sk', None, 'oss sk') -tf.app.flags.DEFINE_integer('oss_threads', 10, '# threads access oss at the same time') -tf.app.flags.DEFINE_integer('oss_timeout', 10, 'connect to oss, time_out in seconds') +tf.app.flags.DEFINE_integer('oss_threads', 10, + '# threads access oss at the same time') +tf.app.flags.DEFINE_integer('oss_timeout', 10, + 'connect to oss, time_out in seconds') tf.app.flags.DEFINE_integer('oss_expire', 24, 'oss expire time in hours') -tf.app.flags.DEFINE_integer('oss_write_kv', 1, 'whether to write embedding to oss') +tf.app.flags.DEFINE_integer('oss_write_kv', 1, + 'whether to write embedding to oss') tf.app.flags.DEFINE_string('oss_embedding_version', '', 'oss embedding version') tf.app.flags.DEFINE_string('asset_files', '', 'more files to add to asset') @@ -52,7 +65,8 @@ tf.app.flags.mark_flag_as_required('export_dir') tf.app.flags.DEFINE_bool('clear_export', False, 'remove export_dir if exists') -tf.app.flags.DEFINE_string('export_done_file', '', 'a flag file to signal that export model is done') +tf.app.flags.DEFINE_string('export_done_file', '', + 'a flag file to signal that export model is done') FLAGS = tf.app.flags.FLAGS @@ -104,14 +118,15 @@ def main(argv): if FLAGS.oss_embedding_version: extra_params['oss_embedding_version'] = FLAGS.oss_embedding_version - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path) if pipeline_config.train_config.train_distribute in [ - DistributionStrategy.HorovodStrategy, + DistributionStrategy.HorovodStrategy, ]: estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ - DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy, + DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy, ]: estimator_utils.init_hvd() estimator_utils.init_sok() @@ -122,12 +137,12 @@ def main(argv): gfile.DeleteRecursively(FLAGS.export_dir) export_out_dir = export( - FLAGS.export_dir, - pipeline_config_path, - FLAGS.checkpoint_path, - FLAGS.asset_files, - FLAGS.verbose, - **extra_params, + FLAGS.export_dir, + pipeline_config_path, + FLAGS.checkpoint_path, + FLAGS.asset_files, + FLAGS.verbose, + **extra_params, ) if FLAGS.export_done_file: diff --git a/easy_rec/python/feature_column/feature_column.py b/easy_rec/python/feature_column/feature_column.py index 2ca60da1d..53791c727 100644 --- a/easy_rec/python/feature_column/feature_column.py +++ b/easy_rec/python/feature_column/feature_column.py @@ -9,22 +9,24 @@ from tensorflow.python.platform import gfile from easy_rec.python.builders import hyperparams_builder -from easy_rec.python.compat.feature_column import feature_column_v2 as feature_column # NOQA -from easy_rec.python.compat.feature_column import ( - sequence_feature_column, -) -from easy_rec.python.protos.feature_config_pb2 import FeatureConfig, WideOrDeep +from easy_rec.python.compat.feature_column import sequence_feature_column +from easy_rec.python.protos.feature_config_pb2 import FeatureConfig +from easy_rec.python.protos.feature_config_pb2 import WideOrDeep from easy_rec.python.utils.proto_util import copy_obj +from easy_rec.python.compat.feature_column import feature_column_v2 as feature_column # NOQA + MAX_HASH_BUCKET_SIZE = 9223372036854775807 class FeatureKeyError(KeyError): + def __init__(self, feature_name): super(FeatureKeyError, self).__init__(feature_name) class SharedEmbedding(object): + def __init__(self, embedding_name, index, sequence_combiner=None): self.embedding_name = embedding_name self.index = index @@ -32,15 +34,22 @@ def __init__(self, embedding_name, index, sequence_combiner=None): EVParams = collections.namedtuple( - 'EVParams', - ['filter_freq', 'steps_to_live', 'use_cache', 'init_capacity', 'max_capacity'], + 'EVParams', + [ + 'filter_freq', 'steps_to_live', 'use_cache', 'init_capacity', + 'max_capacity' + ], ) class FeatureColumnParser(object): """Parse and generate feature columns.""" - def __init__(self, feature_configs, wide_deep_dict={}, wide_output_dim=-1, ev_params=None): + def __init__(self, + feature_configs, + wide_deep_dict={}, + wide_output_dim=-1, + ev_params=None): """Initializes a `FeatureColumnParser`. Args: @@ -70,13 +79,10 @@ def __init__(self, feature_configs, wide_deep_dict={}, wide_output_dim=-1, ev_pa self._global_ev_params = self._build_ev_params(ev_params) def _cmp_embed_config(a, b): - return ( - a.embedding_dim == b.embedding_dim - and a.combiner == b.combiner - and a.initializer == b.initializer - and a.max_partitions == b.max_partitions - and a.embedding_name == b.embedding_name - ) + return (a.embedding_dim == b.embedding_dim and + a.combiner == b.combiner and a.initializer == b.initializer and + a.max_partitions == b.max_partitions and + a.embedding_name == b.embedding_name) for config in self._feature_configs: if not config.HasField('embedding_name'): @@ -85,13 +91,11 @@ def _cmp_embed_config(a, b): if embed_name in self._share_embed_names: assert _cmp_embed_config(config, self._share_embed_infos[embed_name]), ( - 'shared embed info of [%s] is not matched [%s] vs [%s]' - % ( - embed_name, - config, - self._share_embed_infos[embed_name], - ) - ) + 'shared embed info of [%s] is not matched [%s] vs [%s]' % ( + embed_name, + config, + self._share_embed_infos[embed_name], + )) self._share_embed_names[embed_name] += 1 if config.feature_type == FeatureConfig.FeatureType.SequenceFeature: self._share_embed_infos[embed_name] = copy_obj(config) @@ -100,23 +104,26 @@ def _cmp_embed_config(a, b): self._share_embed_infos[embed_name] = copy_obj(config) # remove not shared embedding names - not_shared = [x for x in self._share_embed_names if self._share_embed_names[x] == 1] + not_shared = [ + x for x in self._share_embed_names if self._share_embed_names[x] == 1 + ] for embed_name in not_shared: del self._share_embed_names[embed_name] del self._share_embed_infos[embed_name] logging.info('shared embeddings[num=%d]' % len(self._share_embed_names)) for embed_name in self._share_embed_names: - logging.info( - '\t%s: share_num[%d], share_info[%s]' - % ( + logging.info('\t%s: share_num[%d], share_info[%s]' % ( embed_name, self._share_embed_names[embed_name], self._share_embed_infos[embed_name], - ) - ) - self._deep_share_embed_columns = {embed_name: [] for embed_name in self._share_embed_names} - self._wide_share_embed_columns = {embed_name: [] for embed_name in self._share_embed_names} + )) + self._deep_share_embed_columns = { + embed_name: [] for embed_name in self._share_embed_names + } + self._wide_share_embed_columns = { + embed_name: [] for embed_name in self._share_embed_names + } self._feature_vocab_size = {} for config in self._feature_configs: @@ -144,28 +151,31 @@ def _cmp_embed_config(a, b): for embed_name in self._share_embed_names: initializer = None if self._share_embed_infos[embed_name].HasField('initializer'): - initializer = hyperparams_builder.build_initializer(self._share_embed_infos[embed_name].initializer) + initializer = hyperparams_builder.build_initializer( + self._share_embed_infos[embed_name].initializer) partitioner = self._build_partitioner(self._share_embed_infos[embed_name]) if self._share_embed_infos[embed_name].HasField('ev_params'): - ev_params = self._build_ev_params(self._share_embed_infos[embed_name].ev_params) + ev_params = self._build_ev_params( + self._share_embed_infos[embed_name].ev_params) else: ev_params = self._global_ev_params # for handling share embedding columns if len(self._deep_share_embed_columns[embed_name]) > 0: share_embed_fcs = feature_column.shared_embedding_columns( - self._deep_share_embed_columns[embed_name], - self._share_embed_infos[embed_name].embedding_dim, - initializer=initializer, - shared_embedding_collection_name=embed_name, - combiner=self._share_embed_infos[embed_name].combiner, - partitioner=partitioner, - ev_params=ev_params, + self._deep_share_embed_columns[embed_name], + self._share_embed_infos[embed_name].embedding_dim, + initializer=initializer, + shared_embedding_collection_name=embed_name, + combiner=self._share_embed_infos[embed_name].combiner, + partitioner=partitioner, + ev_params=ev_params, ) config = self._share_embed_infos[embed_name] - max_seq_len = config.max_seq_len if config.HasField('max_seq_len') else -1 + max_seq_len = config.max_seq_len if config.HasField( + 'max_seq_len') else -1 for fc in share_embed_fcs: fc.max_seq_length = max_seq_len self._deep_share_embed_columns[embed_name] = share_embed_fcs @@ -173,16 +183,17 @@ def _cmp_embed_config(a, b): # for handling wide share embedding columns if len(self._wide_share_embed_columns[embed_name]) > 0: share_embed_fcs = feature_column.shared_embedding_columns( - self._wide_share_embed_columns[embed_name], - self._wide_output_dim, - initializer=initializer, - shared_embedding_collection_name=embed_name + '_wide', - combiner='sum', - partitioner=partitioner, - ev_params=ev_params, + self._wide_share_embed_columns[embed_name], + self._wide_output_dim, + initializer=initializer, + shared_embedding_collection_name=embed_name + '_wide', + combiner='sum', + partitioner=partitioner, + ev_params=ev_params, ) config = self._share_embed_infos[embed_name] - max_seq_len = config.max_seq_len if config.HasField('max_seq_len') else -1 + max_seq_len = config.max_seq_len if config.HasField( + 'max_seq_len') else -1 for fc in share_embed_fcs: fc.max_seq_length = max_seq_len self._wide_share_embed_columns[embed_name] = share_embed_fcs @@ -195,7 +206,8 @@ def _cmp_embed_config(a, b): for fc_name in self._wide_columns: fc = self._wide_columns[fc_name] if isinstance(fc, SharedEmbedding): - self._wide_columns[fc_name] = self._get_shared_embedding_column(fc, deep=False) + self._wide_columns[fc_name] = self._get_shared_embedding_column( + fc, deep=False) for fc_name in self._sequence_columns: fc = self._sequence_columns[fc_name] @@ -222,8 +234,8 @@ def is_wide(self, config): if feature_name not in self._wide_deep_dict: raise FeatureKeyError(feature_name) return self._wide_deep_dict[feature_name] in [ - WideOrDeep.WIDE, - WideOrDeep.WIDE_AND_DEEP, + WideOrDeep.WIDE, + WideOrDeep.WIDE_AND_DEEP, ] def is_deep(self, config): @@ -235,8 +247,8 @@ def is_deep(self, config): if feature_name not in self._wide_deep_dict: raise FeatureKeyError(feature_name) return self._wide_deep_dict[feature_name] in [ - WideOrDeep.DEEP, - WideOrDeep.WIDE_AND_DEEP, + WideOrDeep.DEEP, + WideOrDeep.WIDE_AND_DEEP, ] def get_feature_vocab_size(self, feature): @@ -268,35 +280,35 @@ def parse_id_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] hash_bucket_size = self._get_hash_bucket_size(config) if hash_bucket_size > 0: fc = feature_column.categorical_column_with_hash_bucket( - feature_name, - hash_bucket_size=hash_bucket_size, - feature_name=feature_name, + feature_name, + hash_bucket_size=hash_bucket_size, + feature_name=feature_name, ) elif config.vocab_list: fc = feature_column.categorical_column_with_vocabulary_list( - feature_name, - default_value=0, - vocabulary_list=config.vocab_list, - feature_name=feature_name, + feature_name, + default_value=0, + vocabulary_list=config.vocab_list, + feature_name=feature_name, ) elif config.vocab_file: fc = feature_column.categorical_column_with_vocabulary_file( - feature_name, - default_value=0, - vocabulary_file=config.vocab_file, - vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name, + feature_name, + default_value=0, + vocabulary_file=config.vocab_file, + vocabulary_size=self._get_vocab_size(config.vocab_file), + feature_name=feature_name, ) else: use_ev = self._global_ev_params or config.HasField('ev_params') num_buckets = sys.maxsize if use_ev else config.num_buckets fc = feature_column.categorical_column_with_identity( - feature_name, num_buckets, default_value=0, feature_name=feature_name - ) + feature_name, num_buckets, default_value=0, feature_name=feature_name) if self.is_wide(config): self._add_wide_embedding_column(fc, config) @@ -313,45 +325,43 @@ def parse_tag_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] hash_bucket_size = self._get_hash_bucket_size(config) if hash_bucket_size > 0: tag_fc = feature_column.categorical_column_with_hash_bucket( - feature_name, - hash_bucket_size, - dtype=tf.string, - feature_name=feature_name, + feature_name, + hash_bucket_size, + dtype=tf.string, + feature_name=feature_name, ) elif config.vocab_list: tag_fc = feature_column.categorical_column_with_vocabulary_list( - feature_name, - default_value=0, - vocabulary_list=config.vocab_list, - feature_name=feature_name, + feature_name, + default_value=0, + vocabulary_list=config.vocab_list, + feature_name=feature_name, ) elif config.vocab_file: tag_fc = feature_column.categorical_column_with_vocabulary_file( - feature_name, - default_value=0, - vocabulary_file=config.vocab_file, - vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name, + feature_name, + default_value=0, + vocabulary_file=config.vocab_file, + vocabulary_size=self._get_vocab_size(config.vocab_file), + feature_name=feature_name, ) else: use_ev = self._global_ev_params or config.HasField('ev_params') num_buckets = sys.maxsize if use_ev else config.num_buckets tag_fc = feature_column.categorical_column_with_identity( - feature_name, num_buckets, default_value=0, feature_name=feature_name - ) + feature_name, num_buckets, default_value=0, feature_name=feature_name) if len(config.input_names) > 1: tag_fc = feature_column.weighted_categorical_column( - tag_fc, weight_feature_key=feature_name + '_w', dtype=tf.float32 - ) + tag_fc, weight_feature_key=feature_name + '_w', dtype=tf.float32) elif config.HasField('kv_separator'): tag_fc = feature_column.weighted_categorical_column( - tag_fc, weight_feature_key=feature_name + '_w', dtype=tf.float32 - ) + tag_fc, weight_feature_key=feature_name + '_w', dtype=tf.float32) if self.is_wide(config): self._add_wide_embedding_column(tag_fc, config) @@ -366,8 +376,12 @@ def parse_raw_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] - fc = feature_column.numeric_column(key=feature_name, shape=(config.raw_input_dim,), feature_name=feature_name) + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] + fc = feature_column.numeric_column( + key=feature_name, + shape=(config.raw_input_dim,), + feature_name=feature_name) bounds = None if config.boundaries: @@ -375,14 +389,18 @@ def parse_raw_feature(self, config): bounds.sort() elif config.num_buckets > 1 and config.max_val > config.min_val: # the feature values are already normalized into [0, 1] - bounds = [x / float(config.num_buckets) for x in range(0, config.num_buckets)] - logging.info('discrete %s into %d buckets' % (feature_name, config.num_buckets)) + bounds = [ + x / float(config.num_buckets) for x in range(0, config.num_buckets) + ] + logging.info('discrete %s into %d buckets' % + (feature_name, config.num_buckets)) if bounds: try: fc = feature_column.bucketized_column(fc, bounds) except Exception as e: - logging.error('bucketized_column [%s] with bounds %s error' % (fc.name, str(bounds))) + logging.error('bucketized_column [%s] with bounds %s error' % + (fc.name, str(bounds))) raise e if self.is_wide(config): self._add_wide_embedding_column(fc, config) @@ -390,15 +408,15 @@ def parse_raw_feature(self, config): self._add_deep_embedding_column(fc, config) else: tmp_id_col = feature_column.categorical_column_with_identity( - feature_name + '_raw_proj_id', - config.raw_input_dim, - default_value=0, - feature_name=feature_name, + feature_name + '_raw_proj_id', + config.raw_input_dim, + default_value=0, + feature_name=feature_name, ) wgt_fc = feature_column.weighted_categorical_column( - tmp_id_col, - weight_feature_key=feature_name + '_raw_proj_val', - dtype=tf.float32, + tmp_id_col, + weight_feature_key=feature_name + '_raw_proj_val', + dtype=tf.float32, ) if self.is_wide(config): self._add_wide_embedding_column(wgt_fc, config) @@ -416,8 +434,10 @@ def parse_expr_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] - fc = feature_column.numeric_column(feature_name, shape=(1,), feature_name=feature_name) + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] + fc = feature_column.numeric_column( + feature_name, shape=(1,), feature_name=feature_name) if self.is_wide(config): self._add_wide_embedding_column(fc, config) if self.is_deep(config): @@ -429,7 +449,8 @@ def parse_combo_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') else None + feature_name = config.feature_name if config.HasField( + 'feature_name') else None assert len(config.input_names) >= 2 if len(config.combo_join_sep) == 0: @@ -440,16 +461,16 @@ def parse_combo_feature(self, config): else: input_names.append(feature_name + '_' + str(input_id)) fc = feature_column.crossed_column( - input_names, - self._get_hash_bucket_size(config), - hash_key=None, - feature_name=feature_name, + input_names, + self._get_hash_bucket_size(config), + hash_key=None, + feature_name=feature_name, ) else: fc = feature_column.categorical_column_with_hash_bucket( - feature_name, - hash_bucket_size=self._get_hash_bucket_size(config), - feature_name=feature_name, + feature_name, + hash_bucket_size=self._get_hash_bucket_size(config), + feature_name=feature_name, ) if self.is_wide(config): @@ -463,12 +484,15 @@ def parse_lookup_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] assert config.HasField('hash_bucket_size') hash_bucket_size = self._get_hash_bucket_size(config) fc = feature_column.categorical_column_with_hash_bucket( - feature_name, hash_bucket_size, dtype=tf.string, feature_name=feature_name - ) + feature_name, + hash_bucket_size, + dtype=tf.string, + feature_name=feature_name) if self.is_wide(config): self._add_wide_embedding_column(fc, config) @@ -481,82 +505,91 @@ def parse_sequence_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] sub_feature_type = config.sub_feature_type assert sub_feature_type in [ - config.IdFeature, - config.RawFeature, + config.IdFeature, + config.RawFeature, ], 'Current sub_feature_type only support IdFeature and RawFeature.' if sub_feature_type == config.IdFeature: if config.HasField('hash_bucket_size'): hash_bucket_size = self._get_hash_bucket_size(config) fc = sequence_feature_column.sequence_categorical_column_with_hash_bucket( - feature_name, - hash_bucket_size, - dtype=tf.string, - feature_name=feature_name, + feature_name, + hash_bucket_size, + dtype=tf.string, + feature_name=feature_name, ) elif config.vocab_list: fc = sequence_feature_column.sequence_categorical_column_with_vocabulary_list( - feature_name, - default_value=0, - vocabulary_list=config.vocab_list, - feature_name=feature_name, + feature_name, + default_value=0, + vocabulary_list=config.vocab_list, + feature_name=feature_name, ) elif config.vocab_file: fc = sequence_feature_column.sequence_categorical_column_with_vocabulary_file( - feature_name, - default_value=0, - vocabulary_file=config.vocab_file, - vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name, + feature_name, + default_value=0, + vocabulary_file=config.vocab_file, + vocabulary_size=self._get_vocab_size(config.vocab_file), + feature_name=feature_name, ) else: use_ev = self._global_ev_params or config.HasField('ev_params') num_buckets = sys.maxsize if use_ev else config.num_buckets fc = sequence_feature_column.sequence_categorical_column_with_identity( - feature_name, - num_buckets, - default_value=0, - feature_name=feature_name, + feature_name, + num_buckets, + default_value=0, + feature_name=feature_name, ) else: # raw feature bounds = None - fc = sequence_feature_column.sequence_numeric_column(feature_name, shape=(1,), feature_name=feature_name) + fc = sequence_feature_column.sequence_numeric_column( + feature_name, shape=(1,), feature_name=feature_name) if config.hash_bucket_size > 0: hash_bucket_size = self._get_hash_bucket_size(config) assert ( - sub_feature_type == config.IdFeature + sub_feature_type == config.IdFeature ), 'You should set sub_feature_type to IdFeature to use hash_bucket_size.' elif config.boundaries: bounds = list(config.boundaries) bounds.sort() elif config.num_buckets > 1 and config.max_val > config.min_val: # the feature values are already normalized into [0, 1] - bounds = [x / float(config.num_buckets) for x in range(0, config.num_buckets)] - logging.info('sequence feature discrete %s into %d buckets' % (feature_name, config.num_buckets)) + bounds = [ + x / float(config.num_buckets) for x in range(0, config.num_buckets) + ] + logging.info('sequence feature discrete %s into %d buckets' % + (feature_name, config.num_buckets)) if bounds: try: - fc = sequence_feature_column.sequence_numeric_column_with_bucketized_column(fc, bounds) + fc = sequence_feature_column.sequence_numeric_column_with_bucketized_column( + fc, bounds) except Exception as e: - logging.error('sequence features bucketized_column [%s] with bounds %s error' % (feature_name, str(bounds))) + logging.error( + 'sequence features bucketized_column [%s] with bounds %s error' % + (feature_name, str(bounds))) raise e elif config.hash_bucket_size <= 0: if config.embedding_dim > 0: tmp_id_col = sequence_feature_column.sequence_categorical_column_with_identity( - feature_name + '_raw_proj_id', - config.raw_input_dim, - default_value=0, - feature_name=feature_name, + feature_name + '_raw_proj_id', + config.raw_input_dim, + default_value=0, + feature_name=feature_name, ) wgt_fc = sequence_feature_column.sequence_weighted_categorical_column( - tmp_id_col, - weight_feature_key=feature_name + '_raw_proj_val', - dtype=tf.float32, + tmp_id_col, + weight_feature_key=feature_name + '_raw_proj_val', + dtype=tf.float32, ) fc = wgt_fc else: - fc = sequence_feature_column.sequence_numeric_column_with_raw_column(fc, config.sequence_length) + fc = sequence_feature_column.sequence_numeric_column_with_raw_column( + fc, config.sequence_length) if config.embedding_dim > 0: self._add_deep_embedding_column(fc, config) @@ -567,9 +600,11 @@ def _build_partitioner(self, config): if config.max_partitions > 1: if self._global_ev_params is not None or config.HasField('ev_params'): # pai embedding_variable should use fixed_size_partitioner - return partitioned_variables.fixed_size_partitioner(num_shards=config.max_partitions) + return partitioned_variables.fixed_size_partitioner( + num_shards=config.max_partitions) else: - return partitioned_variables.min_max_variable_partitioner(max_partitions=config.max_partitions) + return partitioned_variables.min_max_variable_partitioner( + max_partitions=config.max_partitions) else: return None @@ -597,10 +632,12 @@ def _add_wide_embedding_column(self, fc, config): We use embedding to simulate wide column, which is more efficient than indicator column for sparse features """ - feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] assert self._wide_output_dim > 0, 'wide_output_dim is not set' if config.embedding_name in self._wide_share_embed_columns: - wide_fc = self._add_shared_embedding_column(config.embedding_name, fc, deep=False) + wide_fc = self._add_shared_embedding_column( + config.embedding_name, fc, deep=False) else: initializer = None if config.HasField('initializer'): @@ -610,18 +647,19 @@ def _add_wide_embedding_column(self, fc, config): else: ev_params = self._global_ev_params wide_fc = feature_column.embedding_column( - fc, - self._wide_output_dim, - combiner='sum', - initializer=initializer, - partitioner=self._build_partitioner(config), - ev_params=ev_params, + fc, + self._wide_output_dim, + combiner='sum', + initializer=initializer, + partitioner=self._build_partitioner(config), + ev_params=ev_params, ) self._wide_columns[feature_name] = wide_fc def _add_deep_embedding_column(self, fc, config): """Generate deep feature columns.""" - feature_name = config.feature_name if config.HasField('feature_name') else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] assert config.embedding_dim > 0, 'embedding_dim is not set for %s' % feature_name self._feature_vocab_size[feature_name] = fc.num_buckets if config.embedding_name in self._deep_share_embed_columns: @@ -635,14 +673,15 @@ def _add_deep_embedding_column(self, fc, config): else: ev_params = self._global_ev_params fc = feature_column.embedding_column( - fc, - config.embedding_dim, - combiner=config.combiner, - initializer=initializer, - partitioner=self._build_partitioner(config), - ev_params=ev_params, + fc, + config.embedding_dim, + combiner=config.combiner, + initializer=initializer, + partitioner=self._build_partitioner(config), + ev_params=ev_params, ) - fc.max_seq_length = config.max_seq_len if config.HasField('max_seq_len') else -1 + fc.max_seq_length = config.max_seq_len if config.HasField( + 'max_seq_len') else -1 if config.feature_type != config.SequenceFeature: self._deep_columns[feature_name] = fc @@ -654,10 +693,10 @@ def _add_deep_embedding_column(self, fc, config): def _build_ev_params(self, ev_params): """Build embedding_variables params.""" ev_params = EVParams( - ev_params.filter_freq, - ev_params.steps_to_live if ev_params.steps_to_live > 0 else None, - ev_params.use_cache, - ev_params.init_capacity, - ev_params.max_capacity, + ev_params.filter_freq, + ev_params.steps_to_live if ev_params.steps_to_live > 0 else None, + ev_params.use_cache, + ev_params.init_capacity, + ev_params.max_capacity, ) return ev_params diff --git a/easy_rec/python/feature_column/feature_group.py b/easy_rec/python/feature_column/feature_group.py index 19552542e..b4dcc83da 100644 --- a/easy_rec/python/feature_column/feature_group.py +++ b/easy_rec/python/feature_column/feature_group.py @@ -3,12 +3,12 @@ import re from easy_rec.python.protos.feature_config_pb2 import ( # NOQA - FeatureGroupConfig, - WideOrDeep, + FeatureGroupConfig, WideOrDeep, ) class FeatureGroup(object): + def __init__(self, feature_group_config): self._config = feature_group_config assert isinstance(self._config, FeatureGroupConfig) diff --git a/easy_rec/python/hpo/emr_hpo.py b/easy_rec/python/hpo/emr_hpo.py index 796e603ca..9dd6e4dbd 100644 --- a/easy_rec/python/hpo/emr_hpo.py +++ b/easy_rec/python/hpo/emr_hpo.py @@ -1,7 +1,6 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Hyperparameter search for easy_rec on emr.""" - import argparse import json import logging @@ -14,12 +13,22 @@ from easy_rec.python.utils import hpo_util file_dir, _ = os.path.split(os.path.abspath(__file__)) -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') -def hpo_config(config_path, hyperparams, exp_dir, metric_name, el_submit_params): - earlystop = {'type': 'large_is_better', 'threshold': 0.99, 'max_runtime': 2400} - algorithm = {'type': 'gp', 'initial_trials_num': 4, 'stop_when_exception': True} +def hpo_config(config_path, hyperparams, exp_dir, metric_name, + el_submit_params): + earlystop = { + 'type': 'large_is_better', + 'threshold': 0.99, + 'max_runtime': 2400 + } + algorithm = { + 'type': 'gp', + 'initial_trials_num': 4, + 'stop_when_exception': True + } tmp_dir = '/tmp/emr_easy_rec_hpo_%d' % time.time() os.makedirs(tmp_dir) @@ -30,94 +39,82 @@ def hpo_config(config_path, hyperparams, exp_dir, metric_name, el_submit_params) model_path = '%s/trail_{{ trial.id }}' % exp_dir metric_path = os.path.join(model_path, 'res.metric') - pre_task = {'type': 'BashTask', 'cmd': ['hadoop', 'fs', '-mkdir', '-p', model_path]} + pre_task = { + 'type': 'BashTask', + 'cmd': ['hadoop', 'fs', '-mkdir', '-p', model_path] + } adapter_task = { - 'type': 'localadaptertask', - # hpo_param_path for easy_rec - 'param_file': param_path, + 'type': 'localadaptertask', + # hpo_param_path for easy_rec + 'param_file': param_path, } - el_params = [x.strip() for x in el_submit_params.split(' ') if x.strip() != ''] - assert len(el_params) % 2 == 0, 'invalid number of el_submit params: %d[%s]' % ( - len(el_params), - str(el_params), - ) + el_params = [ + x.strip() for x in el_submit_params.split(' ') if x.strip() != '' + ] + assert len( + el_params) % 2 == 0, 'invalid number of el_submit params: %d[%s]' % ( + len(el_params), str(el_params)) for i in range(0, len(el_params), 2): assert el_params[i] in [ - '-t', - '-m', - '-pn', - '-pc', - '-pg', - '-pm', - '-wn', - '-wc', - '-wm', - '-wg', + '-t', '-m', '-pn', '-pc', '-pg', '-pm', '-wn', '-wc', '-wm', '-wg' ] - cmd = ( - ['el_submit'] - + el_params - + [ - '-a', - 'easy_rec_hpo', - '-m', - 'local', - '-f', - '{},train_eval.py,{}'.format(config_path, param_path), - '--interact', - 'INTERACT', - '-c', + cmd = ['el_submit'] + el_params + [ + '-a', 'easy_rec_hpo', '-m', 'local', '-f', '{},train_eval.py,{}'.format( + config_path, param_path), '--interact', 'INTERACT', '-c', 'python -m easy_rec.python.train_eval --hpo_metric_save_path {} ' '--hpo_param_path {} --pipeline_config_path {} --model_dir {}'.format( - metric_path, param_path_file, config_path, model_path - ), - ] - ) + metric_path, param_path_file, config_path, model_path) + ] train_task = { - 'type': 'BashTask', - 'cmd': cmd, - 'metric_reader': { - 'type': 'hdfs_reader', - 'location': metric_path, - 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name, - }, + 'type': 'BashTask', + 'cmd': cmd, + 'metric_reader': { + 'type': 'hdfs_reader', + 'location': metric_path, + 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name + } } tasks = [pre_task, adapter_task, train_task] data = { - 'earlystop': earlystop, - 'algorithm': algorithm, - 'hyperparams': hyperparams, - 'tasks': tasks, + 'earlystop': earlystop, + 'algorithm': algorithm, + 'hyperparams': hyperparams, + 'tasks': tasks } return data, tmp_dir if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--hyperparams', type=str, help='hyper parameters', default=None) - parser.add_argument('--config_path', type=str, help='pipeline config', default=None) - parser.add_argument('--exp_dir', type=str, help='hpo experiment directory', default=None) parser.add_argument( - '--el_submit_params', - type=str, - help='el_submit parameters(-t x -m x [-pn x -pc x -pm x] -wn x -wc x -wm x -wg x)', - default='-t standalone -m local -wn 1 -wc 6 -wm 20000 -wg 1', - ) - parser.add_argument('--metric_name', type=str, help='metric_name', default='auc') + '--hyperparams', type=str, help='hyper parameters', default=None) + parser.add_argument( + '--config_path', type=str, help='pipeline config', default=None) + parser.add_argument( + '--exp_dir', type=str, help='hpo experiment directory', default=None) + parser.add_argument( + '--el_submit_params', + type=str, + help='el_submit parameters(-t x -m x [-pn x -pc x -pm x] -wn x -wc x -wm x -wg x)', + default='-t standalone -m local -wn 1 -wc 6 -wm 20000 -wg 1') + parser.add_argument( + '--metric_name', type=str, help='metric_name', default='auc') + parser.add_argument( + '--max_parallel', + type=int, + help='max number of trials run at the same time', + default=4) parser.add_argument( - '--max_parallel', - type=int, - help='max number of trials run at the same time', - default=4, - ) - parser.add_argument('--total_trial_num', type=int, help='total number of trials will run', default=6) + '--total_trial_num', + type=int, + help='total number of trials will run', + default=6) parser.add_argument( - '--debug', - action='store_true', - help='debug mode, will keep the temporary folder', - ) + '--debug', + action='store_true', + help='debug mode, will keep the temporary folder') args = parser.parse_args() @@ -128,17 +125,13 @@ def hpo_config(config_path, hyperparams, exp_dir, metric_name, el_submit_params) with open(args.hyperparams, 'r') as fin: hyperparams = json.load(fin) - data, tmp_dir = hpo_config( - args.config_path, - hyperparams, - args.exp_dir, - args.metric_name, - args.el_submit_params, - ) + data, tmp_dir = hpo_config(args.config_path, hyperparams, args.exp_dir, + args.metric_name, args.el_submit_params) hpo_util.kill_old_proc(tmp_dir, platform='emr') - tuner = AutoTuner.create_tuner(data, max_parallel=args.max_parallel, max_trial_num=args.total_trial_num) + tuner = AutoTuner.create_tuner( + data, max_parallel=args.max_parallel, max_trial_num=args.total_trial_num) tuner.fit(synchronize=True) if not args.debug: diff --git a/easy_rec/python/hpo/generate_hpo_sql.py b/easy_rec/python/hpo/generate_hpo_sql.py index 2a0fcf6ea..6b7dc68ea 100644 --- a/easy_rec/python/hpo/generate_hpo_sql.py +++ b/easy_rec/python/hpo/generate_hpo_sql.py @@ -6,31 +6,43 @@ import argparse parser = argparse.ArgumentParser() - parser.add_argument('--sql_path', type=str, help='output sql path', default=None) - parser.add_argument('--config_path', type=str, help='config path', default=None) - parser.add_argument('--tables', type=str, help='train_table and test_table', default=None) - parser.add_argument('--train_tables', type=str, help='train_tables', default=None) - parser.add_argument('--eval_tables', type=str, help='eval_tables', default=None) - parser.add_argument( - '--cluster', - type=str, - help='specify tensorflow train jobs cluster parameter', - default=None, + parser.add_argument( + '--sql_path', type=str, help='output sql path', default=None) + parser.add_argument( + '--config_path', type=str, help='config path', default=None) + parser.add_argument( + '--tables', type=str, help='train_table and test_table', default=None) + parser.add_argument( + '--train_tables', type=str, help='train_tables', default=None) + parser.add_argument( + '--eval_tables', type=str, help='eval_tables', default=None) + parser.add_argument( + '--cluster', + type=str, + help='specify tensorflow train jobs cluster parameter', + default=None, ) parser.add_argument('--bucket', type=str, help='oss bucket', default=None) - parser.add_argument('--hpo_param_path', type=str, help='hpo param path', default=None) - parser.add_argument('--hpo_metric_save_path', type=str, help='hpo metric save path', default=None) + parser.add_argument( + '--hpo_param_path', type=str, help='hpo param path', default=None) + parser.add_argument( + '--hpo_metric_save_path', + type=str, + help='hpo metric save path', + default=None) parser.add_argument('--model_dir', type=str, help='model_dir', default=None) parser.add_argument('--oss_host', type=str, help='oss endpoint', default=None) parser.add_argument('--role_arn', type=str, help='role arn', default=None) parser.add_argument( - '--algo_proj_name', - type=str, - help='algorithm project name', - default='algo_public', + '--algo_proj_name', + type=str, + help='algorithm project name', + default='algo_public', ) - parser.add_argument('--algo_res_proj', type=str, help='algo resource project', default=None) - parser.add_argument('--algo_version', type=str, help='algo version', default=None) + parser.add_argument( + '--algo_res_proj', type=str, help='algo resource project', default=None) + parser.add_argument( + '--algo_version', type=str, help='algo version', default=None) args = parser.parse_args() diff --git a/easy_rec/python/hpo/pai_hpo.py b/easy_rec/python/hpo/pai_hpo.py index 7a909faf8..3473552d7 100644 --- a/easy_rec/python/hpo/pai_hpo.py +++ b/easy_rec/python/hpo/pai_hpo.py @@ -13,14 +13,17 @@ from easy_rec.python.utils import hpo_util file_dir, _ = os.path.split(os.path.abspath(__file__)) -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') try: import subprocess subprocess.check_output('which odpscmd', shell=True) except Exception: - logging.error('odpscmd is not in path, please install from https://help.aliyun.com/document_detail/27971.html') + logging.error( + 'odpscmd is not in path, please install from https://help.aliyun.com/document_detail/27971.html' + ) def get_tuner(data, max_parallel, max_trial_num): @@ -57,39 +60,43 @@ def get_tuner(data, max_parallel, max_trial_num): algo = hpo.algorithm.create(**param_dict['algorithm']) tuner = hpo.autotuner.AutoTuner( - earlystop=early_stop, - algorithm=algo, - hyperparams=params, - task_list=tasks, - max_parallel=max_parallel, - max_trial_num=max_trial_num, - mode='local', - user_id='your_cloud_id', + earlystop=early_stop, + algorithm=algo, + hyperparams=params, + task_list=tasks, + max_parallel=max_parallel, + max_trial_num=max_trial_num, + mode='local', + user_id='your_cloud_id', ) return tuner def hpo_config( - config_path, - hyperparams, - environment, - exp_dir, - tables, - train_tables, - eval_tables, - cluster, - algo_proj_name, - algo_res_proj, - algo_version, - metric_name, - odps_config_path, + config_path, + hyperparams, + environment, + exp_dir, + tables, + train_tables, + eval_tables, + cluster, + algo_proj_name, + algo_res_proj, + algo_version, + metric_name, + odps_config_path, ): earlystop = {'type': 'large_is_better', 'max_runtime': 3600 * 12} - algorithm = {'type': 'gp', 'initial_trials_num': 4, 'stop_when_exception': True} + algorithm = { + 'type': 'gp', + 'initial_trials_num': 4, + 'stop_when_exception': True + } if exp_dir.startswith('oss://'): exp_dir = exp_dir.replace('oss://', '') - exp_dir = exp_dir[exp_dir.find('/') + 1 :] + exp_dir = exp_dir[exp_dir.find('/') + 1:] param_path = '%s/hpo_test_{{ trial.id }}.json' % exp_dir metric_path = '%s/easy_rec_hpo_{{ trial.id }}.metric' % exp_dir @@ -97,9 +104,9 @@ def hpo_config( bucket = 'oss://' + environment['bucket'].strip('/') + '/' adapter_task = { - 'type': 'ossadaptertask', - # hpo_param_path for easy_rec - 'param_file': param_path, + 'type': 'ossadaptertask', + # hpo_param_path for easy_rec + 'param_file': param_path, } tmp_dir = '/tmp/pai_easy_rec_hpo_%d' % time.time() @@ -125,35 +132,36 @@ def _add_prefix(table_name): sql_path = '%s/train_ext_hpo_{{ trial.id }}.sql' % tmp_dir cmd_args = [ - 'python', - '-m', - 'easy_rec.python.hpo.generate_hpo_sql', - '--sql_path', - sql_path, - '--config_path', - config_path, - '--cluster', - cluster, - '--bucket', - bucket, - '--hpo_param_path', - os.path.join(bucket, param_path), - '--hpo_metric_save_path', - os.path.join(bucket, metric_path), - '--model_dir', - os.path.join(bucket, model_path), - '--oss_host', - environment['oss_endpoint'], - '--role_arn', - environment['role_arn'], - '--algo_proj_name', - algo_proj_name, + 'python', + '-m', + 'easy_rec.python.hpo.generate_hpo_sql', + '--sql_path', + sql_path, + '--config_path', + config_path, + '--cluster', + cluster, + '--bucket', + bucket, + '--hpo_param_path', + os.path.join(bucket, param_path), + '--hpo_metric_save_path', + os.path.join(bucket, metric_path), + '--model_dir', + os.path.join(bucket, model_path), + '--oss_host', + environment['oss_endpoint'], + '--role_arn', + environment['role_arn'], + '--algo_proj_name', + algo_proj_name, ] if tables: cmd_args.extend(['--tables', tables]) if train_tables and eval_tables: - cmd_args.extend(['--train_tables', train_tables, '--eval_tables', eval_tables]) + cmd_args.extend( + ['--train_tables', train_tables, '--eval_tables', eval_tables]) if algo_res_proj: cmd_args.extend(['--algo_res_proj', algo_res_proj]) @@ -162,22 +170,23 @@ def _add_prefix(table_name): prepare_sql_task = {'type': 'BashTask', 'cmd': cmd_args} train_task = { - 'type': 'BashTask', - 'cmd': ['odpscmd', '--config=%s' % odps_config_path, '-f', sql_path], - 'metric_reader': { - 'type': 'oss_reader', - 'location': metric_path, - 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name, - }, + 'type': 'BashTask', + 'cmd': ['odpscmd', + '--config=%s' % odps_config_path, '-f', sql_path], + 'metric_reader': { + 'type': 'oss_reader', + 'location': metric_path, + 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name, + }, } tasks = [adapter_task, prepare_sql_task, train_task] data = { - 'earlystop': earlystop, - 'algorithm': algorithm, - 'hyperparams': hyperparams, - 'tasks': tasks, - 'environment': environment, + 'earlystop': earlystop, + 'algorithm': algorithm, + 'hyperparams': hyperparams, + 'tasks': tasks, + 'environment': environment, } return data, tmp_dir @@ -186,37 +195,56 @@ def _add_prefix(table_name): import argparse parser = argparse.ArgumentParser() - parser.add_argument('--odps_config', type=str, help='odps_config.ini', default=None) - parser.add_argument('--oss_config', type=str, help='excel config path', default='') + parser.add_argument( + '--odps_config', type=str, help='odps_config.ini', default=None) + parser.add_argument( + '--oss_config', type=str, help='excel config path', default='') parser.add_argument('--bucket', type=str, help='bucket name', default=None) parser.add_argument('--role_arn', type=str, help='role arn', default=None) - parser.add_argument('--hyperparams', type=str, help='hyper parameters', default=None) - parser.add_argument('--config_path', type=str, help='pipeline config', default=None) - parser.add_argument('--tables', type=str, help='train table and test table', default=None) - parser.add_argument('--train_tables', type=str, help='train tables', default=None) - parser.add_argument('--eval_tables', type=str, help='eval tables', default=None) - parser.add_argument('--exp_dir', type=str, help='hpo experiment directory', default=None) parser.add_argument( - '--cluster', - type=str, - help='cluster spec', - default='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}', + '--hyperparams', type=str, help='hyper parameters', default=None) + parser.add_argument( + '--config_path', type=str, help='pipeline config', default=None) + parser.add_argument( + '--tables', type=str, help='train table and test table', default=None) + parser.add_argument( + '--train_tables', type=str, help='train tables', default=None) + parser.add_argument( + '--eval_tables', type=str, help='eval tables', default=None) + parser.add_argument( + '--exp_dir', type=str, help='hpo experiment directory', default=None) + parser.add_argument( + '--cluster', + type=str, + help='cluster spec', + default='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}', ) - parser.add_argument('--algo_proj_name', type=str, help='algo project name', default='algo_public') - parser.add_argument('--algo_version', type=str, help='algo version', default=None) - parser.add_argument('--algo_res_proj', type=str, help='algo resource project', default=None) - parser.add_argument('--metric_name', type=str, help='evaluate metric name', default='auc') parser.add_argument( - '--max_parallel', - type=int, - help='max number of trials run at the same time', - default=4, + '--algo_proj_name', + type=str, + help='algo project name', + default='algo_public') + parser.add_argument( + '--algo_version', type=str, help='algo version', default=None) + parser.add_argument( + '--algo_res_proj', type=str, help='algo resource project', default=None) + parser.add_argument( + '--metric_name', type=str, help='evaluate metric name', default='auc') + parser.add_argument( + '--max_parallel', + type=int, + help='max number of trials run at the same time', + default=4, ) - parser.add_argument('--total_trial_num', type=int, help='total number of trials will run', default=6) parser.add_argument( - '--debug', - action='store_true', - help='debug mode, will keep the temporary folder', + '--total_trial_num', + type=int, + help='total number of trials will run', + default=6) + parser.add_argument( + '--debug', + action='store_true', + help='debug mode, will keep the temporary folder', ) args = parser.parse_args() @@ -233,7 +261,7 @@ def _add_prefix(table_name): if '=' in line_str: tmp_id = line_str.find('=') key = line_str[:tmp_id].strip() - val = line_str[(tmp_id + 1) :].strip() + val = line_str[(tmp_id + 1):].strip() odps_config[key] = val if args.oss_config is None: @@ -250,27 +278,27 @@ def _add_prefix(table_name): if '=' in line_str: tmp_id = line_str.find('=') key = line_str[:tmp_id].strip() - val = line_str[(tmp_id + 1) :].strip() + val = line_str[(tmp_id + 1):].strip() oss_config[key] = val assert args.bucket is not None assert args.role_arn is not None if args.bucket.startswith('oss://'): - args.bucket = args.bucket[len('oss://') :] + args.bucket = args.bucket[len('oss://'):] args.bucket = args.bucket.strip('/') environment = { - 'access_id': odps_config['access_id'], - 'access_key': odps_config['access_key'], - 'oss_access_id': oss_config['accessKeyID'], - 'oss_access_key': oss_config['accessKeySecret'], - 'project': odps_config['project_name'], - 'odps_endpoint': odps_config['end_point'], - 'biz_id': '147331^paistudio^xxxxxxx^2020-03-18', - 'role_arn': args.role_arn, - 'bucket': args.bucket, - 'oss_endpoint': oss_config['endpoint'], + 'access_id': odps_config['access_id'], + 'access_key': odps_config['access_key'], + 'oss_access_id': oss_config['accessKeyID'], + 'oss_access_key': oss_config['accessKeySecret'], + 'project': odps_config['project_name'], + 'odps_endpoint': odps_config['end_point'], + 'biz_id': '147331^paistudio^xxxxxxx^2020-03-18', + 'role_arn': args.role_arn, + 'bucket': args.bucket, + 'oss_endpoint': oss_config['endpoint'], } assert args.hyperparams is not None @@ -279,22 +307,23 @@ def _add_prefix(table_name): assert args.config_path is not None assert args.exp_dir is not None - assert args.tables is not None or (args.train_tables is not None and args.eval_tables is not None) + assert args.tables is not None or (args.train_tables is not None and + args.eval_tables is not None) data, tmp_dir = hpo_config( - args.config_path, - hyperparams, - environment, - args.exp_dir, - args.tables, - args.train_tables, - args.eval_tables, - args.cluster, - args.algo_proj_name, - args.algo_res_proj, - args.algo_version, - args.metric_name, - args.odps_config, + args.config_path, + hyperparams, + environment, + args.exp_dir, + args.tables, + args.train_tables, + args.eval_tables, + args.cluster, + args.algo_proj_name, + args.algo_res_proj, + args.algo_version, + args.metric_name, + args.odps_config, ) hpo_util.kill_old_proc(tmp_dir, platform='pai') diff --git a/easy_rec/python/inference/client/client_demo.py b/easy_rec/python/inference/client/client_demo.py index f2608bdcf..e6bd4df1f 100644 --- a/easy_rec/python/inference/client/client_demo.py +++ b/easy_rec/python/inference/client/client_demo.py @@ -6,9 +6,11 @@ import traceback from easy_rec.python.inference.client.easyrec_request import EasyrecRequest -from easy_rec.python.protos.predict_pb2 import PBFeature, PBRequest +from easy_rec.python.protos.predict_pb2 import PBFeature +from easy_rec.python.protos.predict_pb2 import PBRequest -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') try: from eas_prediction import PredictClient # TFRequest @@ -47,8 +49,8 @@ def build_request(table_cols, table_data, item_ids=None): def parse_table_schema(create_table_sql): create_table_sql = create_table_sql.lower() spos = create_table_sql.index('(') - epos = create_table_sql[spos + 1 :].index(')') - cols = create_table_sql[(spos + 1) : epos] + epos = create_table_sql[spos + 1:].index(')') + cols = create_table_sql[(spos + 1):epos] cols = [x.strip().lower() for x in cols.split(',')] col_info_arr = [] for col in cols: @@ -69,15 +71,25 @@ def send_request(req_pb, client, debug_level=0): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--endpoint', - type=str, - default=None, - help='eas endpoint, such as 12345.cn-beijing.pai-eas.aliyuncs.com', + '--endpoint', + type=str, + default=None, + help='eas endpoint, such as 12345.cn-beijing.pai-eas.aliyuncs.com', ) - parser.add_argument('--service_name', type=str, default=None, help='eas service name') - parser.add_argument('--token', type=str, default=None, help='eas service token') - parser.add_argument('--table_schema', type=str, default=None, help='user feature table schema path') - parser.add_argument('--table_data', type=str, default=None, help='user feature table data path') + parser.add_argument( + '--service_name', type=str, default=None, help='eas service name') + parser.add_argument( + '--token', type=str, default=None, help='eas service token') + parser.add_argument( + '--table_schema', + type=str, + default=None, + help='user feature table schema path') + parser.add_argument( + '--table_data', + type=str, + default=None, + help='user feature table data path') parser.add_argument('--item_lst', type=str, default=None, help='item list') args, _ = parser.parse_known_args() diff --git a/easy_rec/python/inference/client/easyrec_request.py b/easy_rec/python/inference/client/easyrec_request.py index ba44446d9..4980b5064 100644 --- a/easy_rec/python/inference/client/easyrec_request.py +++ b/easy_rec/python/inference/client/easyrec_request.py @@ -2,7 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from eas_prediction.request import Request -from easy_rec.python.protos.predict_pb2 import PBRequest, PBResponse +from easy_rec.python.protos.predict_pb2 import PBRequest +from easy_rec.python.protos.predict_pb2 import PBResponse # from eas_prediction.request import Response diff --git a/easy_rec/python/inference/csv_predictor.py b/easy_rec/python/inference/csv_predictor.py index 0a1ffe69d..ae5fc1fcc 100644 --- a/easy_rec/python/inference/csv_predictor.py +++ b/easy_rec/python/inference/csv_predictor.py @@ -1,6 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import logging import os @@ -8,28 +10,29 @@ import tensorflow as tf from tensorflow.python.platform import gfile -from easy_rec.python.inference.predictor import ( # NOQA - SINGLE_PLACEHOLDER_FEATURE_KEY, - Predictor, -) from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils.check_utils import check_split +from easy_rec.python.inference.predictor import ( # NOQA + SINGLE_PLACEHOLDER_FEATURE_KEY, Predictor, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 class CSVPredictor(Predictor): + def __init__( - self, - model_path, - data_config, - with_header=False, - ds_vector_recall=False, - fg_json_path=None, - profiling_file=None, - selected_cols=None, - output_sep=chr(1), + self, + model_path, + data_config, + with_header=False, + ds_vector_recall=False, + fg_json_path=None, + profiling_file=None, + selected_cols=None, + output_sep=chr(1), ): super(CSVPredictor, self).__init__(model_path, profiling_file, fg_json_path) self._output_sep = output_sep @@ -74,18 +77,19 @@ def _get_reserved_cols(self, reserved_cols): def _parse_line(self, line): check_list = [ - tf.py_func( - check_split, - [line, self._input_sep, len(self._record_defaults)], - Tout=tf.bool, - ) + tf.py_func( + check_split, + [line, self._input_sep, + len(self._record_defaults)], + Tout=tf.bool, + ) ] with tf.control_dependencies(check_list): fields = tf.decode_csv( - line, - field_delim=self._input_sep, - record_defaults=self._record_defaults, - name='decode_csv', + line, + field_delim=self._input_sep, + record_defaults=self._record_defaults, + name='decode_csv', ) if self._is_rtp: if self._with_header: @@ -113,9 +117,8 @@ def _get_num_cols(self, file_paths): line_tok = line_str.strip().split(self._input_sep) if num_cols != -1: assert num_cols == len(line_tok), ( - 'num selected cols is %d, not equal to %d, current line is: %s, please check input_sep and data' - % (num_cols, len(line_tok), line_str) - ) + 'num selected cols is %d, not equal to %d, current line is: %s, please check input_sep and data' + % (num_cols, len(line_tok), line_str)) num_cols = len(line_tok) num_lines += 1 if num_lines > 10: @@ -123,7 +126,8 @@ def _get_num_cols(self, file_paths): logging.info('num selected cols = %d' % num_cols) return num_cols - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, + slice_id): file_paths = [] for path in input_path.split(','): for x in gfile.Glob(path): @@ -153,14 +157,16 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, sl default_val = self._get_defaults(col_name) self._record_defaults[col_idx] = default_val else: - self._record_defaults = [self._get_defaults(col_name) for col_name in self._all_fields] + self._record_defaults = [ + self._get_defaults(col_name) for col_name in self._all_fields + ] dataset = tf.data.Dataset.from_tensor_slices(file_paths) parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset(x).skip(int(self._with_header)), - cycle_length=parallel_num, - num_parallel_calls=parallel_num, + lambda x: tf.data.TextLineDataset(x).skip(int(self._with_header)), + cycle_length=parallel_num, + num_parallel_calls=parallel_num, ) dataset = dataset.shard(slice_num, slice_id) dataset = dataset.batch(batch_size) @@ -172,15 +178,18 @@ def _get_writer(self, output_path, slice_id): gfile.MakeDirs(output_path) res_path = os.path.join(output_path, 'part-%d.csv' % slice_id) table_writer = gfile.GFile(res_path, 'w') - table_writer.write(self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') + table_writer.write( + self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') return table_writer def _write_lines(self, table_writer, outputs): - outputs = '\n'.join([self._output_sep.join([str(i) for i in output]) for output in outputs]) + outputs = '\n'.join( + [self._output_sep.join([str(i) for i in output]) for output in outputs]) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [outputs[x] for x in output_cols] + [all_vals[k] for k in reserved_cols] + reserve_vals = [outputs[x] for x in output_cols + ] + [all_vals[k] for k in reserved_cols] return reserve_vals @property diff --git a/easy_rec/python/inference/hive_parquet_predictor.py b/easy_rec/python/inference/hive_parquet_predictor.py index 33f9ae84e..77cbf291b 100644 --- a/easy_rec/python/inference/hive_parquet_predictor.py +++ b/easy_rec/python/inference/hive_parquet_predictor.py @@ -1,6 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import os import time @@ -21,18 +23,20 @@ class HiveParquetPredictor(Predictor): + def __init__( - self, - model_path, - data_config, - hive_config, - fg_json_path=None, - profiling_file=None, - output_sep=chr(1), - all_cols=None, - all_col_types=None, + self, + model_path, + data_config, + hive_config, + fg_json_path=None, + profiling_file=None, + output_sep=chr(1), + all_cols=None, + all_col_types=None, ): - super(HiveParquetPredictor, self).__init__(model_path, profiling_file, fg_json_path) + super(HiveParquetPredictor, self).__init__(model_path, profiling_file, + fg_json_path) self._data_config = data_config self._hive_config = hive_config @@ -45,7 +49,8 @@ def __init__( self._all_cols = [x.strip() for x in all_cols if x != ''] self._all_col_types = [x.strip() for x in all_col_types if x != ''] self._record_defaults = [ - self._get_defaults(col_name, col_type) for col_name, col_type in zip(self._all_cols, self._all_col_types) + self._get_defaults(col_name, col_type) + for col_name, col_type in zip(self._all_cols, self._all_col_types) ] def _get_reserved_cols(self, reserved_cols): @@ -60,20 +65,24 @@ def _parse_line(self, *fields): field_dict = {self._all_cols[i]: fields[i] for i in range(len(fields))} return field_dict - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): - self._hive_util = HiveUtils(data_config=self._data_config, hive_config=self._hive_config) + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, + slice_id): + self._hive_util = HiveUtils( + data_config=self._data_config, hive_config=self._hive_config) hdfs_path = self._hive_util.get_table_location(input_path) self._input_hdfs_path = gfile.Glob(os.path.join(hdfs_path, '*')) assert len(self._input_hdfs_path) > 0, 'match no files with %s' % input_path list_type = [] - input_field_type_map = {x.input_name: x.input_type for x in self._data_config.input_fields} + input_field_type_map = { + x.input_name: x.input_type for x in self._data_config.input_fields + } type_2_tftype = { - 'string': tf.string, - 'double': tf.double, - 'float': tf.float32, - 'bigint': tf.int32, - 'boolean': tf.bool, + 'string': tf.string, + 'double': tf.double, + 'float': tf.float32, + 'bigint': tf.int32, + 'boolean': tf.bool, } for col_name, col_type in zip(self._all_cols, self._all_col_types): if col_name in input_field_type_map: @@ -105,7 +114,8 @@ def parquet_read(): inputs.append(batch_data[k].to_numpy()) yield tuple(inputs) - dataset = tf.data.Dataset.from_generator(parquet_read, output_types=list_type, output_shapes=list_shapes) + dataset = tf.data.Dataset.from_generator( + parquet_read, output_types=list_type, output_shapes=list_shapes) dataset = dataset.shard(slice_num, slice_id) dataset = dataset.prefetch(buffer_size=64) return dataset @@ -121,13 +131,14 @@ def get_table_info(self, output_path): def _get_writer(self, output_path, slice_id): table_name, partition_name, partition_val = self.get_table_info(output_path) - is_exist = self._hive_util.is_table_or_partition_exist(table_name, partition_name, partition_val) + is_exist = self._hive_util.is_table_or_partition_exist( + table_name, partition_name, partition_val) assert not is_exist, '%s is already exists. Please drop it.' % output_path output_path = output_path.replace('.', '/') self._hdfs_path = 'hdfs://%s:9000/user/easy_rec/%s_tmp' % ( - self._hive_config.host, - output_path, + self._hive_config.host, + output_path, ) if not gfile.Exists(self._hdfs_path): gfile.MakeDirs(self._hdfs_path) @@ -136,11 +147,13 @@ def _get_writer(self, output_path, slice_id): return table_writer def _write_lines(self, table_writer, outputs): - outputs = '\n'.join([self._output_sep.join([str(i) for i in output]) for output in outputs]) + outputs = '\n'.join( + [self._output_sep.join([str(i) for i in output]) for output in outputs]) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [outputs[x] for x in output_cols] + [all_vals[k] for k in reserved_cols] + reserve_vals = [outputs[x] for x in output_cols + ] + [all_vals[k] for k in reserved_cols] return reserve_vals def load_to_table(self, output_path, slice_num, slice_id): @@ -173,24 +186,24 @@ def load_to_table(self, output_path, slice_num, slice_id): if partition_name and partition_val: sql = 'create table if not exists %s (%s) PARTITIONED BY (%s string)' % ( - table_name, - schema, - partition_name, + table_name, + schema, + partition_name, ) self._hive_util.run_sql(sql) sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s PARTITION (%s=%s)" % ( - self._hdfs_path, - table_name, - partition_name, - partition_val, + self._hdfs_path, + table_name, + partition_name, + partition_val, ) self._hive_util.run_sql(sql) else: sql = 'create table if not exists %s (%s)' % (table_name, schema) self._hive_util.run_sql(sql) sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s" % ( - self._hdfs_path, - table_name, + self._hdfs_path, + table_name, ) self._hive_util.run_sql(sql) diff --git a/easy_rec/python/inference/hive_predictor.py b/easy_rec/python/inference/hive_predictor.py index f199bad6c..043bf5463 100644 --- a/easy_rec/python/inference/hive_predictor.py +++ b/easy_rec/python/inference/hive_predictor.py @@ -1,6 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import os import time @@ -18,18 +20,20 @@ class HivePredictor(Predictor): + def __init__( - self, - model_path, - data_config, - hive_config, - fg_json_path=None, - profiling_file=None, - output_sep=chr(1), - all_cols=None, - all_col_types=None, + self, + model_path, + data_config, + hive_config, + fg_json_path=None, + profiling_file=None, + output_sep=chr(1), + all_cols=None, + all_col_types=None, ): - super(HivePredictor, self).__init__(model_path, profiling_file, fg_json_path) + super(HivePredictor, self).__init__(model_path, profiling_file, + fg_json_path) self._data_config = data_config self._hive_config = hive_config @@ -42,7 +46,8 @@ def __init__( self._all_cols = [x.strip() for x in all_cols if x != ''] self._all_col_types = [x.strip() for x in all_col_types if x != ''] self._record_defaults = [ - self._get_defaults(col_name, col_type) for col_name, col_type in zip(self._all_cols, self._all_col_types) + self._get_defaults(col_name, col_type) + for col_name, col_type in zip(self._all_cols, self._all_col_types) ] def _get_reserved_cols(self, reserved_cols): @@ -55,16 +60,18 @@ def _get_reserved_cols(self, reserved_cols): def _parse_line(self, line): field_delim = self._data_config.rtp_separator if self._is_rtp else self._data_config.separator fields = tf.decode_csv( - line, - field_delim=field_delim, - record_defaults=self._record_defaults, - name='decode_csv', + line, + field_delim=field_delim, + record_defaults=self._record_defaults, + name='decode_csv', ) inputs = {self._all_cols[x]: fields[x] for x in range(len(fields))} return inputs - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): - self._hive_util = HiveUtils(data_config=self._data_config, hive_config=self._hive_config) + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, + slice_id): + self._hive_util = HiveUtils( + data_config=self._data_config, hive_config=self._hive_config) self._input_hdfs_path = self._hive_util.get_table_location(input_path) file_paths = tf.gfile.Glob(os.path.join(self._input_hdfs_path, '*')) assert len(file_paths) > 0, 'match no files with %s' % input_path @@ -72,9 +79,9 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, sl dataset = tf.data.Dataset.from_tensor_slices(file_paths) parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - tf.data.TextLineDataset, - cycle_length=parallel_num, - num_parallel_calls=parallel_num, + tf.data.TextLineDataset, + cycle_length=parallel_num, + num_parallel_calls=parallel_num, ) dataset = dataset.shard(slice_num, slice_id) dataset = dataset.batch(batch_size) @@ -92,13 +99,14 @@ def get_table_info(self, output_path): def _get_writer(self, output_path, slice_id): table_name, partition_name, partition_val = self.get_table_info(output_path) - is_exist = self._hive_util.is_table_or_partition_exist(table_name, partition_name, partition_val) + is_exist = self._hive_util.is_table_or_partition_exist( + table_name, partition_name, partition_val) assert not is_exist, '%s is already exists. Please drop it.' % output_path output_path = output_path.replace('.', '/') self._hdfs_path = 'hdfs://%s:9000/user/easy_rec/%s_tmp' % ( - self._hive_config.host, - output_path, + self._hive_config.host, + output_path, ) if not gfile.Exists(self._hdfs_path): gfile.MakeDirs(self._hdfs_path) @@ -107,11 +115,13 @@ def _get_writer(self, output_path, slice_id): return table_writer def _write_lines(self, table_writer, outputs): - outputs = '\n'.join([self._output_sep.join([str(i) for i in output]) for output in outputs]) + outputs = '\n'.join( + [self._output_sep.join([str(i) for i in output]) for output in outputs]) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [outputs[x] for x in output_cols] + [all_vals[k] for k in reserved_cols] + reserve_vals = [outputs[x] for x in output_cols + ] + [all_vals[k] for k in reserved_cols] return reserve_vals def load_to_table(self, output_path, slice_num, slice_id): @@ -144,24 +154,24 @@ def load_to_table(self, output_path, slice_num, slice_id): if partition_name and partition_val: sql = 'create table if not exists %s (%s) PARTITIONED BY (%s string)' % ( - table_name, - schema, - partition_name, + table_name, + schema, + partition_name, ) self._hive_util.run_sql(sql) sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s PARTITION (%s=%s)" % ( - self._hdfs_path, - table_name, - partition_name, - partition_val, + self._hdfs_path, + table_name, + partition_name, + partition_val, ) self._hive_util.run_sql(sql) else: sql = 'create table if not exists %s (%s)' % (table_name, schema) self._hive_util.run_sql(sql) sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s" % ( - self._hdfs_path, - table_name, + self._hdfs_path, + table_name, ) self._hive_util.run_sql(sql) diff --git a/easy_rec/python/inference/odps_predictor.py b/easy_rec/python/inference/odps_predictor.py index 737ea52b5..1cdb4b9e2 100644 --- a/easy_rec/python/inference/odps_predictor.py +++ b/easy_rec/python/inference/odps_predictor.py @@ -1,6 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import tensorflow as tf @@ -8,19 +10,24 @@ class ODPSPredictor(Predictor): + def __init__( - self, - model_path, - fg_json_path=None, - profiling_file=None, - all_cols='', - all_col_types='', + self, + model_path, + fg_json_path=None, + profiling_file=None, + all_cols='', + all_col_types='', ): - super(ODPSPredictor, self).__init__(model_path, profiling_file, fg_json_path) + super(ODPSPredictor, self).__init__(model_path, profiling_file, + fg_json_path) self._all_cols = [x.strip() for x in all_cols.split(',') if x != ''] - self._all_col_types = [x.strip() for x in all_col_types.split(',') if x != ''] + self._all_col_types = [ + x.strip() for x in all_col_types.split(',') if x != '' + ] self._record_defaults = [ - self._get_defaults(col_name, col_type) for col_name, col_type in zip(self._all_cols, self._all_col_types) + self._get_defaults(col_name, col_type) + for col_name, col_type in zip(self._all_cols, self._all_col_types) ] def _get_reserved_cols(self, reserved_cols): @@ -32,14 +39,15 @@ def _parse_line(self, *fields): field_dict = {self._all_cols[i]: fields[i] for i in range(len(fields))} return field_dict - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, + slice_id): input_list = input_path.split(',') dataset = tf.data.TableRecordDataset( - input_list, - record_defaults=self._record_defaults, - slice_id=slice_id, - slice_count=slice_num, - selected_cols=','.join(self._all_cols), + input_list, + record_defaults=self._record_defaults, + slice_id=slice_id, + slice_count=slice_num, + selected_cols=','.join(self._all_cols), ) dataset = dataset.batch(batch_size) dataset = dataset.prefetch(buffer_size=64) @@ -61,5 +69,6 @@ def out_of_range_exception(self): return (tf.python_io.OutOfRangeException, tf.errors.OutOfRangeError) def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [all_vals[k] for k in reserved_cols] + [outputs[x] for x in output_cols] + reserve_vals = [all_vals[k] for k in reserved_cols + ] + [outputs[x] for x in output_cols] return reserve_vals diff --git a/easy_rec/python/inference/parquet_predictor.py b/easy_rec/python/inference/parquet_predictor.py index 4669f26ed..7f59e6573 100644 --- a/easy_rec/python/inference/parquet_predictor.py +++ b/easy_rec/python/inference/parquet_predictor.py @@ -1,6 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import logging import os @@ -13,7 +15,8 @@ from easy_rec.python.inference.predictor import Predictor from easy_rec.python.input.parquet_input import ParquetInput from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.utils import config_util, input_utils +from easy_rec.python.utils import config_util +from easy_rec.python.utils import input_utils try: from tensorflow.python.framework.load_library import load_op_library @@ -27,18 +30,20 @@ class ParquetPredictor(Predictor): + def __init__( - self, - model_path, - data_config, - ds_vector_recall=False, - fg_json_path=None, - profiling_file=None, - selected_cols=None, - output_sep=chr(1), - pipeline_config=None, + self, + model_path, + data_config, + ds_vector_recall=False, + fg_json_path=None, + profiling_file=None, + selected_cols=None, + output_sep=chr(1), + pipeline_config=None, ): - super(ParquetPredictor, self).__init__(model_path, profiling_file, fg_json_path) + super(ParquetPredictor, self).__init__(model_path, profiling_file, + fg_json_path) self._output_sep = output_sep self._ds_vector_recall = ds_vector_recall input_type = DatasetConfig.InputType.Name(data_config.input_type).lower() @@ -72,8 +77,10 @@ def _get_reserved_cols(self, reserved_cols): # already parsed in _get_dataset return self._reserved_cols - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): - feature_configs = config_util.get_compatible_feature_configs(self.pipeline_config) + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, + slice_id): + feature_configs = config_util.get_compatible_feature_configs( + self.pipeline_config) kwargs = {} if self._reserved_args is not None and len(self._reserved_args) > 0: @@ -85,32 +92,33 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, sl kwargs['reserve_fields'] = all_cols self._all_fields = all_cols self._reserved_cols = all_cols - kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file(all_cols, parquet_file) + kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( + all_cols, parquet_file) else: - self._reserved_cols = [x.strip() for x in self._reserved_args.split(',') if x.strip() != ''] + self._reserved_cols = [ + x.strip() for x in self._reserved_args.split(',') if x.strip() != '' + ] kwargs['reserve_fields'] = self._reserved_cols parquet_file = gfile.Glob(input_path.split(',')[0])[0] - kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file(self._reserved_cols, parquet_file) - logging.info( - 'reserve_fields=%s reserve_types=%s' - % ( + kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( + self._reserved_cols, parquet_file) + logging.info('reserve_fields=%s reserve_types=%s' % ( ','.join(self._reserved_cols), ','.join([str(x) for x in kwargs['reserve_types']]), - ) - ) + )) else: self._reserved_cols = [] self.pipeline_config.data_config.batch_size = batch_size kwargs['is_predictor'] = True parquet_input = ParquetInput( - self.pipeline_config.data_config, - feature_configs, - input_path, - task_index=slice_id, - task_num=slice_num, - pipeline_config=self.pipeline_config, - **kwargs, + self.pipeline_config.data_config, + feature_configs, + input_path, + task_index=slice_id, + task_num=slice_num, + pipeline_config=self.pipeline_config, + **kwargs, ) return parquet_input._build(tf.estimator.ModeKeys.PREDICT, {}) @@ -119,11 +127,13 @@ def _get_writer(self, output_path, slice_id): gfile.MakeDirs(output_path) res_path = os.path.join(output_path, 'part-%d.csv' % slice_id) table_writer = gfile.GFile(res_path, 'w') - table_writer.write(self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') + table_writer.write( + self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') return table_writer def _write_lines(self, table_writer, outputs): - outputs = '\n'.join([self._output_sep.join([str(i) for i in output]) for output in outputs]) + outputs = '\n'.join( + [self._output_sep.join([str(i) for i in output]) for output in outputs]) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): diff --git a/easy_rec/python/inference/parquet_predictor_v2.py b/easy_rec/python/inference/parquet_predictor_v2.py index b936e6740..c3dbf3078 100644 --- a/easy_rec/python/inference/parquet_predictor_v2.py +++ b/easy_rec/python/inference/parquet_predictor_v2.py @@ -1,6 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import logging import os @@ -13,7 +15,8 @@ from easy_rec.python.inference.predictor import Predictor from easy_rec.python.input.parquet_input_v2 import ParquetInputV2 from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.utils import config_util, input_utils +from easy_rec.python.utils import config_util +from easy_rec.python.utils import input_utils try: from tensorflow.python.framework.load_library import load_op_library @@ -27,18 +30,20 @@ class ParquetPredictorV2(Predictor): + def __init__( - self, - model_path, - data_config, - ds_vector_recall=False, - fg_json_path=None, - profiling_file=None, - selected_cols=None, - output_sep=chr(1), - pipeline_config=None, + self, + model_path, + data_config, + ds_vector_recall=False, + fg_json_path=None, + profiling_file=None, + selected_cols=None, + output_sep=chr(1), + pipeline_config=None, ): - super(ParquetPredictorV2, self).__init__(model_path, profiling_file, fg_json_path) + super(ParquetPredictorV2, self).__init__(model_path, profiling_file, + fg_json_path) self._output_sep = output_sep self._ds_vector_recall = ds_vector_recall input_type = DatasetConfig.InputType.Name(data_config.input_type).lower() @@ -72,8 +77,10 @@ def _get_reserved_cols(self, reserved_cols): # already parsed in _get_dataset return self._reserved_cols - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): - feature_configs = config_util.get_compatible_feature_configs(self.pipeline_config) + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, + slice_id): + feature_configs = config_util.get_compatible_feature_configs( + self.pipeline_config) kwargs = {} if self._reserved_args is not None and len(self._reserved_args) > 0: @@ -85,32 +92,33 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, sl kwargs['reserve_fields'] = all_cols self._all_fields = all_cols self._reserved_cols = all_cols - kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file(all_cols, parquet_file) + kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( + all_cols, parquet_file) else: - self._reserved_cols = [x.strip() for x in self._reserved_args.split(',') if x.strip() != ''] + self._reserved_cols = [ + x.strip() for x in self._reserved_args.split(',') if x.strip() != '' + ] kwargs['reserve_fields'] = self._reserved_cols parquet_file = gfile.Glob(input_path.split(',')[0])[0] - kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file(self._reserved_cols, parquet_file) - logging.info( - 'reserve_fields=%s reserve_types=%s' - % ( + kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( + self._reserved_cols, parquet_file) + logging.info('reserve_fields=%s reserve_types=%s' % ( ','.join(self._reserved_cols), ','.join([str(x) for x in kwargs['reserve_types']]), - ) - ) + )) else: self._reserved_cols = [] self.pipeline_config.data_config.batch_size = batch_size kwargs['is_predictor'] = True parquet_input = ParquetInputV2( - self.pipeline_config.data_config, - feature_configs, - input_path, - task_index=slice_id, - task_num=slice_num, - pipeline_config=self.pipeline_config, - **kwargs, + self.pipeline_config.data_config, + feature_configs, + input_path, + task_index=slice_id, + task_num=slice_num, + pipeline_config=self.pipeline_config, + **kwargs, ) return parquet_input._build(tf.estimator.ModeKeys.PREDICT, {}) @@ -119,11 +127,13 @@ def _get_writer(self, output_path, slice_id): gfile.MakeDirs(output_path) res_path = os.path.join(output_path, 'part-%d.csv' % slice_id) table_writer = gfile.GFile(res_path, 'w') - table_writer.write(self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') + table_writer.write( + self._output_sep.join(self._output_cols + self._reserved_cols) + '\n') return table_writer def _write_lines(self, table_writer, outputs): - outputs = '\n'.join([self._output_sep.join([str(i) for i in output]) for output in outputs]) + outputs = '\n'.join( + [self._output_sep.join([str(i) for i in output]) for output in outputs]) table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): diff --git a/easy_rec/python/inference/predictor.py b/easy_rec/python/inference/predictor.py index 24aafc02b..618789266 100644 --- a/easy_rec/python/inference/predictor.py +++ b/easy_rec/python/inference/predictor.py @@ -1,6 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import abc import json @@ -14,18 +16,18 @@ import tensorflow as tf from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.python.platform import gfile -from tensorflow.python.saved_model import constants, signature_constants +from tensorflow.python.saved_model import constants +from tensorflow.python.saved_model import signature_constants import easy_rec from easy_rec.python.utils import numpy_utils -from easy_rec.python.utils.config_util import ( # NOQA - get_configs_from_pipeline_file, - get_input_name_from_fg_json, - search_fg_json, -) from easy_rec.python.utils.input_utils import get_type_defaults from easy_rec.python.utils.load_class import get_register_class_meta +from easy_rec.python.utils.config_util import ( # NOQA + get_configs_from_pipeline_file, get_input_name_from_fg_json, search_fg_json, +) + try: tf.load_op_library(os.path.join(easy_rec.ops_dir, 'libcustom_ops.so')) except Exception as ex: @@ -37,7 +39,8 @@ SINGLE_PLACEHOLDER_FEATURE_KEY = 'features' _PREDICTOR_CLASS_MAP = {} -_register_abc_meta = get_register_class_meta(_PREDICTOR_CLASS_MAP, have_abstract_class=True) +_register_abc_meta = get_register_class_meta( + _PREDICTOR_CLASS_MAP, have_abstract_class=True) class PredictorInterface(six.with_metaclass(_register_abc_meta, object)): @@ -96,6 +99,7 @@ def get_output_type(self): class PredictorImpl(object): + def __init__(self, model_path, profiling_file=None, use_latest=False): """Impl class for predictor. @@ -152,12 +156,13 @@ def search_pb(self, directory): if self._use_latest: logging.info('find %d models: %s' % (len(dir_list), ','.join(dir_list))) dir_list = sorted( - dir_list, - key=lambda x: int(x.split('/')[(-2 if (x[-1] == '/') else -1)]), + dir_list, + key=lambda x: int(x.split('/')[(-2 if (x[-1] == '/') else -1)]), ) return dir_list[-1] else: - raise ValueError('multiple saved model found in directory %s' % directory) + raise ValueError('multiple saved model found in directory %s' % + directory) return dir_list[0] @@ -165,13 +170,15 @@ def _get_input_fields_from_pipeline_config(self, model_path): pipeline_path = os.path.join(model_path, 'assets/pipeline.config') if not gfile.Exists(pipeline_path): logging.warning( - '%s not exists, default values maybe inconsistent with the values used in training.' % pipeline_path - ) + '%s not exists, default values maybe inconsistent with the values used in training.' + % pipeline_path) return {} pipeline_config = get_configs_from_pipeline_file(pipeline_path) input_fields = pipeline_config.data_config.input_fields input_fields_info = { - input_field.input_name: (input_field.input_type, input_field.default_val) for input_field in input_fields + input_field.input_name: + (input_field.input_type, input_field.default_val) + for input_field in input_fields } input_fields_list = [input_field.input_name for input_field in input_fields] @@ -183,9 +190,9 @@ def _build_model(self): self._graph = tf.Graph() gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=(self._profiling_file is not None), + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=(self._profiling_file is not None), ) self._session = tf.Session(config=session_config, graph=self._graph) @@ -198,20 +205,20 @@ def _build_model(self): model_path = self.search_pb(model_path) logging.info('model find in %s' % model_path) ( - self._input_fields_info, - self._input_fields_list, + self._input_fields_info, + self._input_fields_list, ) = self._get_input_fields_from_pipeline_config(model_path) - assert tf.saved_model.loader.maybe_saved_model_directory(model_path), ( - 'saved model does not exists in %s' % model_path - ) + assert tf.saved_model.loader.maybe_saved_model_directory( + model_path), ('saved model does not exists in %s' % model_path) self._is_saved_model = True meta_graph_def = tf.saved_model.loader.load( - self._session, - [tf.saved_model.tag_constants.SERVING], - model_path, + self._session, + [tf.saved_model.tag_constants.SERVING], + model_path, ) # parse signature - signature_def = meta_graph_def.signature_def[signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] + signature_def = meta_graph_def.signature_def[ + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] inputs = signature_def.inputs # each input_info is a tuple of input_id, name, data_type input_info = [] @@ -230,16 +237,20 @@ def _build_model(self): # in which case, the order of inputs may not be the # same as they are defined, thereforce, list input # could not be supported, only dict input could be supported - logging.warning('could not determine input_id from input_name: %s' % input_name) + logging.warning( + 'could not determine input_id from input_name: %s' % + input_name) input_id = gid input_info.append((input_id, name, tensor.dtype)) - self._inputs_map[name] = self._graph.get_tensor_by_name(tensor.name) + self._inputs_map[name] = self._graph.get_tensor_by_name( + tensor.name) else: # only one input, all features concatenate together for name, tensor in inputs.items(): logging.info('Load input binding: %s -> %s' % (name, tensor.name)) input_info.append((0, name, tensor.dtype)) - self._inputs_map[name] = self._graph.get_tensor_by_name(tensor.name) + self._inputs_map[name] = self._graph.get_tensor_by_name( + tensor.name) # sort inputs by input_ids so as to match the order of csv data input_info.sort(key=lambda t: t[0]) self._input_names = [t[1] for t in input_info] @@ -247,7 +258,8 @@ def _build_model(self): outputs = signature_def.outputs for name, tensor in outputs.items(): logging.info('Load output binding: %s -> %s' % (name, tensor.name)) - self._outputs_map[name] = self._graph.get_tensor_by_name(tensor.name) + self._outputs_map[name] = self._graph.get_tensor_by_name( + tensor.name) # get assets self._assets = {} @@ -256,7 +268,8 @@ def _build_model(self): asset_file = meta_graph_pb2.AssetFileDef() any_proto.Unpack(asset_file) type_name = asset_file.tensor_info.name.split(':')[0] - asset_path = os.path.join(model_path, constants.ASSETS_DIRECTORY, asset_file.filename) + asset_path = os.path.join(model_path, constants.ASSETS_DIRECTORY, + asset_file.filename) # assert gfile.Exists( # asset_path), '%s is missing in saved model' % asset_path if gfile.Exists(asset_path): @@ -290,13 +303,12 @@ def predict(self, input_data_dict, output_names=None): tensor_shape = tensor.get_shape().as_list() input_shape = input_data_dict[input_name].shape assert tensor_shape[0] is None or (tensor_shape[0] == input_shape[0]), ( - 'input %s batchsize %d is not the same as the exported batch_size %d' - % ( - input_name, - input_shape[0], - tensor_shape[0], - ) - ) + 'input %s batchsize %d is not the same as the exported batch_size %d' + % ( + input_name, + input_shape[0], + tensor_shape[0], + )) feed_dict[tensor] = input_data_dict[input_name] fetch_dict = {} if output_names is not None: @@ -314,10 +326,10 @@ def predict(self, input_data_dict, output_names=None): run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() results = self._session.run( - fetch_dict, - feed_dict, - options=run_options, - run_metadata=run_metadata, + fetch_dict, + feed_dict, + options=run_options, + run_metadata=run_metadata, ) # Create the Timeline object, and write it to a json from tensorflow.python.client import timeline @@ -330,7 +342,12 @@ def predict(self, input_data_dict, output_names=None): class Predictor(PredictorInterface): - def __init__(self, model_path, profiling_file=None, fg_json_path=None, use_latest=True): + + def __init__(self, + model_path, + profiling_file=None, + fg_json_path=None, + use_latest=True): """Initialize a `Predictor`. Args: @@ -380,20 +397,20 @@ def _get_defaults(self, col_name, col_type='string'): else: defaults = {'string': '', 'double': 0.0, 'bigint': 0} assert col_type in defaults, 'invalid col_type: %s, col_type: %s' % ( - col_name, - col_type, + col_name, + col_type, ) default_val = defaults[col_type] logging.info( - 'col_name: %s, default_val: %s.[not defined in saved_model_dir/assets/pipeline.config]' - % (col_name, default_val) - ) + 'col_name: %s, default_val: %s.[not defined in saved_model_dir/assets/pipeline.config]' + % (col_name, default_val)) return default_val def _parse_line(self, line): pass - def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, slice_id): + def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, + slice_id): pass def _get_writer(self, output_path, slice_id): @@ -430,14 +447,14 @@ def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): pass def predict_impl( - self, - input_path, - output_path, - reserved_cols='', - output_cols=None, - batch_size=1024, - slice_id=0, - slice_num=1, + self, + input_path, + output_path, + reserved_cols='', + output_cols=None, + batch_size=1024, + slice_id=0, + slice_num=1, ): """Predict table input with loaded model. @@ -468,8 +485,10 @@ def predict_impl( with tf.Graph().as_default(), tf.Session() as sess: num_parallel_calls = 8 self._reserved_args = reserved_cols - dataset = self._get_dataset(input_path, num_parallel_calls, batch_size, slice_num, slice_id) - dataset = dataset.map(self._parse_line, num_parallel_calls=num_parallel_calls) + dataset = self._get_dataset(input_path, num_parallel_calls, batch_size, + slice_num, slice_id) + dataset = dataset.map( + self._parse_line, num_parallel_calls=num_parallel_calls) if hasattr(tf.data, 'make_one_shot_iterator'): iterator = tf.data.make_one_shot_iterator(dataset) else: @@ -493,10 +512,9 @@ def _parse_value(all_vals): else: assert self._all_input_names, 'must set fg_json_path when use fg input' assert fg_input_size == len(self._all_input_names), ( - 'The number of features defined in fg_json != the size of fg input. ' - 'The number of features defined in fg_json is: %d; The size of fg input is: %d' - % (len(self._all_input_names), fg_input_size) - ) + 'The number of features defined in fg_json != the size of fg input. ' + 'The number of features defined in fg_json is: %d; The size of fg input is: %d' + % (len(self._all_input_names), fg_input_size)) for i, k in enumerate(self._all_input_names): split_index.append(k) split_vals[k] = [] @@ -525,13 +543,20 @@ def _parse_value(all_vals): # automatic flatten only one element array outputs[x] = [val[0] for val in outputs[x]] elif len(outputs[x].shape) > 1: - outputs[x] = [json.dumps(val, cls=numpy_utils.NumpyEncoder) for val in outputs[x]] + outputs[x] = [ + json.dumps(val, cls=numpy_utils.NumpyEncoder) + for val in outputs[x] + ] for k in self._reserved_cols: if k in all_vals and all_vals[k].dtype == np.object: - all_vals[k] = [val.decode('utf-8', errors='ignore') for val in all_vals[k]] + all_vals[k] = [ + val.decode('utf-8', errors='ignore') for val in all_vals[k] + ] ts2 = time.time() - reserve_vals = self._get_reserve_vals(self._reserved_cols, self._output_cols, all_vals, outputs) + reserve_vals = self._get_reserve_vals(self._reserved_cols, + self._output_cols, all_vals, + outputs) outputs = [x for x in zip(*reserve_vals)] logging.info('predict size: %s' % len(outputs)) self._write_lines(table_writer, outputs) @@ -544,9 +569,12 @@ def _parse_value(all_vals): except self.out_of_range_exception: break if progress % 100 == 0: - logging.info('progress: batch_num=%d sample_num=%d' % (progress, progress * batch_size)) - logging.info('time_stats: read: %.2f predict: %.2f write: %.2f' % (sum_t0, sum_t1, sum_t2)) - logging.info('Final_time_stats: read: %.2f predict: %.2f write: %.2f' % (sum_t0, sum_t1, sum_t2)) + logging.info('progress: batch_num=%d sample_num=%d' % + (progress, progress * batch_size)) + logging.info('time_stats: read: %.2f predict: %.2f write: %.2f' % + (sum_t0, sum_t1, sum_t2)) + logging.info('Final_time_stats: read: %.2f predict: %.2f write: %.2f' % + (sum_t0, sum_t1, sum_t2)) table_writer.close() self.load_to_table(output_path, slice_num, slice_id) logging.info('Predict %s done.' % input_path) @@ -564,11 +592,10 @@ def predict(self, input_data_dict_list, output_names=None, batch_size=1): """ num_example = len(input_data_dict_list) assert num_example > 0, 'input data should not be an empty list' - assert ( - isinstance(input_data_dict_list[0], dict) - or isinstance(input_data_dict_list[0], list) - or isinstance(input_data_dict_list[0], str) - ), 'input is not a list or dict or str' + assert (isinstance(input_data_dict_list[0], dict) or + isinstance(input_data_dict_list[0], list) or + isinstance(input_data_dict_list[0], + str)), 'input is not a list or dict or str' if batch_size > 0: num_batches = int(math.ceil(float(num_example) / batch_size)) else: @@ -577,7 +604,9 @@ def predict(self, input_data_dict_list, output_names=None, batch_size=1): outputs_list = [] for batch_idx in range(num_batches): - batch_data_list = input_data_dict_list[batch_idx * batch_size : (batch_idx + 1) * batch_size] + batch_data_list = input_data_dict_list[batch_idx * + batch_size:(batch_idx + 1) * + batch_size] feed_dict = self.batch(batch_data_list) outputs = self._predictor_impl.predict(feed_dict, output_names) for idx in range(len(batch_data_list)): @@ -596,12 +625,10 @@ def batch(self, data_list): batch_input[key].append(data[key]) elif isinstance(data, list): assert len(self._predictor_impl.input_names) == len(data), ( - 'input fields number incorrect, should be %d, but %d' - % ( - len(self._predictor_impl.input_names), - len(data), - ) - ) + 'input fields number incorrect, should be %d, but %d' % ( + len(self._predictor_impl.input_names), + len(data), + )) for key, v in zip(self._predictor_impl.input_names, data): if key != '': batch_input[key].append(v) diff --git a/easy_rec/python/inference/processor/test.py b/easy_rec/python/inference/processor/test.py index 413719560..df24edbdd 100644 --- a/easy_rec/python/inference/processor/test.py +++ b/easy_rec/python/inference/processor/test.py @@ -12,9 +12,12 @@ import numpy as np from google.protobuf import text_format -from easy_rec.python.protos import dataset_pb2, pipeline_pb2, tf_predict_pb2 +from easy_rec.python.protos import dataset_pb2 +from easy_rec.python.protos import pipeline_pb2 +from easy_rec.python.protos import tf_predict_pb2 -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') PROCESSOR_VERSION = 'LaRec-0.9.5d-b1b1604-TF-2.5.0-Linux' PROCESSOR_FILE = PROCESSOR_VERSION + '.tar.gz' @@ -47,37 +50,53 @@ def build_array_proto(array_proto, data, dtype): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--input_path', type=str, default=None, help='input data path') - parser.add_argument('--output_path', type=str, default=None, help='output data path') - parser.add_argument('--libc_path', type=str, default='/lib64/libc.so.6', help='libc.so.6 path') - parser.add_argument('--saved_model_dir', type=str, default=None, help='saved model directory') - parser.add_argument('--test_dir', type=str, default=None, help='test directory') + parser.add_argument( + '--input_path', type=str, default=None, help='input data path') + parser.add_argument( + '--output_path', type=str, default=None, help='output data path') + parser.add_argument( + '--libc_path', + type=str, + default='/lib64/libc.so.6', + help='libc.so.6 path') + parser.add_argument( + '--saved_model_dir', type=str, default=None, help='saved model directory') + parser.add_argument( + '--test_dir', type=str, default=None, help='test directory') args = parser.parse_args() if not os.path.exists('processor'): os.mkdir('processor') if not os.path.exists(PROCESSOR_ENTRY_LIB): if not os.path.exists('processor/' + PROCESSOR_FILE): - subprocess.check_output('wget %s -O processor/%s' % (PROCESSOR_URL, PROCESSOR_FILE), shell=True) - subprocess.check_output('cd processor && tar -zvxf %s' % PROCESSOR_FILE, shell=True) - assert os.path.exists(PROCESSOR_ENTRY_LIB), 'invalid processor path: %s' % PROCESSOR_ENTRY_LIB + subprocess.check_output( + 'wget %s -O processor/%s' % (PROCESSOR_URL, PROCESSOR_FILE), + shell=True) + subprocess.check_output( + 'cd processor && tar -zvxf %s' % PROCESSOR_FILE, shell=True) + assert os.path.exists( + PROCESSOR_ENTRY_LIB), 'invalid processor path: %s' % PROCESSOR_ENTRY_LIB assert os.path.exists(args.libc_path), '%s does not exist' % args.libc_path - assert args.saved_model_dir is not None and os.path.isdir(args.saved_model_dir), ( - '%s is not a valid directory' % args.saved_model_dir - ) - assert args.input_path is not None and os.path.exists(args.input_path), '%s does not exist' % args.input_path + assert args.saved_model_dir is not None and os.path.isdir( + args.saved_model_dir), ('%s is not a valid directory' % + args.saved_model_dir) + assert args.input_path is not None and os.path.exists( + args.input_path), '%s does not exist' % args.input_path assert args.output_path is not None, 'output_path is not set' pipeline_config = pipeline_pb2.EasyRecConfig() - pipeline_config_path = os.path.join(args.saved_model_dir, 'assets/pipeline.config') + pipeline_config_path = os.path.join(args.saved_model_dir, + 'assets/pipeline.config') with open(pipeline_config_path) as fin: config_str = fin.read() text_format.Merge(config_str, pipeline_config) data_config = pipeline_config.data_config - input_fields = [[] for x in data_config.input_fields if x.input_name not in data_config.label_fields] + input_fields = [[] + for x in data_config.input_fields + if x.input_name not in data_config.label_fields] with open(args.input_path, 'r') as fin: for line_str in fin: @@ -90,9 +109,9 @@ def build_array_proto(array_proto, data, dtype): req.signature_name = 'serving_default' for i in range(len(input_fields)): build_array_proto( - req.inputs[data_config.input_fields[i + 1].input_name], - input_fields[i], - data_config.input_fields[i + 1].input_type, + req.inputs[data_config.input_fields[i + 1].input_name], + input_fields[i], + data_config.input_fields[i + 1].input_type, ) tf_predictor = ctypes.cdll.LoadLibrary(PROCESSOR_ENTRY_LIB) @@ -105,7 +124,8 @@ def build_array_proto(array_proto, data, dtype): # last_step could be greater than num_steps for sync_replicas: false train_dir = os.path.dirname(args.saved_model_dir.strip('/')) - all_models = glob.glob(os.path.join(args.test_dir, 'train/model.ckpt-*.index')) + all_models = glob.glob( + os.path.join(args.test_dir, 'train/model.ckpt-*.index')) iters = [int(x.split('-')[-1].replace('.index', '')) for x in all_models] iters.sort() last_step = iters[-1] @@ -115,10 +135,14 @@ def build_array_proto(array_proto, data, dtype): dense_step = ctypes.c_int(0) start_ts = time.time() while sparse_step.value < last_step or dense_step.value < last_step: - tf_predictor.saved_model_step(ctypes.c_void_p(handle), ctypes.byref(sparse_step), ctypes.byref(dense_step)) + tf_predictor.saved_model_step( + ctypes.c_void_p(handle), ctypes.byref(sparse_step), + ctypes.byref(dense_step)) time.sleep(1) if time.time() - start_ts > 300: - logging.warning('could not reach last_step, sparse_step=%d dense_step=%d' % (sparse_step.value, dense_step.value)) + logging.warning( + 'could not reach last_step, sparse_step=%d dense_step=%d' % + (sparse_step.value, dense_step.value)) break data_bin = req.SerializeToString() @@ -130,10 +154,10 @@ def build_array_proto(array_proto, data, dtype): tf_predictor.saved_model_predict.restype = ctypes.c_void_p out_len = ctypes.c_int(0) res_p = tf_predictor.saved_model_predict( - ctypes.c_void_p(handle), - data_bin, - ctypes.c_int32(len(data_bin)), - ctypes.byref(out_len), + ctypes.c_void_p(handle), + data_bin, + ctypes.c_int32(len(data_bin)), + ctypes.byref(out_len), ) res_bytes = bytearray(ctypes.string_at(res_p, out_len)) res = tf_predict_pb2.PredictResponse() diff --git a/easy_rec/python/inference/vector_retrieve.py b/easy_rec/python/inference/vector_retrieve.py index 19b50f87e..37cf220bf 100644 --- a/easy_rec/python/inference/vector_retrieve.py +++ b/easy_rec/python/inference/vector_retrieve.py @@ -1,6 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import logging from datetime import datetime @@ -13,7 +15,7 @@ import graphlearn as gl except: # noqa: E722 logging.warning( - 'GraphLearn is not installed. You can install it by "pip install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-0.7-cp27-cp27mu-linux_x86_64.whl.' # noqa: E501 + 'GraphLearn is not installed. You can install it by "pip install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-0.7-cp27-cp27mu-linux_x86_64.whl.' # noqa: E501 ) if tf.__version__ >= '2.0': @@ -21,19 +23,20 @@ class VectorRetrieve(object): + def __init__( - self, - query_table, - doc_table, - out_table, - ndim, - delimiter=',', - batch_size=4, - index_type='ivfflat', - nlist=10, - nprobe=2, - distance=1, - m=8, + self, + query_table, + doc_table, + out_table, + ndim, + delimiter=',', + batch_size=4, + index_type='ivfflat', + nlist=10, + nprobe=2, + distance=1, + m=8, ): """Retrieve top n neighbours by query vector. @@ -70,14 +73,16 @@ def __init__( def __call__(self, top_n, task_index, task_count, *args, **kwargs): g = gl.Graph() g.node( - self.doc_table, - 'doc', - decoder=gl.Decoder(attr_types=['float'] * self.ndim, attr_delimiter=self.delimiter), - option=self.knn_option, + self.doc_table, + 'doc', + decoder=gl.Decoder( + attr_types=['float'] * self.ndim, attr_delimiter=self.delimiter), + option=self.knn_option, ) g.init(task_index=task_index, task_count=task_count) - query_reader = common_io.table.TableReader(self.query_table, slice_id=task_index, slice_count=task_count) + query_reader = common_io.table.TableReader( + self.query_table, slice_id=task_index, slice_count=task_count) num_records = query_reader.get_row_count() total_batch_num = num_records // self.batch_size + 1.0 batch_num = 0 @@ -85,19 +90,24 @@ def __call__(self, top_n, task_index, task_count, *args, **kwargs): print('total_batch_num: {}'.format(total_batch_num)) print('output_table: {}'.format(self.out_table)) - output_table_writer = common_io.table.TableWriter(self.out_table, task_index) + output_table_writer = common_io.table.TableWriter(self.out_table, + task_index) count = 0 while True: try: - batch_query_nodes, batch_query_feats = zip(*query_reader.read(self.batch_size, allow_smaller_final_batch=True)) + batch_query_nodes, batch_query_feats = zip( + *query_reader.read(self.batch_size, allow_smaller_final_batch=True)) batch_num += 1.0 - print('{} process: {:.2f}'.format(datetime.now().time(), batch_num / total_batch_num)) + print('{} process: {:.2f}'.format(datetime.now().time(), + batch_num / total_batch_num)) feats = to_np_array(batch_query_feats, self.delimiter) rt_ids, rt_dists = g.search('doc', feats, gl.KnnOption(k=top_n)) - for query_node, nodes, dists in zip(batch_query_nodes, rt_ids, rt_dists): + for query_node, nodes, dists in zip(batch_query_nodes, rt_ids, + rt_dists): query = np.array([query_node] * len(nodes), dtype='int64') - output_table_writer.write(zip(query, nodes, dists), (0, 1, 2), allow_type_cast=False) + output_table_writer.write( + zip(query, nodes, dists), (0, 1, 2), allow_type_cast=False) count += 1 if np.mod(count, 100) == 0: print('write ', count, ' query nodes totally') @@ -113,6 +123,6 @@ def __call__(self, top_n, task_index, task_count, *args, **kwargs): def to_np_array(batch_query_feats, attr_delimiter): return np.array( - [map(float, feat.split(attr_delimiter)) for feat in batch_query_feats], - dtype='float32', + [map(float, feat.split(attr_delimiter)) for feat in batch_query_feats], + dtype='float32', ) diff --git a/easy_rec/python/input/batch_tfrecord_input.py b/easy_rec/python/input/batch_tfrecord_input.py index f739f0329..77af079a8 100644 --- a/easy_rec/python/input/batch_tfrecord_input.py +++ b/easy_rec/python/input/batch_tfrecord_input.py @@ -20,56 +20,56 @@ class BatchTFRecordInput(Input): """ def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(BatchTFRecordInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(BatchTFRecordInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) - assert data_config.HasField('n_data_batch_tfrecord'), 'Need to set n_data_batch_tfrecord in config.' + assert data_config.HasField( + 'n_data_batch_tfrecord'), 'Need to set n_data_batch_tfrecord in config.' self._input_shapes = [x.input_shape for x in data_config.input_fields] self.feature_desc = {} for x, t, d, s in zip( - self._input_fields, - self._input_field_types, - self._input_field_defaults, - self._input_shapes, + self._input_fields, + self._input_field_types, + self._input_field_defaults, + self._input_shapes, ): d = self.get_type_defaults(t, d) t = get_tf_type(t) - self.feature_desc[x] = tf.io.FixedLenSequenceFeature(dtype=t, shape=s, allow_missing=True) + self.feature_desc[x] = tf.io.FixedLenSequenceFeature( + dtype=t, shape=s, allow_missing=True) def _parse_tfrecord(self, example): try: - _, features, _ = tf.parse_sequence_example(example, sequence_features=self.feature_desc) + _, features, _ = tf.parse_sequence_example( + example, sequence_features=self.feature_desc) except AttributeError: - _, features, _ = tf.io.parse_sequence_example(example, sequence_features=self.feature_desc) + _, features, _ = tf.io.parse_sequence_example( + example, sequence_features=self.feature_desc) # Below code will reduce one dimension when the data dimension > 2. - features = dict( - ( + features = dict(( key, tf.reshape( - value, - [ - -1, - ] - + [x for i, x in enumerate(value.shape) if i not in (0, 1)], + value, + [ + -1, + ] + [x for i, x in enumerate(value.shape) if i not in (0, 1)], ), - ) - for (key, value) in features.items() - ) + ) for (key, value) in features.items()) return features def _build(self, mode, params): @@ -83,7 +83,8 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls data_compression_type = self._data_config.data_compression_type if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('train files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.shuffle: # shuffle input files @@ -92,36 +93,42 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TFRecordDataset(x, compression_type=data_compression_type), - cycle_length=parallel_num, - num_parallel_calls=parallel_num, + lambda x: tf.data.TFRecordDataset( + x, compression_type=data_compression_type), + cycle_length=parallel_num, + num_parallel_calls=parallel_num, ) dataset = dataset.shard(self._task_num, self._task_index) if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) - dataset = tf.data.TFRecordDataset(file_paths, compression_type=data_compression_type) + logging.info('eval files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) + dataset = tf.data.TFRecordDataset( + file_paths, compression_type=data_compression_type) dataset = dataset.repeat(1) # We read n data from tfrecord one time. cur_batch = self._data_config.batch_size // self._data_config.n_data_batch_tfrecord cur_batch = max(1, cur_batch) dataset = dataset.batch(cur_batch) - dataset = dataset.map(self._parse_tfrecord, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + self._parse_tfrecord, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/criteo_binary_reader.py b/easy_rec/python/input/criteo_binary_reader.py index 00bd14708..c27b4aae3 100644 --- a/easy_rec/python/input/criteo_binary_reader.py +++ b/easy_rec/python/input/criteo_binary_reader.py @@ -13,16 +13,17 @@ class BinaryDataset: + def __init__( - self, - label_bins, - dense_bins, - category_bins, - batch_size=1, - drop_last=False, - prefetch=1, - global_rank=0, - global_size=1, + self, + label_bins, + dense_bins, + category_bins, + batch_size=1, + drop_last=False, + prefetch=1, + global_rank=0, + global_size=1, ): total_sample_num = 0 self._sample_num_arr = [] @@ -35,24 +36,30 @@ def __init__( self._batch_size = batch_size - self._compute_global_start_pos(total_sample_num, batch_size, global_rank, global_size, drop_last) + self._compute_global_start_pos(total_sample_num, batch_size, global_rank, + global_size, drop_last) self._label_file_arr = [None for _ in self._sample_num_arr] self._dense_file_arr = [None for _ in self._sample_num_arr] self._category_file_arr = [None for _ in self._sample_num_arr] for tmp_file_id in range(self._start_file_id, self._end_file_id + 1): - self._label_file_arr[tmp_file_id] = os.open(label_bins[tmp_file_id], os.O_RDONLY) - self._dense_file_arr[tmp_file_id] = os.open(dense_bins[tmp_file_id], os.O_RDONLY) - self._category_file_arr[tmp_file_id] = os.open(category_bins[tmp_file_id], os.O_RDONLY) + self._label_file_arr[tmp_file_id] = os.open(label_bins[tmp_file_id], + os.O_RDONLY) + self._dense_file_arr[tmp_file_id] = os.open(dense_bins[tmp_file_id], + os.O_RDONLY) + self._category_file_arr[tmp_file_id] = os.open(category_bins[tmp_file_id], + os.O_RDONLY) self._prefetch = min(prefetch, self._num_entries) self._prefetch_queue = queue.Queue() - self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=self._prefetch) + self._executor = concurrent.futures.ThreadPoolExecutor( + max_workers=self._prefetch) self._os_close_func = os.close - def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, global_size, drop_last): + def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, + global_size, drop_last): # ensure all workers have the same number of samples avg_sample_num = total_sample_num // global_size res_num = total_sample_num % global_size @@ -72,7 +79,8 @@ def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, g if not drop_last and (self._num_samples % batch_size != 0): self._num_entries += 1 self._last_batch_size = self._num_samples % batch_size - logging.info('num_batches = %d num_samples = %d' % (self._num_entries, self._num_samples)) + logging.info('num_batches = %d num_samples = %d' % + (self._num_entries, self._num_samples)) start_file_id = 0 curr_pos = 0 @@ -82,7 +90,8 @@ def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, g self._start_file_id = start_file_id self._start_file_pos = global_start_pos - curr_pos - logging.info('start_file_id = %d start_file_pos = %d' % (start_file_id, self._start_file_pos)) + logging.info('start_file_id = %d start_file_pos = %d' % + (start_file_id, self._start_file_pos)) # find the start of each batch self._start_pos_arr = np.zeros([self._num_entries, 2], dtype=np.uint32) @@ -94,19 +103,24 @@ def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, g # the last batch if batch_id == self._num_entries: tmp_start_pos += self._last_batch_size - while start_file_id < len(self._sample_num_arr) and tmp_start_pos > self._sample_num_arr[start_file_id]: + while start_file_id < len( + self._sample_num_arr + ) and tmp_start_pos > self._sample_num_arr[start_file_id]: tmp_start_pos -= self._sample_num_arr[start_file_id] start_file_id += 1 else: tmp_start_pos += batch_size - while start_file_id < len(self._sample_num_arr) and tmp_start_pos >= self._sample_num_arr[start_file_id]: + while start_file_id < len( + self._sample_num_arr + ) and tmp_start_pos >= self._sample_num_arr[start_file_id]: tmp_start_pos -= self._sample_num_arr[start_file_id] start_file_id += 1 self._end_file_id = start_file_id self._end_file_pos = tmp_start_pos - logging.info('end_file_id = %d end_file_pos = %d' % (self._end_file_id, self._end_file_pos)) + logging.info('end_file_id = %d end_file_pos = %d' % + (self._end_file_id, self._end_file_pos)) def __del__(self): for f in self._label_file_arr: @@ -134,7 +148,8 @@ def __getitem__(self, idx): self._prefetch_queue.put(self._executor.submit(self._get, (i))) if idx < (self._num_entries - self._prefetch): - self._prefetch_queue.put(self._executor.submit(self._get, (idx + self._prefetch))) + self._prefetch_queue.put( + self._executor.submit(self._get, (idx + self._prefetch))) return self._prefetch_queue.get().result() @@ -148,28 +163,34 @@ def _get(self, idx): label_read_arr = [] dense_read_arr = [] cate_read_arr = [] - while total_read_num < self._batch_size and curr_file_id < len(self._sample_num_arr): - tmp_read_num = min(end_read_pos, self._sample_num_arr[curr_file_id]) - start_read_pos - - label_raw_data = os.pread(self._label_file_arr[curr_file_id], 4 * tmp_read_num, start_read_pos * 4) - tmp_lbl_np = np.frombuffer(label_raw_data, dtype=np.int32).reshape([tmp_read_num, 1]) + while total_read_num < self._batch_size and curr_file_id < len( + self._sample_num_arr): + tmp_read_num = min(end_read_pos, + self._sample_num_arr[curr_file_id]) - start_read_pos + + label_raw_data = os.pread(self._label_file_arr[curr_file_id], + 4 * tmp_read_num, start_read_pos * 4) + tmp_lbl_np = np.frombuffer( + label_raw_data, dtype=np.int32).reshape([tmp_read_num, 1]) label_read_arr.append(tmp_lbl_np) dense_raw_data = os.pread( - self._dense_file_arr[curr_file_id], - 52 * tmp_read_num, - start_read_pos * 52, + self._dense_file_arr[curr_file_id], + 52 * tmp_read_num, + start_read_pos * 52, ) - part_dense_np = np.frombuffer(dense_raw_data, dtype=np.float32).reshape([tmp_read_num, 13]) + part_dense_np = np.frombuffer( + dense_raw_data, dtype=np.float32).reshape([tmp_read_num, 13]) # part_dense_np = np.log(part_dense_np + 3, dtype=np.float32) dense_read_arr.append(part_dense_np) category_raw_data = os.pread( - self._category_file_arr[curr_file_id], - 104 * tmp_read_num, - start_read_pos * 104, + self._category_file_arr[curr_file_id], + 104 * tmp_read_num, + start_read_pos * 104, ) - part_cate_np = np.frombuffer(category_raw_data, dtype=np.uint32).reshape([tmp_read_num, 26]) + part_cate_np = np.frombuffer( + category_raw_data, dtype=np.uint32).reshape([tmp_read_num, 26]) cate_read_arr.append(part_cate_np) curr_file_id += 1 @@ -197,10 +218,12 @@ def _get(self, idx): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--batch_size', type=int, default=1024, help='batch_size') - parser.add_argument('--dataset_dir', type=str, default='./', help='dataset_dir') + parser.add_argument( + '--dataset_dir', type=str, default='./', help='dataset_dir') parser.add_argument('--task_num', type=int, default=1, help='task number') parser.add_argument('--task_index', type=int, default=0, help='task index') - parser.add_argument('--prefetch_size', type=int, default=10, help='prefetch size') + parser.add_argument( + '--prefetch_size', type=int, default=10, help='prefetch size') args = parser.parse_args() batch_size = args.batch_size @@ -217,14 +240,14 @@ def _get(self, idx): category_files.sort() test_dataset = BinaryDataset( - label_files, - dense_files, - category_files, - batch_size=batch_size, - drop_last=False, - prefetch=args.prefetch_size, - global_rank=args.task_index, - global_size=args.task_num, + label_files, + dense_files, + category_files, + batch_size=batch_size, + drop_last=False, + prefetch=args.prefetch_size, + global_rank=args.task_index, + global_size=args.task_num, ) for step, (dense, category, labels) in enumerate(test_dataset): @@ -235,8 +258,8 @@ def _get(self, idx): start_time = time.time() if step == 1000: logging.info('1000 steps time = %.3f' % (time.time() - start_time)) - logging.info('total_steps = %d total_time = %.3f' % (step + 1, time.time() - start_time)) + logging.info('total_steps = %d total_time = %.3f' % + (step + 1, time.time() - start_time)) logging.info( - 'final step[%d] dense_shape=%s category_shape=%s labels_shape=%s' - % (step, dense.shape, category.shape, labels.shape) - ) + 'final step[%d] dense_shape=%s category_shape=%s labels_shape=%s' % + (step, dense.shape, category.shape, labels.shape)) diff --git a/easy_rec/python/input/criteo_input.py b/easy_rec/python/input/criteo_input.py index 0fe0d758e..00412a4c9 100644 --- a/easy_rec/python/input/criteo_input.py +++ b/easy_rec/python/input/criteo_input.py @@ -12,48 +12,52 @@ class CriteoInput(Input): + def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(CriteoInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(CriteoInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) all_label_paths = [] all_dense_paths = [] all_category_paths = [] if input_path is not None: - assert len(input_path.label_path) == len(input_path.dense_path) and len(input_path.label_path) == len( - input_path.category_path + assert len(input_path.label_path) == len(input_path.dense_path) and len( + input_path.label_path + ) == len( + input_path.category_path ), 'label_path_num(%d), dense_path_num(%d), category_path_num(%d) must be the same' % ( - len(input_path.label_path), - len(input_path.dense_path), - len(input_path.category_path), + len(input_path.label_path), + len(input_path.dense_path), + len(input_path.category_path), ) for label_path, dense_path, category_path in zip( - input_path.label_path, input_path.dense_path, input_path.category_path - ): + input_path.label_path, input_path.dense_path, + input_path.category_path): label_paths = tf.gfile.Glob(input_path.label_path) dense_paths = tf.gfile.Glob(input_path.dense_path) category_paths = tf.gfile.Glob(input_path.category_path) - assert len(label_paths) == len(dense_paths) and len(label_paths) == len(category_paths), ( - 'label_path(%s) dense_path(%s) category_path(%s) ' - + 'matched different number of files(%d %d %d)' % (len(label_paths), len(dense_paths), len(category_paths)) - ) + assert len(label_paths) == len(dense_paths) and len(label_paths) == len( + category_paths), ( + 'label_path(%s) dense_path(%s) category_path(%s) ' + + 'matched different number of files(%d %d %d)' % + (len(label_paths), len(dense_paths), len(category_paths))) label_paths.sort() dense_paths.sort() category_paths.sort() @@ -64,13 +68,13 @@ def __init__( logging.info('total number of input parts: %s' % len(all_label_paths)) self._binary_reader = BinaryDataset( - all_label_paths, - all_dense_paths, - all_category_paths, - self._batch_size, - prefetch=self._prefetch_size, - global_rank=self._task_index, - global_size=self._task_num, + all_label_paths, + all_dense_paths, + all_category_paths, + self._batch_size, + prefetch=self._prefetch_size, + global_rank=self._task_index, + global_size=self._task_num, ) else: self._binary_reader = None @@ -98,22 +102,25 @@ def _to_fea_dict(self, dense, category, labels): def _build(self, mode, params): dataset = tf.data.Dataset.from_generator( - self._sample_generator, - output_types=(tf.float32, tf.int32, tf.int32), - output_shapes=( - tf.TensorShape([None, 13]), - tf.TensorShape([None, 26]), - tf.TensorShape([None]), - ), + self._sample_generator, + output_types=(tf.float32, tf.int32, tf.int32), + output_shapes=( + tf.TensorShape([None, 13]), + tf.TensorShape([None, 26]), + tf.TensorShape([None]), + ), ) num_parallel_calls = self._data_config.num_parallel_calls - dataset = dataset.map(self._to_fea_dict, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + self._to_fea_dict, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/csv_input.py b/easy_rec/python/input/csv_input.py index 08397e0ce..d4e974f49 100644 --- a/easy_rec/python/input/csv_input.py +++ b/easy_rec/python/input/csv_input.py @@ -15,31 +15,33 @@ class CSVInput(Input): + def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(CSVInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(CSVInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) self._with_header = data_config.with_header self._field_names = None def _parse_csv(self, line): record_defaults = [ - self.get_type_defaults(t, v) for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) + for t, v in zip(self._input_field_types, self._input_field_defaults) ] if self._field_names: @@ -49,39 +51,36 @@ def _parse_csv(self, line): if field_name in self._input_fields: tid = self._input_fields.index(field_name) record_defaults.append( - self.get_type_defaults( - self._input_field_types[tid], - self._input_field_defaults[tid], - ) - ) + self.get_type_defaults( + self._input_field_types[tid], + self._input_field_defaults[tid], + )) else: record_defaults.append('') - check_list = ( - [ + check_list = ([ tf.py_func( - check_split, - [ - line, - self._data_config.separator, - len(record_defaults), - self._check_mode, - ], - Tout=tf.bool, + check_split, + [ + line, + self._data_config.separator, + len(record_defaults), + self._check_mode, + ], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): fields = tf.decode_csv( - line, - field_delim=self._data_config.separator, - record_defaults=record_defaults, - name='decode_csv', + line, + field_delim=self._data_config.separator, + record_defaults=record_defaults, + name='decode_csv', ) if self._field_names is not None: - fields = [fields[self._field_names.index(x)] for x in self._input_fields] + fields = [ + fields[self._field_names.index(x)] for x in self._input_fields + ] # filter only valid fields inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids} @@ -101,7 +100,8 @@ def _build(self, mode, params): file_paths.append(x) assert len(file_paths) > 0, 'match no files with %s' % self._input_path - assert not file_paths[0].endswith('.tar.gz'), 'could only support .csv or .gz(not .tar.gz) files.' + assert not file_paths[0].endswith( + '.tar.gz'), 'could only support .csv or .gz(not .tar.gz) files.' compression_type = 'GZIP' if file_paths[0].endswith('.gz') else '' if compression_type: @@ -117,7 +117,8 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('train files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -131,9 +132,11 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset(x, compression_type=compression_type).skip(int(self._with_header)), - cycle_length=parallel_num, - num_parallel_calls=parallel_num, + lambda x: tf.data.TextLineDataset( + x, compression_type=compression_type).skip( + int(self._with_header)), + cycle_length=parallel_num, + num_parallel_calls=parallel_num, ) if not self._data_config.file_shard: @@ -141,43 +144,51 @@ def _build(self, mode, params): if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) elif self._task_num > 1: # For distribute evaluate dataset = tf.data.Dataset.from_tensor_slices(file_paths) parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset(x, compression_type=compression_type).skip(int(self._with_header)), - cycle_length=parallel_num, - num_parallel_calls=parallel_num, + lambda x: tf.data.TextLineDataset( + x, compression_type=compression_type).skip( + int(self._with_header)), + cycle_length=parallel_num, + num_parallel_calls=parallel_num, ) dataset = self._safe_shard(dataset) dataset = dataset.repeat(1) else: - logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('eval files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset(x, compression_type=compression_type).skip(int(self._with_header)), - cycle_length=parallel_num, - num_parallel_calls=parallel_num, + lambda x: tf.data.TextLineDataset( + x, compression_type=compression_type).skip( + int(self._with_header)), + cycle_length=parallel_num, + num_parallel_calls=parallel_num, ) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map(self._parse_csv, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + self._parse_csv, num_parallel_calls=num_parallel_calls) if self._data_config.ignore_error: dataset = dataset.apply(ignore_errors) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/csv_input_ex.py b/easy_rec/python/input/csv_input_ex.py index 88ac356e4..4235ac09e 100644 --- a/easy_rec/python/input/csv_input_ex.py +++ b/easy_rec/python/input/csv_input_ex.py @@ -11,29 +11,31 @@ class CSVInputEx(CSVInput): + def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(CSVInputEx, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(CSVInputEx, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) def _parse_csv(self, line): record_defaults = [ - self.get_type_defaults(t, v) for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) + for t, v in zip(self._input_field_types, self._input_field_defaults) ] def _check_data(line): @@ -41,34 +43,43 @@ def _check_data(line): if type(sep) != type(str): sep = sep.encode('utf-8') field_num = len(line[0].split(sep)) - assert field_num == len(record_defaults), 'sep[%s] maybe invalid: field_num=%d, required_num=%d' % ( - sep, - field_num, - len(record_defaults), + assert field_num == len( + record_defaults + ), 'sep[%s] maybe invalid: field_num=%d, required_num=%d' % ( + sep, + field_num, + len(record_defaults), ) return True - fields = str_split_by_chr(line, self._data_config.separator, skip_empty=False) + fields = str_split_by_chr( + line, self._data_config.separator, skip_empty=False) tmp_fields = tf.reshape(fields.values, [-1, len(record_defaults)]) fields = [] for i in range(len(record_defaults)): if type(record_defaults[i]) == int: - fields.append(tf.string_to_number(tmp_fields[:, i], tf.int64, name='field_as_int_%d' % i)) + fields.append( + tf.string_to_number( + tmp_fields[:, i], tf.int64, name='field_as_int_%d' % i)) elif type(record_defaults[i]) in [float, np.float32, np.float64]: - fields.append(tf.string_to_number(tmp_fields[:, i], tf.float32, name='field_as_flt_%d' % i)) + fields.append( + tf.string_to_number( + tmp_fields[:, i], tf.float32, name='field_as_flt_%d' % i)) elif type(record_defaults[i]) in [str, type(''), bytes]: fields.append(tmp_fields[:, i]) elif type(record_defaults[i]) == bool: fields.append( - tf.logical_or( - tf.equal(tmp_fields[:, i], 'True'), - tf.equal(tmp_fields[:, i], 'true'), - ) - ) + tf.logical_or( + tf.equal(tmp_fields[:, i], 'True'), + tf.equal(tmp_fields[:, i], 'true'), + )) else: assert 'invalid types: %s' % str(type(record_defaults[i])) - keep_ids = [self._input_fields.index(x) for x in self._label_fields + self._effective_fields] + keep_ids = [ + self._input_fields.index(x) + for x in self._label_fields + self._effective_fields + ] inputs = {self._input_fields[x]: fields[x] for x in keep_ids} return inputs diff --git a/easy_rec/python/input/csv_input_v2.py b/easy_rec/python/input/csv_input_v2.py index 5f41cfa21..4495f95de 100644 --- a/easy_rec/python/input/csv_input_v2.py +++ b/easy_rec/python/input/csv_input_v2.py @@ -6,30 +6,32 @@ class CSVInputV2(Input): + def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(CSVInputV2, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(CSVInputV2, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) def _build(self, mode, params): if type(self._input_path) != list: self._input_path = self._input_path.split(',') - assert len(self._input_path) > 0, 'match no files with %s' % self._input_path + assert len( + self._input_path) > 0, 'match no files with %s' % self._input_path if self._input_path[0].startswith('hdfs://'): # support hdfs input @@ -38,24 +40,26 @@ def _build(self, mode, params): num_epochs = self.num_epochs if mode == tf.estimator.ModeKeys.TRAIN else 1 is_train = mode == tf.estimator.ModeKeys.TRAIN record_defaults = [ - self.get_type_defaults(x, v) for x, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(x, v) + for x, v in zip(self._input_field_types, self._input_field_defaults) ] dataset = tf.data.experimental.make_csv_dataset( - self._input_path, - self._data_config.batch_size, - column_names=self._input_fields, - field_delim=self._data_config.separator, - column_defaults=record_defaults, - header=False, - num_epochs=num_epochs, - shuffle=is_train and self._data_config.shuffle, - num_parallel_reads=8, - sloppy=is_train, + self._input_path, + self._data_config.batch_size, + column_names=self._input_fields, + field_delim=self._data_config.separator, + column_defaults=record_defaults, + header=False, + num_epochs=num_epochs, + shuffle=is_train and self._data_config.shuffle, + num_parallel_reads=8, + sloppy=is_train, ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.chief_redundant: - dataset = dataset.shard(max(self._task_num - 1, 1), max(self._task_index - 1, 0)) + dataset = dataset.shard( + max(self._task_num - 1, 1), max(self._task_index - 1, 0)) else: dataset = dataset.shard(self._task_num, self._task_index) else: @@ -66,7 +70,8 @@ def _build(self, mode, params): dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/datahub_input.py b/easy_rec/python/input/datahub_input.py index b36db4f1d..2a19bd92a 100644 --- a/easy_rec/python/input/datahub_input.py +++ b/easy_rec/python/input/datahub_input.py @@ -30,7 +30,9 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) logging.getLogger('datahub.account').setLevel(logging.INFO) except Exception: - logging.warning('DataHub is not installed[%s]. You can install it by: pip install pydatahub' % traceback.format_exc()) + logging.warning( + 'DataHub is not installed[%s]. You can install it by: pip install pydatahub' + % traceback.format_exc()) DataHub = None @@ -38,28 +40,28 @@ class DataHubInput(Input): """DataHubInput is used for online train.""" def __init__( - self, - data_config, - feature_config, - datahub_config, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(DataHubInput, self).__init__( + self, data_config, feature_config, - '', - task_index, - task_num, - check_mode, - pipeline_config, + datahub_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(DataHubInput, self).__init__( + data_config, + feature_config, + '', + task_index, + task_num, + check_mode, + pipeline_config, ) if DataHub is None: logging.error( - 'please install datahub: ', - 'pip install pydatahub ;Python 3.6 recommended', + 'please install datahub: ', + 'pip install pydatahub ;Python 3.6 recommended', ) try: self._num_epoch = 0 @@ -79,10 +81,13 @@ def __init__( logging.info('exception in init datahub: %s' % str(ex)) self._offset_dict = {} if datahub_config: - shard_result = self._datahub.list_shard(self._datahub_config.project, self._datahub_config.topic) + shard_result = self._datahub.list_shard(self._datahub_config.project, + self._datahub_config.topic) shards = shard_result.shards self._all_shards = shards - self._shards = [shards[i] for i in range(len(shards)) if (i % task_num) == task_index] + self._shards = [ + shards[i] for i in range(len(shards)) if (i % task_num) == task_index + ] logging.info('all shards: %s' % str(self._shards)) offset_type = datahub_config.WhichOneof('offset') @@ -91,11 +96,11 @@ def __init__( for x in self._all_shards: ks = str(x.shard_id) cursor_result = self._datahub.get_cursor( - self._datahub_config.project, - self._datahub_config.topic, - ks, - CursorType.SYSTEM_TIME, - ts, + self._datahub_config.project, + self._datahub_config.topic, + ks, + CursorType.SYSTEM_TIME, + ts, ) logging.info('shard[%s] cursor = %s' % (ks, cursor_result)) self._offset_dict[ks] = cursor_result.cursor @@ -107,20 +112,23 @@ def __init__( self._dh_field_names = [] self._dh_field_types = [] topic_info = self._datahub.get_topic( - project_name=self._datahub_config.project, - topic_name=self._datahub_config.topic, + project_name=self._datahub_config.project, + topic_name=self._datahub_config.topic, ) for field in topic_info.record_schema.field_list: self._dh_field_names.append(field.name) self._dh_field_types.append(field.type.value) - assert len(self._feature_fields) > 0, 'data_config.feature_fields are not set.' + assert len( + self._feature_fields) > 0, 'data_config.feature_fields are not set.' for x in self._feature_fields: assert x in self._dh_field_names, 'feature_field[%s] is not in datahub' % x # feature column ids in datahub schema - self._dh_fea_ids = [self._dh_field_names.index(x) for x in self._feature_fields] + self._dh_fea_ids = [ + self._dh_field_names.index(x) for x in self._feature_fields + ] for x in self._label_fields: assert x in self._dh_field_names, 'label_field[%s] is not in datahub' % x @@ -133,18 +141,23 @@ def __init__( if len(self._dh_fea_ids) > 1: self._filter_fea_func = ( - lambda record: ''.join([record.values[x] for x in self._dh_fea_ids]).split(chr(2))[1] == '-1024' - ) + lambda record: ''.join([record.values[x] for x in self._dh_fea_ids] + ).split(chr(2))[1] == '-1024') else: dh_fea_id = self._dh_fea_ids[0] - self._filter_fea_func = lambda record: record.values[dh_fea_id].split(self._data_config.separator)[1] == '-1024' + self._filter_fea_func = lambda record: record.values[dh_fea_id].split( + self._data_config.separator)[1] == '-1024' def _parse_record(self, *fields): field_dict = {} fields = list(fields) def _dump_offsets(): - all_offsets = {x.shard_id: self._offset_dict[x.shard_id] for x in self._shards if x.shard_id in self._offset_dict} + all_offsets = { + x.shard_id: self._offset_dict[x.shard_id] + for x in self._shards + if x.shard_id in self._offset_dict + } return json.dumps(all_offsets) field_dict[Input.DATA_OFFSET] = tf.py_func(_dump_offsets, [], dtypes.string) @@ -155,15 +168,21 @@ def _dump_offsets(): feature_inputs = self.get_feature_input_fields() # only for features, labels and sample_weight excluded - record_types = [t for x, t in zip(self._input_fields, self._input_field_types) if x in feature_inputs] + record_types = [ + t for x, t in zip(self._input_fields, self._input_field_types) + if x in feature_inputs + ] feature_num = len(record_types) - feature_fields = [fields[self._dh_field_names.index(x)] for x in self._feature_fields] + feature_fields = [ + fields[self._dh_field_names.index(x)] for x in self._feature_fields + ] feature = feature_fields[0] for fea_id in range(1, len(feature_fields)): feature = feature + self._data_config.separator + feature_fields[fea_id] - feature = tf.string_split(feature, self._data_config.separator, skip_empty=False) + feature = tf.string_split( + feature, self._data_config.separator, skip_empty=False) fields = tf.reshape(feature.values, [-1, feature_num]) @@ -221,10 +240,13 @@ def _datahub_generator(self): self._num_epoch += 1 try: - self._datahub.wait_shards_ready(self._datahub_config.project, self._datahub_config.topic) - topic_result = self._datahub.get_topic(self._datahub_config.project, self._datahub_config.topic) + self._datahub.wait_shards_ready(self._datahub_config.project, + self._datahub_config.topic) + topic_result = self._datahub.get_topic(self._datahub_config.project, + self._datahub_config.topic) if topic_result.record_type != RecordType.TUPLE: - logging.error('datahub topic type(%s) illegal' % str(topic_result.record_type)) + logging.error('datahub topic type(%s) illegal' % + str(topic_result.record_type)) record_schema = topic_result.record_schema tid = 0 @@ -236,36 +258,39 @@ def _datahub_generator(self): if shard_id not in self._offset_dict: cursor_result = self._datahub.get_cursor( - self._datahub_config.project, - self._datahub_config.topic, - shard_id, - CursorType.OLDEST, + self._datahub_config.project, + self._datahub_config.topic, + shard_id, + CursorType.OLDEST, ) cursor = cursor_result.cursor else: cursor = self._offset_dict[shard_id] get_result = self._datahub.get_tuple_records( - self._datahub_config.project, - self._datahub_config.topic, - shard_id, - record_schema, - cursor, - self._read_cnt, + self._datahub_config.project, + self._datahub_config.topic, + shard_id, + record_schema, + cursor, + self._read_cnt, ) count = get_result.record_count if count == 0: continue for row_id, record in enumerate(get_result.records): if self._is_data_empty(record): - logging.warning('skip empty data record: %s' % self._dump_record(record)) + logging.warning('skip empty data record: %s' % + self._dump_record(record)) continue if self._filter_fea_func is not None: if self._filter_fea_func(record): - logging.warning('filter data record: %s' % self._dump_record(record)) + logging.warning('filter data record: %s' % + self._dump_record(record)) continue yield tuple(list(record.values)) - if shard_id not in self._offset_dict or get_result.next_cursor > self._offset_dict[shard_id]: + if shard_id not in self._offset_dict or get_result.next_cursor > self._offset_dict[ + shard_id]: self._offset_dict[shard_id] = get_result.next_cursor except DatahubException as ex: logging.error('DatahubException: %s' % str(ex)) @@ -277,34 +302,42 @@ def _build(self, mode, params): assert self._datahub is not None, 'datahub_eval_input is not set' # get input types - list_types = [odps_util.odps_type_2_tf_type(x) for x in self._dh_field_types] + list_types = [ + odps_util.odps_type_2_tf_type(x) for x in self._dh_field_types + ] list_types = tuple(list_types) - list_shapes = [tf.TensorShape([]) for x in range(0, len(self._dh_field_types))] + list_shapes = [ + tf.TensorShape([]) for x in range(0, len(self._dh_field_types)) + ] list_shapes = tuple(list_shapes) # read datahub dataset = tf.data.Dataset.from_generator( - self._datahub_generator, output_types=list_types, output_shapes=list_shapes - ) + self._datahub_generator, + output_types=list_types, + output_shapes=list_shapes) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map(self._parse_record, num_parallel_calls=self._data_config.num_parallel_calls) + dataset = dataset.map( + self._parse_record, + num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls, + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/dummy_input.py b/easy_rec/python/input/dummy_input.py index 9ede8ba89..6863b2210 100644 --- a/easy_rec/python/input/dummy_input.py +++ b/easy_rec/python/input/dummy_input.py @@ -17,24 +17,24 @@ class DummyInput(Input): """ def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - input_vals={}, - ): - super(DummyInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + input_vals={}, + ): + super(DummyInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) self._input_vals = input_vals @@ -50,7 +50,9 @@ def _build(self, mode, params): label tensor dict """ features = {} - for field, field_type, def_val in zip(self._input_fields, self._input_field_types, self._input_field_defaults): + for field, field_type, def_val in zip(self._input_fields, + self._input_field_types, + self._input_field_defaults): tf_type = get_tf_type(field_type) def_val = self.get_type_defaults(field_type, default_val=def_val) diff --git a/easy_rec/python/input/hive_input.py b/easy_rec/python/input/hive_input.py index 4e35ae751..b5f2ba7e0 100644 --- a/easy_rec/python/input/hive_input.py +++ b/easy_rec/python/input/hive_input.py @@ -12,23 +12,23 @@ class HiveInput(Input): """Common IO based interface, could run at local or on data science.""" def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(HiveInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(HiveInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) if input_path is None: return @@ -36,11 +36,13 @@ def __init__( self._feature_config = feature_config self._hive_config = input_path - hive_util = HiveUtils(data_config=self._data_config, hive_config=self._hive_config) - self._input_hdfs_path = hive_util.get_table_location(self._hive_config.table_name) + hive_util = HiveUtils( + data_config=self._data_config, hive_config=self._hive_config) + self._input_hdfs_path = hive_util.get_table_location( + self._hive_config.table_name) ( - self._input_table_col_names, - self._input_table_col_types, + self._input_table_col_names, + self._input_table_col_types, ) = hive_util.get_all_cols(self._hive_config.table_name) def _parse_csv(self, line): @@ -48,22 +50,24 @@ def _parse_csv(self, line): for field_name in self._input_table_col_names: if field_name in self._input_fields: tid = self._input_fields.index(field_name) - record_defaults.append(self.get_type_defaults(self._input_field_types[tid], self._input_field_defaults[tid])) + record_defaults.append( + self.get_type_defaults(self._input_field_types[tid], + self._input_field_defaults[tid])) else: record_defaults.append('') tmp_fields = tf.decode_csv( - line, - field_delim=self._data_config.separator, - record_defaults=record_defaults, - name='decode_csv', + line, + field_delim=self._data_config.separator, + record_defaults=record_defaults, + name='decode_csv', ) fields = [] for x in self._input_fields: assert x in self._input_table_col_names, 'Column %s not in Table %s.' % ( - x, - self._hive_config.table_name, + x, + self._hive_config.table_name, ) fields.append(tmp_fields[self._input_table_col_names.index(x)]) @@ -75,11 +79,13 @@ def _parse_csv(self, line): def _build(self, mode, params): file_paths = tf.gfile.Glob(os.path.join(self._input_hdfs_path, '*')) - assert len(file_paths) > 0, 'match no files with %s' % self._hive_config.table_name + assert len( + file_paths) > 0, 'match no files with %s' % self._hive_config.table_name num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('train files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -93,9 +99,9 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset(x), - cycle_length=parallel_num, - num_parallel_calls=parallel_num, + lambda x: tf.data.TextLineDataset(x), + cycle_length=parallel_num, + num_parallel_calls=parallel_num, ) if not self._data_config.file_shard: @@ -103,26 +109,30 @@ def _build(self, mode, params): if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('eval files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map(self._parse_csv, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + self._parse_csv, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/hive_parquet_input.py b/easy_rec/python/input/hive_parquet_input.py index 85162f79c..e771c5237 100644 --- a/easy_rec/python/input/hive_parquet_input.py +++ b/easy_rec/python/input/hive_parquet_input.py @@ -15,23 +15,23 @@ class HiveParquetInput(Input): """Common IO based interface, could run at local or on data science.""" def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(HiveParquetInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(HiveParquetInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) if input_path is None: return @@ -39,22 +39,24 @@ def __init__( self._feature_config = feature_config self._hive_config = input_path - hive_util = HiveUtils(data_config=self._data_config, hive_config=self._hive_config) + hive_util = HiveUtils( + data_config=self._data_config, hive_config=self._hive_config) input_hdfs_path = hive_util.get_table_location(self._hive_config.table_name) ( - self._input_table_col_names, - self._input_table_col_types, + self._input_table_col_names, + self._input_table_col_types, ) = hive_util.get_all_cols(self._hive_config.table_name) self._all_hdfs_path = tf.gfile.Glob(os.path.join(input_hdfs_path, '*')) for x in self._input_fields: assert x in self._input_table_col_names, 'Column %s not in Table %s.' % ( - x, - self._hive_config.table_name, + x, + self._hive_config.table_name, ) self._record_defaults = [ - self.get_type_defaults(t, v) for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) + for t, v in zip(self._input_field_types, self._input_field_defaults) ] def _file_shard(self, file_paths, task_num, task_index): @@ -79,8 +81,10 @@ def _parquet_read(self): for k, v in zip(self._input_fields, self._record_defaults): df[k].fillna(v, inplace=True) - for start_idx in range(0, total_records_num, self._data_config.batch_size): - end_idx = min(total_records_num, start_idx + self._data_config.batch_size) + for start_idx in range(0, total_records_num, + self._data_config.batch_size): + end_idx = min(total_records_num, + start_idx + self._data_config.batch_size) batch_data = df[start_idx:end_idx] inputs = [] for k in self._input_fields: @@ -104,41 +108,47 @@ def _build(self, mode, params): if len(self._all_hdfs_path) >= 2 * self._task_num: file_shard = True - self._input_hdfs_path = self._file_shard(self._all_hdfs_path, self._task_num, self._task_index) + self._input_hdfs_path = self._file_shard(self._all_hdfs_path, + self._task_num, self._task_index) else: file_shard = False self._input_hdfs_path = self._all_hdfs_path logging.info('input path: %s' % self._input_hdfs_path) - assert len(self._input_hdfs_path) > 0, 'match no files with %s' % self._hive_config.table_name + assert len(self._input_hdfs_path + ) > 0, 'match no files with %s' % self._hive_config.table_name - dataset = tf.data.Dataset.from_generator(self._parquet_read, output_types=list_type, output_shapes=list_shapes) + dataset = tf.data.Dataset.from_generator( + self._parquet_read, output_types=list_type, output_shapes=list_shapes) if not file_shard: dataset = self._safe_shard(dataset) if mode == tf.estimator.ModeKeys.TRAIN: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) - dataset = dataset.map(self._parse_csv, num_parallel_calls=self._data_config.num_parallel_calls) + dataset = dataset.map( + self._parse_csv, + num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls, + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/hive_rtp_input.py b/easy_rec/python/input/hive_rtp_input.py index b3d70cc61..00b6971d8 100644 --- a/easy_rec/python/input/hive_rtp_input.py +++ b/easy_rec/python/input/hive_rtp_input.py @@ -14,23 +14,23 @@ class HiveRTPInput(Input): """Common IO based interface, could run at local or on data science.""" def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(HiveRTPInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(HiveRTPInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) if input_path is None: return @@ -38,21 +38,24 @@ def __init__( self._feature_config = feature_config self._hive_config = input_path - logging.info('input_fields: %s label_fields: %s' % (','.join(self._input_fields), ','.join(self._label_fields))) + logging.info('input_fields: %s label_fields: %s' % + (','.join(self._input_fields), ','.join(self._label_fields))) self._rtp_separator = self._data_config.rtp_separator if not isinstance(self._rtp_separator, str): self._rtp_separator = self._rtp_separator.encode('utf-8') logging.info('rtp separator = %s' % self._rtp_separator) - self._selected_cols = ( - [c.strip() for c in self._data_config.selected_cols.split(',')] if self._data_config.selected_cols else None - ) + self._selected_cols = ([ + c.strip() for c in self._data_config.selected_cols.split(',') + ] if self._data_config.selected_cols else None) logging.info('select cols: %s' % self._selected_cols) - hive_util = HiveUtils(data_config=self._data_config, hive_config=self._hive_config) - self._input_hdfs_path = hive_util.get_table_location(self._hive_config.table_name) + hive_util = HiveUtils( + data_config=self._data_config, hive_config=self._hive_config) + self._input_hdfs_path = hive_util.get_table_location( + self._hive_config.table_name) ( - self._input_table_col_names, - self._input_table_col_types, + self._input_table_col_names, + self._input_table_col_types, ) = hive_util.get_all_cols(self._hive_config.table_name) def _parse_csv(self, line): @@ -63,15 +66,17 @@ def _parse_csv(self, line): for tid, field_name in enumerate(self._input_table_col_names): if field_name in self._selected_cols[:-1]: idx = self._input_fields.index(field_name) - record_defaults.append(self.get_type_defaults(self._input_field_types[idx], self._input_field_defaults[idx])) + record_defaults.append( + self.get_type_defaults(self._input_field_types[idx], + self._input_field_defaults[idx])) else: record_defaults.append('') print('record_defaults: ', record_defaults) tmp_fields = tf.decode_csv( - line, - field_delim=self._rtp_separator, - record_defaults=record_defaults, - name='decode_csv', + line, + field_delim=self._rtp_separator, + record_defaults=record_defaults, + name='decode_csv', ) print('tmp_fields: ', tmp_fields) @@ -84,32 +89,35 @@ def _parse_csv(self, line): labels = fields[:-1] # only for features, labels and sample_weight excluded - record_types = [t for x, t in zip(self._input_fields, self._input_field_types) if x not in non_feature_cols] + record_types = [ + t for x, t in zip(self._input_fields, self._input_field_types) + if x not in non_feature_cols + ] feature_num = len(record_types) - check_list = ( - [ + check_list = ([ tf.py_func( - check_split, - [fields[-1], self._data_config.separator, len(record_types)], - Tout=tf.bool, + check_split, + [fields[-1], self._data_config.separator, + len(record_types)], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): - fields = tf.string_split(fields[-1], self._data_config.separator, skip_empty=False) + fields = tf.string_split( + fields[-1], self._data_config.separator, skip_empty=False) tmp_fields = tf.reshape(fields.values, [-1, feature_num]) rtp_record_defaults = [ - str(self.get_type_defaults(t, v)) - for x, t, v in zip(self._input_fields, self._input_field_types, self._input_field_defaults) - if x not in non_feature_cols + str(self.get_type_defaults(t, v)) + for x, t, v in zip(self._input_fields, self._input_field_types, + self._input_field_defaults) + if x not in non_feature_cols ] - fields = labels[len(self._label_fields) :] + fields = labels[len(self._label_fields):] for i in range(feature_num): - field = string_to_number(tmp_fields[:, i], record_types[i], rtp_record_defaults[i], i) + field = string_to_number(tmp_fields[:, i], record_types[i], + rtp_record_defaults[i], i) fields.append(field) field_keys = [x for x in self._input_fields if x not in self._label_fields] @@ -122,11 +130,13 @@ def _parse_csv(self, line): def _build(self, mode, params): file_paths = tf.gfile.Glob(os.path.join(self._input_hdfs_path, '*')) - assert len(file_paths) > 0, 'match no files with %s' % self._hive_config.table_name + assert len( + file_paths) > 0, 'match no files with %s' % self._hive_config.table_name num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('train files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -140,9 +150,9 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TextLineDataset(x), - cycle_length=parallel_num, - num_parallel_calls=parallel_num, + lambda x: tf.data.TextLineDataset(x), + cycle_length=parallel_num, + num_parallel_calls=parallel_num, ) if not self._data_config.file_shard: @@ -150,26 +160,30 @@ def _build(self, mode, params): if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('eval files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map(self._parse_csv, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + self._parse_csv, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/input.py b/easy_rec/python/input/input.py index 66032b22a..227ee861a 100644 --- a/easy_rec/python/input/input.py +++ b/easy_rec/python/input/input.py @@ -8,23 +8,26 @@ import six import tensorflow as tf from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops, sparse_ops, string_ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import sparse_ops +from tensorflow.python.ops import string_ops from tensorflow.python.platform import gfile from easy_rec.python.core import sampler as sampler_lib from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.utils import conditional, config_util, constant -from easy_rec.python.utils.check_utils import ( # NOQA - check_split, - check_string_to_number, -) +from easy_rec.python.utils import conditional +from easy_rec.python.utils import config_util +from easy_rec.python.utils import constant from easy_rec.python.utils.expr_util import get_expression from easy_rec.python.utils.input_utils import get_type_defaults +from easy_rec.python.utils.tf_utils import get_tf_type + +from easy_rec.python.utils.check_utils import ( # NOQA + check_split, check_string_to_number, +) from easy_rec.python.utils.load_class import ( # NOQA - get_register_class_meta, - load_by_path, + get_register_class_meta, load_by_path, ) -from easy_rec.python.utils.tf_utils import get_tf_type if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -37,15 +40,15 @@ class Input(six.with_metaclass(_meta_type, object)): DATA_OFFSET = 'DATA_OFFSET' def __init__( - self, - data_config, - feature_configs, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs, + self, + data_config, + feature_configs, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + **kwargs, ): self._pipeline_config = pipeline_config self._data_config = data_config @@ -54,7 +57,8 @@ def __init__( # tf.estimator.ModeKeys.*, only available before # calling self._build self._mode = None - if pipeline_config is not None and pipeline_config.model_config.HasField('ev_params'): + if pipeline_config is not None and pipeline_config.model_config.HasField( + 'ev_params'): self._has_ev = True else: self._has_ev = False @@ -74,7 +78,9 @@ def __init__( self._input_fields = [x.input_name for x in data_config.input_fields] self._input_dims = [x.input_dim for x in data_config.input_fields] self._input_field_types = [x.input_type for x in data_config.input_fields] - self._input_field_defaults = [x.default_val for x in data_config.input_fields] + self._input_field_defaults = [ + x.default_val for x in data_config.input_fields + ] self._label_fields = list(data_config.label_fields) self._feature_fields = list(data_config.feature_fields) self._label_sep = list(data_config.label_sep) @@ -108,12 +114,14 @@ def __init__( self._normalizer_fn = {} for fc in self._feature_configs: for input_name in fc.input_names: - assert input_name in self._input_fields, 'invalid input_name in %s' % str(fc) + assert input_name in self._input_fields, 'invalid input_name in %s' % str( + fc) if input_name not in self._effective_fields: self._effective_fields.append(input_name) if fc.feature_type in [fc.TagFeature, fc.SequenceFeature]: - if fc.hash_bucket_size > 0 or len(fc.vocab_list) > 0 or fc.HasField('vocab_file'): + if fc.hash_bucket_size > 0 or len( + fc.vocab_list) > 0 or fc.HasField('vocab_file'): self._multi_value_types[fc.input_names[0]] = tf.string self._multi_value_fields.add(fc.input_names[0]) else: @@ -128,7 +136,8 @@ def __init__( self._multi_value_fields.add(fc.input_names[0]) if fc.HasField('normalizer_fn'): - feature_name = fc.feature_name if fc.HasField('feature_name') else fc.input_names[0] + feature_name = fc.feature_name if fc.HasField( + 'feature_name') else fc.input_names[0] self._normalizer_fn[feature_name] = load_by_path(fc.normalizer_fn) # add sample weight to effective fields @@ -154,7 +163,8 @@ def __init__( model_name = model_config.WhichOneof('model') if model_name in {'mmoe', 'esmm', 'dbmtl', 'simple_multi_task', 'ple'}: model = getattr(model_config, model_name) - towers = [model.ctr_tower, model.cvr_tower] if model_name == 'esmm' else model.task_towers + towers = [model.ctr_tower, model.cvr_tower + ] if model_name == 'esmm' else model.task_towers for tower in towers: metrics = tower.metrics_set for metric in metrics: @@ -168,10 +178,14 @@ def __init__( if sid not in self._effective_fields: self._effective_fields.append(sid) - self._effective_fids = [self._input_fields.index(x) for x in self._effective_fields] + self._effective_fids = [ + self._input_fields.index(x) for x in self._effective_fields + ] # sort fids from small to large self._effective_fids = list(set(self._effective_fids)) - self._effective_fields = [self._input_fields[x] for x in self._effective_fids] + self._effective_fields = [ + self._input_fields[x] for x in self._effective_fids + ] self._label_fids = [self._input_fields.index(x) for x in self._label_fields] @@ -189,8 +203,10 @@ def __init__( def _load_label_fn(self, config): udf_class = config.user_define_fn - udf_path = config.user_define_fn_path if config.HasField('user_define_fn_path') else None - dtype = config.user_define_fn_res_type if config.HasField('user_define_fn_res_type') else None + udf_path = config.user_define_fn_path if config.HasField( + 'user_define_fn_path') else None + dtype = config.user_define_fn_res_type if config.HasField( + 'user_define_fn_res_type') else None if udf_path: if udf_path.startswith('oss://') or udf_path.startswith('hdfs://'): @@ -219,7 +235,10 @@ def num_epochs(self): return None def get_feature_input_fields(self): - return [x for x in self._input_fields if x not in self._label_fields and x != self._data_config.sample_weight] + return [ + x for x in self._input_fields + if x not in self._label_fields and x != self._data_config.sample_weight + ] def should_stop(self, curr_epoch): """Check whether have run enough num epochs.""" @@ -252,9 +271,9 @@ def create_multi_placeholders(self, export_config): effective_fids = list(self._effective_fids) else: effective_fids = [ - fid - for fid in range(len(self._input_fields)) - if self._input_fields[fid] not in self._label_fields and self._input_fields[fid] != sample_weight_field + fid for fid in range(len(self._input_fields)) + if self._input_fields[fid] not in self._label_fields and + self._input_fields[fid] != sample_weight_field ] inputs = {} @@ -268,12 +287,13 @@ def create_multi_placeholders(self, export_config): placeholder_name = 'input_%d' % fid if input_name in export_fields_name: tf_type = ( - self._multi_value_types[input_name] - if input_name in self._multi_value_types - else get_tf_type(self._input_field_types[fid]) - ) - logging.info('multi value input_name: %s, dtype: %s' % (input_name, tf_type)) - finput = array_ops.placeholder(tf_type, [None, None], name=placeholder_name) + self._multi_value_types[input_name] + if input_name in self._multi_value_types else get_tf_type( + self._input_field_types[fid])) + logging.info('multi value input_name: %s, dtype: %s' % + (input_name, tf_type)) + finput = array_ops.placeholder( + tf_type, [None, None], name=placeholder_name) else: ftype = self._input_field_types[fid] tf_type = get_tf_type(ftype) @@ -286,8 +306,11 @@ def create_multi_placeholders(self, export_config): def create_placeholders(self, export_config): self._mode = tf.estimator.ModeKeys.PREDICT - inputs_placeholder = array_ops.placeholder(tf.string, [None], name='features') - input_vals = tf.string_split(inputs_placeholder, self._data_config.separator, skip_empty=False).values + inputs_placeholder = array_ops.placeholder( + tf.string, [None], name='features') + input_vals = tf.string_split( + inputs_placeholder, self._data_config.separator, + skip_empty=False).values sample_weight_field = '' if self._data_config.HasField('sample_weight'): @@ -295,17 +318,19 @@ def create_placeholders(self, export_config): if export_config.filter_inputs: effective_fids = list(self._effective_fids) - logging.info( - 'number of effective inputs:%d, total number inputs: %d' % (len(effective_fids), len(self._input_fields)) - ) + logging.info('number of effective inputs:%d, total number inputs: %d' % + (len(effective_fids), len(self._input_fields))) else: effective_fids = [ - fid - for fid in range(len(self._input_fields)) - if self._input_fields[fid] not in self._label_fields and self._input_fields[fid] != sample_weight_field + fid for fid in range(len(self._input_fields)) + if self._input_fields[fid] not in self._label_fields and + self._input_fields[fid] != sample_weight_field ] - logging.info('will not filter any input[except labels], total number inputs:%d' % len(effective_fids)) - input_vals = tf.reshape(input_vals, [-1, len(effective_fids)], name='input_reshape') + logging.info( + 'will not filter any input[except labels], total number inputs:%d' % + len(effective_fids)) + input_vals = tf.reshape( + input_vals, [-1, len(effective_fids)], name='input_reshape') features = {} for tmp_id, fid in enumerate(effective_fids): ftype = self._input_field_types[fid] @@ -313,13 +338,14 @@ def create_placeholders(self, export_config): input_name = self._input_fields[fid] if tf_type in [tf.float32, tf.double, tf.int32, tf.int64]: features[input_name] = tf.string_to_number( - input_vals[:, tmp_id], - tf_type, - name='input_str_to_%s' % tf_type.name, + input_vals[:, tmp_id], + tf_type, + name='input_str_to_%s' % tf_type.name, ) else: if ftype not in [DatasetConfig.STRING]: - logging.warning('unexpected field type: ftype=%s tf_type=%s' % (ftype, tf_type)) + logging.warning('unexpected field type: ftype=%s tf_type=%s' % + (ftype, tf_type)) features[input_name] = input_vals[:, tmp_id] features = self._preprocess(features) return {'features': inputs_placeholder}, features['feature'] @@ -329,32 +355,24 @@ def _get_features(self, fields): def _get_labels(self, fields): labels = fields['label'] - return OrderedDict( - [ - ( - x, - ( - tf.squeeze(labels[x], axis=1) - if len(labels[x].get_shape()) == 2 and labels[x].get_shape()[1] == 1 - else labels[x] - ), - ) - for x in labels - ] - ) + return OrderedDict([( + x, + (tf.squeeze(labels[x], axis=1) if len(labels[x].get_shape()) == 2 and + labels[x].get_shape()[1] == 1 else labels[x]), + ) for x in labels]) def _as_string(self, field, fc): if field.dtype == tf.string: return field if field.dtype in [tf.float32, tf.double]: - feature_name = fc.feature_name if fc.HasField('feature_name') else fc.input_names[0] + feature_name = fc.feature_name if fc.HasField( + 'feature_name') else fc.input_names[0] assert fc.precision > 0, ( - 'fc.precision not set for feature[%s], it is dangerous to convert ' - 'float or double to string due to precision problem, it is suggested ' - ' to convert them into string format before using EasyRec; ' - 'if you really need to do so, please set precision (the number of ' - 'decimal digits) carefully.' % feature_name - ) + 'fc.precision not set for feature[%s], it is dangerous to convert ' + 'float or double to string due to precision problem, it is suggested ' + ' to convert them into string format before using EasyRec; ' + 'if you really need to do so, please set precision (the number of ' + 'decimal digits) carefully.' % feature_name) precision = None if field.dtype in [tf.float32, tf.double]: if fc.precision > 0: @@ -368,12 +386,15 @@ def _as_string(self, field, fc): def _parse_combo_feature(self, fc, parsed_dict, field_dict): # for compatibility with existing implementations - feature_name = fc.feature_name if fc.HasField('feature_name') else fc.input_names[0] + feature_name = fc.feature_name if fc.HasField( + 'feature_name') else fc.input_names[0] if len(fc.combo_input_seps) > 0: - assert len(fc.combo_input_seps) == len(fc.input_names), 'len(combo_separator)[%d] != len(fc.input_names)[%d]' % ( - len(fc.combo_input_seps), - len(fc.input_names), + assert len(fc.combo_input_seps) == len( + fc.input_names + ), 'len(combo_separator)[%d] != len(fc.input_names)[%d]' % ( + len(fc.combo_input_seps), + len(fc.input_names), ) def _get_input_sep(input_id): @@ -391,12 +412,10 @@ def _get_input_sep(input_id): input_sep = _get_input_sep(input_id) if input_sep != '': assert field_dict[input_name].dtype == tf.string, ( - 'could not apply string_split to input-name[%s] dtype=%s' - % ( - input_name, - field_dict[input_name].dtype, - ) - ) + 'could not apply string_split to input-name[%s] dtype=%s' % ( + input_name, + field_dict[input_name].dtype, + )) parsed_dict[key] = tf.string_split(field_dict[input_name], input_sep) else: parsed_dict[key] = self._as_string(field_dict[input_name], fc) @@ -407,19 +426,24 @@ def _get_input_sep(input_id): input_sep = fc.combo_input_seps[input_id] if len(input_sep) > 0: assert field_dict[input_name].dtype == tf.string, ( - 'could not apply string_split to input-name[%s] dtype=%s' - % ( - input_name, - field_dict[input_name].dtype, - ) - ) - split_inputs.append(tf.string_split(field_dict[input_name], fc.combo_input_seps[input_id])) + 'could not apply string_split to input-name[%s] dtype=%s' % ( + input_name, + field_dict[input_name].dtype, + )) + split_inputs.append( + tf.string_split(field_dict[input_name], + fc.combo_input_seps[input_id])) else: split_inputs.append(tf.reshape(field_dict[input_name], [-1, 1])) - parsed_dict[feature_name] = sparse_ops.sparse_cross(split_inputs, fc.combo_join_sep) + parsed_dict[feature_name] = sparse_ops.sparse_cross( + split_inputs, fc.combo_join_sep) else: - inputs = [self._as_string(field_dict[input_name], fc) for input_name in fc.input_names] - parsed_dict[feature_name] = string_ops.string_join(inputs, fc.combo_join_sep) + inputs = [ + self._as_string(field_dict[input_name], fc) + for input_name in fc.input_names + ] + parsed_dict[feature_name] = string_ops.string_join( + inputs, fc.combo_join_sep) def _parse_tag_feature(self, fc, parsed_dict, field_dict): input_0 = fc.input_names[0] @@ -434,9 +458,9 @@ def _parse_tag_feature(self, fc, parsed_dict, field_dict): field = tf.squeeze(field, axis=-1) if fc.HasField('kv_separator') and len(fc.input_names) > 1: assert False, ( - 'Tag Feature Error, ' - 'Cannot set kv_separator and multi input_names in one feature config. Feature: %s.' % input_0 - ) + 'Tag Feature Error, ' + 'Cannot set kv_separator and multi input_names in one feature config. Feature: %s.' + % input_0) parsed_dict[feature_name] = tf.string_split(field, fc.separator) if fc.HasField('kv_separator'): indices = parsed_dict[feature_name].indices @@ -445,35 +469,34 @@ def _parse_tag_feature(self, fc, parsed_dict, field_dict): tmp_kvs = tf.reshape(tmp_kvs.values, [-1, 2]) tmp_ks, tmp_vs = tmp_kvs[:, 0], tmp_kvs[:, 1] - check_list = [tf.py_func(check_string_to_number, [tmp_vs, input_0], Tout=tf.bool)] if self._check_mode else [] + check_list = [ + tf.py_func(check_string_to_number, [tmp_vs, input_0], Tout=tf.bool) + ] if self._check_mode else [] with tf.control_dependencies(check_list): - tmp_vs = tf.string_to_number(tmp_vs, tf.float32, name='kv_tag_wgt_str_2_flt_%s' % input_0) - parsed_dict[feature_name] = tf.sparse.SparseTensor(indices, tmp_ks, parsed_dict[feature_name].dense_shape) + tmp_vs = tf.string_to_number( + tmp_vs, tf.float32, name='kv_tag_wgt_str_2_flt_%s' % input_0) + parsed_dict[feature_name] = tf.sparse.SparseTensor( + indices, tmp_ks, parsed_dict[feature_name].dense_shape) parsed_dict[feature_name + '_w'] = tf.sparse.SparseTensor( - indices, tmp_vs, parsed_dict[feature_name].dense_shape - ) + indices, tmp_vs, parsed_dict[feature_name].dense_shape) if not fc.HasField('hash_bucket_size') and fc.num_buckets > 0: - check_list = ( - [ + check_list = ([ tf.py_func( - check_string_to_number, - [parsed_dict[feature_name].values, input_0], - Tout=tf.bool, + check_string_to_number, + [parsed_dict[feature_name].values, input_0], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): vals = tf.string_to_number( - parsed_dict[feature_name].values, - tf.int32, - name='tag_fea_%s' % input_0, + parsed_dict[feature_name].values, + tf.int32, + name='tag_fea_%s' % input_0, ) parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, - vals, - parsed_dict[feature_name].dense_shape, + parsed_dict[feature_name].indices, + vals, + parsed_dict[feature_name].dense_shape, ) if len(fc.input_names) > 1: input_1 = fc.input_names[1] @@ -481,27 +504,25 @@ def _parse_tag_feature(self, fc, parsed_dict, field_dict): if len(field.get_shape()) == 0: field = tf.expand_dims(field, axis=0) field = tf.string_split(field, fc.separator) - check_list = ( - [ + check_list = ([ tf.py_func( - check_string_to_number, - [field.values, input_1], - Tout=tf.bool, + check_string_to_number, + [field.values, input_1], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): - field_vals = tf.string_to_number(field.values, tf.float32, name='tag_wgt_str_2_flt_%s' % input_1) + field_vals = tf.string_to_number( + field.values, tf.float32, name='tag_wgt_str_2_flt_%s' % input_1) assert_op = tf.assert_equal( - tf.shape(field_vals)[0], - tf.shape(parsed_dict[feature_name].values)[0], - message='TagFeature Error: The size of %s not equal to %s. Please check input: %s and %s.' - % (input_0, input_1, input_0, input_1), + tf.shape(field_vals)[0], + tf.shape(parsed_dict[feature_name].values)[0], + message='TagFeature Error: The size of %s not equal to %s. Please check input: %s and %s.' + % (input_0, input_1, input_0, input_1), ) with tf.control_dependencies([assert_op]): - field = tf.sparse.SparseTensor(field.indices, tf.identity(field_vals), field.dense_shape) + field = tf.sparse.SparseTensor(field.indices, tf.identity(field_vals), + field.dense_shape) parsed_dict[feature_name + '_w'] = field else: parsed_dict[feature_name] = field_dict[input_0] @@ -515,32 +536,30 @@ def _parse_expr_feature(self, fc, parsed_dict, field_dict): for input_name in fc.input_names: new_input_name = prefix + input_name if field_dict[input_name].dtype == tf.string: - check_list = ( - [ + check_list = ([ tf.py_func( - check_string_to_number, - [field_dict[input_name], input_name], - Tout=tf.bool, + check_string_to_number, + [field_dict[input_name], input_name], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): parsed_dict[new_input_name] = tf.string_to_number( - field_dict[input_name], - tf.float64, - name='%s_str_2_int_for_expr' % new_input_name, + field_dict[input_name], + tf.float64, + name='%s_str_2_int_for_expr' % new_input_name, ) elif field_dict[input_name].dtype in [ - tf.int32, - tf.int64, - tf.double, - tf.float32, + tf.int32, + tf.int64, + tf.double, + tf.float32, ]: - parsed_dict[new_input_name] = tf.cast(field_dict[input_name], tf.float64) + parsed_dict[new_input_name] = tf.cast(field_dict[input_name], + tf.float64) else: - assert False, 'invalid input dtype[%s] for expr feature' % str(field_dict[input_name].dtype) + assert False, 'invalid input dtype[%s] for expr feature' % str( + field_dict[input_name].dtype) expression = get_expression(fc.expression, fc.input_names, prefix=prefix) logging.info('expression: %s' % expression) @@ -556,22 +575,18 @@ def _parse_id_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name] = self._as_string(field_dict[input_0], fc) elif fc.num_buckets > 0: if parsed_dict[feature_name].dtype == tf.string: - check_list = ( - [ + check_list = ([ tf.py_func( - check_string_to_number, - [parsed_dict[feature_name], input_0], - Tout=tf.bool, + check_string_to_number, + [parsed_dict[feature_name], input_0], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.string_to_number( - parsed_dict[feature_name], - tf.int32, - name='%s_str_2_int' % input_0, + parsed_dict[feature_name], + tf.int32, + name='%s_str_2_int' % input_0, ) def _parse_raw_feature(self, fc, parsed_dict, field_dict): @@ -586,35 +601,27 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): vals = field_dict[input_0] segment_ids = tf.range(0, tf.shape(vals)[0]) if fc.raw_input_dim > 1: - check_list = ( - [ + check_list = ([ tf.py_func( - check_split, - [vals, fc.separator, fc.raw_input_dim, input_0], - Tout=tf.bool, + check_split, + [vals, fc.separator, fc.raw_input_dim, input_0], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): tmp_fea = tf.string_split(vals, fc.separator) - check_list = ( - [ + check_list = ([ tf.py_func( - check_string_to_number, - [tmp_fea.values, input_0], - Tout=tf.bool, + check_string_to_number, + [tmp_fea.values, input_0], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): tmp_vals = tf.string_to_number( - tmp_fea.values, - tf.float32, - name='multi_raw_fea_to_flt_%s' % input_0, + tmp_fea.values, + tf.float32, + name='multi_raw_fea_to_flt_%s' % input_0, ) if fc.HasField('seq_multi_sep') and fc.HasField('combiner'): emb = tf.reshape(tmp_vals, [-1, fc.raw_input_dim]) @@ -631,15 +638,18 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name] = emb else: parsed_dict[feature_name] = tf.sparse_to_dense( - tmp_fea.indices, - [tf.shape(field_dict[input_0])[0], fc.raw_input_dim], - tmp_vals, - default_value=0, + tmp_fea.indices, + [tf.shape(field_dict[input_0])[0], fc.raw_input_dim], + tmp_vals, + default_value=0, ) elif fc.HasField('seq_multi_sep') and fc.HasField('combiner'): - check_list = [tf.py_func(check_string_to_number, [vals, input_0], Tout=tf.bool)] if self._check_mode else [] + check_list = [ + tf.py_func(check_string_to_number, [vals, input_0], Tout=tf.bool) + ] if self._check_mode else [] with tf.control_dependencies(check_list): - emb = tf.string_to_number(vals, tf.float32, name='raw_fea_to_flt_%s' % input_0) + emb = tf.string_to_number( + vals, tf.float32, name='raw_fea_to_flt_%s' % input_0) if fc.combiner == 'max': emb = tf.segment_max(emb, segment_ids) elif fc.combiner == 'sum': @@ -652,33 +662,35 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): assert False, 'unsupported combine operator: ' + fc.combiner parsed_dict[feature_name] = emb else: - check_list = ( - [ + check_list = ([ tf.py_func( - check_string_to_number, - [field_dict[input_0], input_0], - Tout=tf.bool, + check_string_to_number, + [field_dict[input_0], input_0], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): - parsed_dict[feature_name] = tf.string_to_number(field_dict[input_0], tf.float32) - elif field_dict[input_0].dtype in [tf.int32, tf.int64, tf.double, tf.float32]: + parsed_dict[feature_name] = tf.string_to_number( + field_dict[input_0], tf.float32) + elif field_dict[input_0].dtype in [ + tf.int32, tf.int64, tf.double, tf.float32 + ]: parsed_dict[feature_name] = tf.to_float(field_dict[input_0]) else: - assert False, 'invalid dtype[%s] for raw feature' % str(field_dict[input_0].dtype) + assert False, 'invalid dtype[%s] for raw feature' % str( + field_dict[input_0].dtype) if fc.max_val > fc.min_val: - parsed_dict[feature_name] = (parsed_dict[feature_name] - fc.min_val) / (fc.max_val - fc.min_val) + parsed_dict[feature_name] = (parsed_dict[feature_name] - fc.min_val) / ( + fc.max_val - fc.min_val) if fc.HasField('normalizer_fn'): - logging.info('apply normalizer_fn %s to `%s`' % (fc.normalizer_fn, feature_name)) - parsed_dict[feature_name] = self._normalizer_fn[feature_name](parsed_dict[feature_name]) + logging.info('apply normalizer_fn %s to `%s`' % + (fc.normalizer_fn, feature_name)) + parsed_dict[feature_name] = self._normalizer_fn[feature_name]( + parsed_dict[feature_name]) - if ( - not fc.boundaries and fc.num_buckets <= 1 and fc.embedding_dim > 0 and self._data_config.sample_weight != input_0 - ): + if (not fc.boundaries and fc.num_buckets <= 1 and fc.embedding_dim > 0 and + self._data_config.sample_weight != input_0): # may need by wide model and deep model to project # raw values to a vector, it maybe better implemented # by a ProjectionColumn later @@ -695,14 +707,14 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): tmp_parsed = parsed_dict[feature_name] parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( - indices=indices, - values=indices_1[:, 0], - dense_shape=[sample_num, fc.raw_input_dim], + indices=indices, + values=indices_1[:, 0], + dense_shape=[sample_num, fc.raw_input_dim], ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( - indices=indices, - values=tf.reshape(tmp_parsed, [-1]), - dense_shape=[sample_num, fc.raw_input_dim], + indices=indices, + values=tf.reshape(tmp_parsed, [-1]), + dense_shape=[sample_num, fc.raw_input_dim], ) # self._appended_fields.append(input_0 + '_raw_proj_id') # self._appended_fields.append(input_0 + '_raw_proj_val') @@ -725,81 +737,67 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): out_indices = tf.concat([indices, indices_1[:, 1:]], axis=1) # 3 dimensional sparse tensor out_shape = tf.concat( - [parsed_dict[feature_name].dense_shape, multi_vals.dense_shape[1:]], - axis=0, + [parsed_dict[feature_name].dense_shape, multi_vals.dense_shape[1:]], + axis=0, ) - parsed_dict[feature_name] = tf.sparse.SparseTensor(out_indices, multi_vals.values, out_shape) + parsed_dict[feature_name] = tf.sparse.SparseTensor( + out_indices, multi_vals.values, out_shape) if fc.num_buckets > 1 and fc.max_val == fc.min_val: - check_list = ( - [ + check_list = ([ tf.py_func( - check_string_to_number, - [parsed_dict[feature_name].values, input_0], - Tout=tf.bool, + check_string_to_number, + [parsed_dict[feature_name].values, input_0], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, - tf.string_to_number( - parsed_dict[feature_name].values, - tf.int64, - name='sequence_str_2_int_%s' % input_0, - ), - parsed_dict[feature_name].dense_shape, + parsed_dict[feature_name].indices, + tf.string_to_number( + parsed_dict[feature_name].values, + tf.int64, + name='sequence_str_2_int_%s' % input_0, + ), + parsed_dict[feature_name].dense_shape, ) elif sub_feature_type == fc.RawFeature: - check_list = ( - [ + check_list = ([ tf.py_func( - check_string_to_number, - [parsed_dict[feature_name].values, input_0], - Tout=tf.bool, + check_string_to_number, + [parsed_dict[feature_name].values, input_0], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, - tf.string_to_number( - parsed_dict[feature_name].values, - tf.float32, - name='sequence_str_2_float_%s' % input_0, - ), - parsed_dict[feature_name].dense_shape, + parsed_dict[feature_name].indices, + tf.string_to_number( + parsed_dict[feature_name].values, + tf.float32, + name='sequence_str_2_float_%s' % input_0, + ), + parsed_dict[feature_name].dense_shape, ) if fc.num_buckets > 1 and fc.max_val > fc.min_val: - normalized_values = (parsed_dict[feature_name].values - fc.min_val) / (fc.max_val - fc.min_val) + normalized_values = (parsed_dict[feature_name].values - fc.min_val) / ( + fc.max_val - fc.min_val) parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, - normalized_values, - parsed_dict[feature_name].dense_shape, + parsed_dict[feature_name].indices, + normalized_values, + parsed_dict[feature_name].dense_shape, ) else: parsed_dict[feature_name] = field - if ( - not fc.boundaries - and fc.num_buckets <= 1 - and self._data_config.sample_weight != input_0 - and sub_feature_type == fc.RawFeature - and fc.raw_input_dim == 1 - ): - logging.info( - ( - 'Not set boundaries or num_buckets or hash_bucket_size, ' - '%s will process as two dimension sequence raw feature' - ) - % feature_name - ) + if (not fc.boundaries and fc.num_buckets <= 1 and + self._data_config.sample_weight != input_0 and + sub_feature_type == fc.RawFeature and fc.raw_input_dim == 1): + logging.info(('Not set boundaries or num_buckets or hash_bucket_size, ' + '%s will process as two dimension sequence raw feature') % + feature_name) parsed_dict[feature_name] = tf.sparse_to_dense( - parsed_dict[feature_name].indices, - [tf.shape(parsed_dict[feature_name])[0], fc.sequence_length], - parsed_dict[feature_name].values, + parsed_dict[feature_name].indices, + [tf.shape(parsed_dict[feature_name])[0], fc.sequence_length], + parsed_dict[feature_name].values, ) sample_num = tf.to_int64(tf.shape(parsed_dict[feature_name])[0]) indices_0 = tf.range(sample_num, dtype=tf.int64) @@ -813,35 +811,30 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): indices = tf.concat([indices_0, indices_1], axis=1) tmp_parsed = parsed_dict[feature_name] parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( - indices=indices, - values=indices_1[:, 0], - dense_shape=[sample_num, fc.sequence_length], + indices=indices, + values=indices_1[:, 0], + dense_shape=[sample_num, fc.sequence_length], ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( - indices=indices, - values=tf.reshape(tmp_parsed, [-1]), - dense_shape=[sample_num, fc.sequence_length], + indices=indices, + values=tf.reshape(tmp_parsed, [-1]), + dense_shape=[sample_num, fc.sequence_length], ) - elif ( - not fc.boundaries - and fc.num_buckets <= 1 - and self._data_config.sample_weight != input_0 - and sub_feature_type == fc.RawFeature - and fc.raw_input_dim > 1 - ): + elif (not fc.boundaries and fc.num_buckets <= 1 and + self._data_config.sample_weight != input_0 and + sub_feature_type == fc.RawFeature and fc.raw_input_dim > 1): # for 3 dimension sequence feature input. - logging.info( - 'Not set boundaries or num_buckets or hash_bucket_size,' - ' %s will process as three dimension sequence raw feature' % feature_name - ) + logging.info('Not set boundaries or num_buckets or hash_bucket_size,' + ' %s will process as three dimension sequence raw feature' % + feature_name) parsed_dict[feature_name] = tf.sparse_to_dense( - parsed_dict[feature_name].indices, - [ - tf.shape(parsed_dict[feature_name])[0], - fc.sequence_length, - fc.raw_input_dim, - ], - parsed_dict[feature_name].values, + parsed_dict[feature_name].indices, + [ + tf.shape(parsed_dict[feature_name])[0], + fc.sequence_length, + fc.raw_input_dim, + ], + parsed_dict[feature_name].values, ) sample_num = tf.to_int64(tf.shape(parsed_dict[feature_name])[0]) indices_0 = tf.range(sample_num, dtype=tf.int64) @@ -860,14 +853,14 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): tmp_parsed = parsed_dict[feature_name] parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( - indices=indices, - values=indices_1[:, 0], - dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim], + indices=indices, + values=indices_1[:, 0], + dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim], ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( - indices=indices, - values=tf.reshape(parsed_dict[feature_name], [-1]), - dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim], + indices=indices, + values=tf.reshape(parsed_dict[feature_name], [-1]), + dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim], ) # self._appended_fields.append(input_0 + '_raw_proj_id') # self._appended_fields.append(input_0 + '_raw_proj_val') @@ -933,7 +926,8 @@ def _preprocess(self, field_dict): elif feature_type == fc.ComboFeature: self._parse_combo_feature(fc, parsed_dict, field_dict) else: - feature_name = fc.feature_name if fc.HasField('feature_name') else fc.input_names[0] + feature_name = fc.feature_name if fc.HasField( + 'feature_name') else fc.input_names[0] for input_id, input_name in enumerate(fc.input_names): if input_id > 0: key = feature_name + '_' + str(input_id) @@ -953,52 +947,48 @@ def _preprocess(self, field_dict): else: assert dtype is not None, 'must set user_define_fn_res_type' logging.info('apply py_func transform: %s' % udf_class) - field_dict[input_name] = tf.py_func(udf, [field_dict[input_name]], Tout=get_tf_type(dtype)) + field_dict[input_name] = tf.py_func( + udf, [field_dict[input_name]], Tout=get_tf_type(dtype)) field_dict[input_name].set_shape(tf.TensorShape([None])) if field_dict[input_name].dtype == tf.string: if self._label_dim[input_id] > 1: logging.info('will split labels[%d]=%s' % (input_id, input_name)) - check_list = ( - [ + check_list = ([ tf.py_func( - check_split, - [ - field_dict[input_name], - self._label_sep[input_id], - self._label_dim[input_id], - input_name, - ], - Tout=tf.bool, + check_split, + [ + field_dict[input_name], + self._label_sep[input_id], + self._label_dim[input_id], + input_name, + ], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): - label_dict[input_name] = tf.string_split(field_dict[input_name], self._label_sep[input_id]).values - label_dict[input_name] = tf.reshape(label_dict[input_name], [-1, self._label_dim[input_id]]) + label_dict[input_name] = tf.string_split( + field_dict[input_name], self._label_sep[input_id]).values + label_dict[input_name] = tf.reshape(label_dict[input_name], + [-1, self._label_dim[input_id]]) else: label_dict[input_name] = field_dict[input_name] - check_list = ( - [ + check_list = ([ tf.py_func( - check_string_to_number, - [label_dict[input_name], input_name], - Tout=tf.bool, + check_string_to_number, + [label_dict[input_name], input_name], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): - label_dict[input_name] = tf.string_to_number(label_dict[input_name], tf.float32, name=input_name) + label_dict[input_name] = tf.string_to_number( + label_dict[input_name], tf.float32, name=input_name) else: assert field_dict[input_name].dtype in [ - tf.float32, - tf.double, - tf.int32, - tf.int64, + tf.float32, + tf.double, + tf.int32, + tf.int64, ], 'invalid label dtype: %s' % str(field_dict[input_name].dtype) label_dict[input_name] = field_dict[input_name] @@ -1006,12 +996,14 @@ def _preprocess(self, field_dict): for func_config in self._data_config.extra_label_func: lbl_name = func_config.label_name func_name = func_config.label_func - logging.info('generating new label `%s` by transform: %s' % (lbl_name, func_name)) + logging.info('generating new label `%s` by transform: %s' % + (lbl_name, func_name)) lbl_fn = load_by_path(func_name) label_dict[lbl_name] = lbl_fn(label_dict) if self._data_config.HasField('sample_weight'): - parsed_dict[constant.SAMPLE_WEIGHT] = field_dict[self._data_config.sample_weight] + parsed_dict[constant.SAMPLE_WEIGHT] = field_dict[ + self._data_config.sample_weight] if Input.DATA_OFFSET in field_dict: parsed_dict[Input.DATA_OFFSET] = field_dict[Input.DATA_OFFSET] @@ -1059,13 +1051,14 @@ def _lookup(args, pad=True): indices_0 = tf.zeros([n], dtype=tf.int64) indices_1 = tf.range(0, n, dtype=tf.int64) indices = [ - tf.expand_dims(indices_0, axis=1), - tf.expand_dims(indices_1, axis=1), + tf.expand_dims(indices_0, axis=1), + tf.expand_dims(indices_1, axis=1), ] indices = tf.concat(indices, axis=1) return tf.sparse.SparseTensor(indices, vals, [1, n]) - vals, masks, indices = tf.map_fn(_lookup, [key_fields, map_fields], dtype=(tf.string, tf.bool, tf.int64)) + vals, masks, indices = tf.map_fn( + _lookup, [key_fields, map_fields], dtype=(tf.string, tf.bool, tf.int64)) batch_size = tf.to_int64(tf.shape(vals)[0]) vals = tf.boolean_mask(vals, masks) indices_1 = tf.boolean_mask(indices, masks) @@ -1074,8 +1067,9 @@ def _lookup(args, pad=True): indices_0 = indices_0 + tf.zeros([1, max_sel_num], dtype=tf.int64) indices_0 = tf.boolean_mask(indices_0, masks) indices = tf.concat( - [tf.expand_dims(indices_0, axis=1), tf.expand_dims(indices_1, axis=1)], - axis=1, + [tf.expand_dims(indices_0, axis=1), + tf.expand_dims(indices_1, axis=1)], + axis=1, ) shapes = tf.stack([batch_size, tf.reduce_max(indices_1) + 1]) return tf.sparse.SparseTensor(indices, vals, shapes) @@ -1095,11 +1089,13 @@ def stop(self): def _safe_shard(self, dataset): if self._data_config.chief_redundant: - return dataset.shard(max(self._task_num - 1, 1), max(self._task_index - 1, 0)) + return dataset.shard( + max(self._task_num - 1, 1), max(self._task_index - 1, 0)) else: return dataset.shard(self._task_num, self._task_index) def create_input(self, export_config=None): + def _input_fn(mode=None, params=None, config=None): """Build input_fn for estimator. @@ -1117,9 +1113,9 @@ def _input_fn(mode=None, params=None, config=None): """ self._pre_build(mode, params) if mode in ( - tf.estimator.ModeKeys.TRAIN, - tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT, + tf.estimator.ModeKeys.TRAIN, + tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT, ): # build dataset from self._config.input_path self._mode = mode @@ -1135,7 +1131,8 @@ def _input_fn(mode=None, params=None, config=None): else: with conditional(place_on_cpu, ops.device('/CPU:0')): inputs, features = self.create_placeholders(export_config) - print('built feature placeholders. features: {}'.format(features.keys())) + print('built feature placeholders. features: {}'.format( + features.keys())) return tf.estimator.export.ServingInputReceiver(features, inputs) _input_fn.input_creator = self diff --git a/easy_rec/python/input/kafka_dataset.py b/easy_rec/python/input/kafka_dataset.py index fcaa985da..20ed51872 100644 --- a/easy_rec/python/input/kafka_dataset.py +++ b/easy_rec/python/input/kafka_dataset.py @@ -18,7 +18,9 @@ import traceback from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes, ops, tensor_shape +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape try: from easy_rec.python.ops import gen_kafka_ops @@ -30,16 +32,16 @@ class KafkaDataset(dataset_ops.Dataset): """A Kafka Dataset that consumes the message.""" def __init__( - self, - topics, - servers='localhost', - group='', - eof=False, - timeout=1000, - config_global=None, - config_topic=None, - message_key=False, - message_offset=False, + self, + topics, + servers='localhost', + group='', + eof=False, + timeout=1000, + config_global=None, + config_topic=None, + message_key=False, + message_offset=False, ): """Create a KafkaReader. @@ -66,15 +68,21 @@ def __init__( message_key: If True, the kafka will output both message value and key. message_offset: If True, the kafka will output both message value and offset. """ - self._topics = ops.convert_to_tensor(topics, dtype=dtypes.string, name='topics') - self._servers = ops.convert_to_tensor(servers, dtype=dtypes.string, name='servers') - self._group = ops.convert_to_tensor(group, dtype=dtypes.string, name='group') + self._topics = ops.convert_to_tensor( + topics, dtype=dtypes.string, name='topics') + self._servers = ops.convert_to_tensor( + servers, dtype=dtypes.string, name='servers') + self._group = ops.convert_to_tensor( + group, dtype=dtypes.string, name='group') self._eof = ops.convert_to_tensor(eof, dtype=dtypes.bool, name='eof') - self._timeout = ops.convert_to_tensor(timeout, dtype=dtypes.int64, name='timeout') + self._timeout = ops.convert_to_tensor( + timeout, dtype=dtypes.int64, name='timeout') config_global = config_global if config_global else [] - self._config_global = ops.convert_to_tensor(config_global, dtype=dtypes.string, name='config_global') + self._config_global = ops.convert_to_tensor( + config_global, dtype=dtypes.string, name='config_global') config_topic = config_topic if config_topic else [] - self._config_topic = ops.convert_to_tensor(config_topic, dtype=dtypes.string, name='config_topic') + self._config_topic = ops.convert_to_tensor( + config_topic, dtype=dtypes.string, name='config_topic') self._message_key = message_key self._message_offset = message_offset super(KafkaDataset, self).__init__() @@ -84,15 +92,15 @@ def _inputs(self): def _as_variant_tensor(self): return gen_kafka_ops.io_kafka_dataset_v2( - self._topics, - self._servers, - self._group, - self._eof, - self._timeout, - self._config_global, - self._config_topic, - self._message_key, - self._message_offset, + self._topics, + self._servers, + self._group, + self._eof, + self._timeout, + self._config_global, + self._config_topic, + self._message_key, + self._message_offset, ) @property @@ -109,9 +117,9 @@ def output_shapes(self): return (tensor_shape.TensorShape([]), tensor_shape.TensorShape([])) elif self._message_key and self._message_offset: return ( - tensor_shape.TensorShape([]), - tensor_shape.TensorShape([]), - tensor_shape.TensorShape([]), + tensor_shape.TensorShape([]), + tensor_shape.TensorShape([]), + tensor_shape.TensorShape([]), ) return tensor_shape.TensorShape([]) @@ -137,4 +145,5 @@ def write_kafka_v2(message, topic, servers='localhost', name=None): Returns: A `Tensor` of type `string`. 0-D. """ - return gen_kafka_ops.io_write_kafka_v2(message=message, topic=topic, servers=servers, name=name) + return gen_kafka_ops.io_write_kafka_v2( + message=message, topic=topic, servers=servers, name=name) diff --git a/easy_rec/python/input/kafka_input.py b/easy_rec/python/input/kafka_input.py index 721654660..38eab121f 100644 --- a/easy_rec/python/input/kafka_input.py +++ b/easy_rec/python/input/kafka_input.py @@ -20,8 +20,8 @@ from kafka import KafkaConsumer, TopicPartition except ImportError: logging.warning( - 'kafka-python is not installed[%s]. You can install it by: pip install kafka-python' % traceback.format_exc() - ) + 'kafka-python is not installed[%s]. You can install it by: pip install kafka-python' + % traceback.format_exc()) if tf.__version__ >= '2.0': ignore_errors = tf.data.experimental.ignore_errors() @@ -31,35 +31,27 @@ class KafkaInput(Input): + DATA_OFFSET = 'DATA_OFFSET' - def __init__( - self, - data_config, - feature_config, - kafka_config, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(KafkaInput, self).__init__( - data_config, - feature_config, - '', - task_index, - task_num, - check_mode, - pipeline_config, - ) + def __init__(self, + data_config, + feature_config, + kafka_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None): + super(KafkaInput, + self).__init__(data_config, feature_config, '', task_index, task_num, + check_mode, pipeline_config) self._kafka = kafka_config self._offset_dict = {} if self._kafka is not None: consumer = KafkaConsumer( - group_id='kafka_dataset_consumer', - bootstrap_servers=[self._kafka.server], - api_version_auto_timeout_ms=60000, - ) # in miliseconds + group_id='kafka_dataset_consumer', + bootstrap_servers=[self._kafka.server], + api_version_auto_timeout_ms=60000) # in miliseconds partitions = consumer.partitions_for_topic(self._kafka.topic) self._num_partition = len(partitions) logging.info('all partitions[%d]: %s' % (self._num_partition, partitions)) @@ -69,7 +61,10 @@ def __init__( if offset_type is not None: if offset_type == 'offset_time': ts = parse_time(self._kafka.offset_time) - input_map = {TopicPartition(partition=part_id, topic=self._kafka.topic): ts * 1000 for part_id in partitions} + input_map = { + TopicPartition(partition=part_id, topic=self._kafka.topic): + ts * 1000 for part_id in partitions + } part_offsets = consumer.offsets_for_times(input_map) # part_offsets is a dictionary: # { @@ -79,15 +74,9 @@ def __init__( for part in part_offsets: self._offset_dict[part.partition] = part_offsets[part].offset logging.info( - ('Find offset by time, topic[%s], partition[%d], timestamp[%ss], ' 'offset[%d], offset_timestamp[%dms]') - % ( - self._kafka.topic, - part.partition, - ts, - part_offsets[part].offset, - part_offsets[part].timestamp, - ) - ) + 'Find offset by time, topic[%s], partition[%d], timestamp[%ss], offset[%d], offset_timestamp[%dms]' + % (self._kafka.topic, part.partition, ts, + part_offsets[part].offset, part_offsets[part].timestamp)) elif offset_type == 'offset_info': offset_dict = json.loads(self._kafka.offset_info) for part in offset_dict: @@ -112,16 +101,16 @@ def _preprocess(self, field_dict): def _parse_csv(self, line, message_key, message_offset): record_defaults = [ - self.get_type_defaults(t, v) for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) + for t, v in zip(self._input_field_types, self._input_field_defaults) ] fields = tf.decode_csv( - line, - use_quote_delim=False, - field_delim=self._data_config.separator, - record_defaults=record_defaults, - name='decode_csv', - ) + line, + use_quote_delim=False, + field_delim=self._data_config.separator, + record_defaults=record_defaults, + name='decode_csv') inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids} @@ -140,7 +129,8 @@ def _parse_offset(message_offset): self._task_offset_dict[k] = v return json.dumps(self._task_offset_dict) - inputs[Input.DATA_OFFSET] = tf.py_func(_parse_offset, [message_offset], tf.string) + inputs[Input.DATA_OFFSET] = tf.py_func(_parse_offset, [message_offset], + tf.string) return inputs def restore(self, checkpoint_path): @@ -176,10 +166,9 @@ def _get_topics(self): topics.append('%s:%d:%d' % (self._kafka.topic, part_id, offset)) self._task_offset_dict[part_id] = offset logging.info('assigned topic partitions: %s' % (','.join(topics))) - assert len(topics) > 0, 'no partitions are assigned for this task(%d/%d)' % ( - self._task_index, - self._task_num, - ) + assert len( + topics) > 0, 'no partitions are assigned for this task(%d/%d)' % ( + self._task_index, self._task_num) return topics def _build(self, mode, params): @@ -189,70 +178,58 @@ def _build(self, mode, params): assert self._kafka is not None, 'kafka_train_input is not set.' train_kafka = self._kafka logging.info( - 'train kafka server: %s topic: %s task_num: %d task_index: %d topics: %s' - % ( - train_kafka.server, - train_kafka.topic, - self._task_num, - self._task_index, - task_topics, - ) - ) + 'train kafka server: %s topic: %s task_num: %d task_index: %d topics: %s' + % (train_kafka.server, train_kafka.topic, self._task_num, + self._task_index, task_topics)) dataset = KafkaDataset( - task_topics, - servers=train_kafka.server, - group=train_kafka.group, - eof=False, - config_global=list(self._kafka.config_global), - config_topic=list(self._kafka.config_topic), - message_key=True, - message_offset=True, - ) + task_topics, + servers=train_kafka.server, + group=train_kafka.group, + eof=False, + config_global=list(self._kafka.config_global), + config_topic=list(self._kafka.config_topic), + message_key=True, + message_offset=True) if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, - ) + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True) else: eval_kafka = self._kafka assert self._kafka is not None, 'kafka_eval_input is not set.' logging.info( - 'eval kafka server: %s topic: %s task_num: %d task_index: %d topics: %s' - % ( - eval_kafka.server, - eval_kafka.topic, - self._task_num, - self._task_index, - task_topics, - ) - ) + 'eval kafka server: %s topic: %s task_num: %d task_index: %d topics: %s' + % (eval_kafka.server, eval_kafka.topic, self._task_num, + self._task_index, task_topics)) dataset = KafkaDataset( - task_topics, - servers=self._kafka.server, - group=eval_kafka.group, - eof=False, - config_global=list(self._kafka.config_global), - config_topic=list(self._kafka.config_topic), - message_key=True, - message_offset=True, - ) + task_topics, + servers=self._kafka.server, + group=eval_kafka.group, + eof=False, + config_global=list(self._kafka.config_global), + config_topic=list(self._kafka.config_topic), + message_key=True, + message_offset=True) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map(self._parse_csv, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + self._parse_csv, num_parallel_calls=num_parallel_calls) if self._data_config.ignore_error: dataset = dataset.apply(ignore_errors) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/load_parquet.py b/easy_rec/python/input/load_parquet.py index 71e7f6767..a798efb58 100644 --- a/easy_rec/python/input/load_parquet.py +++ b/easy_rec/python/input/load_parquet.py @@ -6,47 +6,31 @@ import pandas as pd -def start_data_proc( - task_index, - task_num, - num_proc, - file_que, - data_que, - proc_start_que, - proc_stop_que, - batch_size, - label_fields, - sparse_fea_names, - dense_fea_names, - dense_fea_cfgs, - reserve_fields, - drop_remainder, - need_pack=True, -): +def start_data_proc(task_index, + task_num, + num_proc, + file_que, + data_que, + proc_start_que, + proc_stop_que, + batch_size, + label_fields, + sparse_fea_names, + dense_fea_names, + dense_fea_cfgs, + reserve_fields, + drop_remainder, + need_pack=True): mp_ctxt = multiprocessing.get_context('spawn') proc_arr = [] for proc_id in range(num_proc): proc = mp_ctxt.Process( - target=load_data_proc, - args=( - proc_id, - file_que, - data_que, - proc_start_que, - proc_stop_que, - batch_size, - label_fields, - sparse_fea_names, - dense_fea_names, - dense_fea_cfgs, - reserve_fields, - drop_remainder, - task_index, - task_num, - need_pack, - ), - name='task_%d_data_proc_%d' % (task_index, proc_id), - ) + target=load_data_proc, + args=(proc_id, file_que, data_que, proc_start_que, proc_stop_que, + batch_size, label_fields, sparse_fea_names, dense_fea_names, + dense_fea_cfgs, reserve_fields, drop_remainder, task_index, + task_num, need_pack), + name='task_%d_data_proc_%d' % (task_index, proc_id)) proc.daemon = True proc.start() proc_arr.append(proc) @@ -117,27 +101,22 @@ def _pack_dense_feas(data_dict, dense_fea_names, dense_fea_cfgs): def _reshape_dense_feas(data_dict, dense_fea_names, dense_fea_cfgs): for fea_name, fea_cfg in zip(dense_fea_names, dense_fea_cfgs): - data_dict[fea_name] = data_dict[fea_name].reshape([-1, fea_cfg.raw_input_dim]) + data_dict[fea_name] = data_dict[fea_name].reshape( + [-1, fea_cfg.raw_input_dim]) def _load_dense(input_data, field_names, sid, eid, dense_dict): for k in field_names: if isinstance(input_data[k][0], np.ndarray): np_dtype = type(input_data[k][sid][0]) - dense_dict[k] = np.array([x[0] for x in input_data[k][sid:eid]], dtype=np_dtype) + dense_dict[k] = np.array([x[0] for x in input_data[k][sid:eid]], + dtype=np_dtype) else: dense_dict[k] = input_data[k][sid:eid].to_numpy() -def _load_and_pad_dense( - input_data, - field_names, - sid, - dense_dict, - part_dense_dict, - part_dense_dict_n, - batch_size, -): +def _load_and_pad_dense(input_data, field_names, sid, dense_dict, + part_dense_dict, part_dense_dict_n, batch_size): for k in field_names: if isinstance(input_data[k][0], np.ndarray): np_dtype = type(input_data[k][sid][0]) @@ -157,24 +136,12 @@ def _load_and_pad_dense( part_dense_dict_n[k] = tmp_lbls -def load_data_proc( - proc_id, - file_que, - data_que, - proc_start_que, - proc_stop_que, - batch_size, - label_fields, - sparse_fea_names, - dense_fea_names, - dense_fea_cfgs, - reserve_fields, - drop_remainder, - task_index, - task_num, - need_pack, -): - logging.info('data proc %d start, proc_start_que=%s' % (proc_id, proc_start_que.qsize())) +def load_data_proc(proc_id, file_que, data_que, proc_start_que, proc_stop_que, + batch_size, label_fields, sparse_fea_names, dense_fea_names, + dense_fea_cfgs, reserve_fields, drop_remainder, task_index, + task_num, need_pack): + logging.info('data proc %d start, proc_start_que=%s' % + (proc_id, proc_start_que.qsize())) proc_start_que.get() effective_fields = sparse_fea_names + dense_fea_names all_fields = effective_fields @@ -184,7 +151,8 @@ def load_data_proc( for tmp in reserve_fields: if tmp not in all_fields: all_fields.append(tmp) - logging.info('data proc %d start, file_que.qsize=%d' % (proc_id, file_que.qsize())) + logging.info('data proc %d start, file_que.qsize=%d' % + (proc_id, file_que.qsize())) num_files = 0 part_data_dict = {} @@ -226,10 +194,9 @@ def load_data_proc( else: all_lens = np.ones([len(val)], dtype=np.int32) all_vals = val.to_numpy() - assert np.sum(all_lens) == len(all_vals), 'len(all_vals)=%d np.sum(all_lens)=%d' % ( - len(all_vals), - np.sum(all_lens), - ) + assert np.sum(all_lens) == len( + all_vals), 'len(all_vals)=%d np.sum(all_lens)=%d' % ( + len(all_vals), np.sum(all_lens)) data_dict[k] = (all_lens, all_vals) if len(dense_fea_names) > 0: @@ -257,39 +224,19 @@ def load_data_proc( part_data_dict_n = {} if label_fields is not None and len(label_fields) > 0: - _load_and_pad_dense( - input_data, - label_fields, - sid, - data_dict, - part_data_dict, - part_data_dict_n, - batch_size, - ) + _load_and_pad_dense(input_data, label_fields, sid, data_dict, + part_data_dict, part_data_dict_n, batch_size) if reserve_fields is not None and len(reserve_fields) > 0: data_dict['reserve'] = {} part_data_dict_n['reserve'] = {} - _load_and_pad_dense( - input_data, - label_fields, - sid, - data_dict['reserve'], - part_data_dict['reserve'], - part_data_dict_n['reserve'], - batch_size, - ) + _load_and_pad_dense(input_data, label_fields, sid, data_dict['reserve'], + part_data_dict['reserve'], + part_data_dict_n['reserve'], batch_size) if len(dense_fea_names) > 0: - _load_and_pad_dense( - input_data, - dense_fea_names, - sid, - data_dict, - part_data_dict, - part_data_dict_n, - batch_size, - ) + _load_and_pad_dense(input_data, dense_fea_names, sid, data_dict, + part_data_dict, part_data_dict_n, batch_size) if len(sparse_fea_names) > 0: for k in sparse_fea_names: @@ -346,17 +293,18 @@ def load_data_proc( else: if len(dense_fea_names) > 0: _reshape_dense_feas(part_data_dict, dense_fea_names, dense_fea_cfgs) - logging.info('remainder batch: %s sample_num=%d' % (','.join(part_data_dict.keys()), batch_len)) + logging.info('remainder batch: %s sample_num=%d' % + (','.join(part_data_dict.keys()), batch_len)) _add_to_que(part_data_dict, data_que, proc_stop_que) total_batch_cnt += 1 else: - logging.warning('drop remain %d samples as drop_remainder is set' % batch_len) + logging.warning('drop remain %d samples as drop_remainder is set' % + batch_len) if is_good: is_good = _add_to_que(None, data_que, proc_stop_que) logging.info( - 'data_proc_id[%d]: is_good = %s, total_batch_cnt=%d, total_sample_cnt=%d' - % (proc_id, is_good, total_batch_cnt, total_sample_cnt) - ) + 'data_proc_id[%d]: is_good = %s, total_batch_cnt=%d, total_sample_cnt=%d' + % (proc_id, is_good, total_batch_cnt, total_sample_cnt)) data_que.close(wait_send_finish=is_good) while not is_good: diff --git a/easy_rec/python/input/odps_input.py b/easy_rec/python/input/odps_input.py index b33b6afa2..c3068deff 100644 --- a/easy_rec/python/input/odps_input.py +++ b/easy_rec/python/input/odps_input.py @@ -12,24 +12,25 @@ class OdpsInput(Input): + def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(OdpsInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(OdpsInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) def _build(self, mode, params): @@ -39,53 +40,56 @@ def _build(self, mode, params): selected_cols = ','.join(self._input_fields) if self._data_config.chief_redundant and mode == tf.estimator.ModeKeys.TRAIN: reader = tf.TableRecordReader( - csv_delimiter=self._data_config.separator, - selected_cols=selected_cols, - slice_count=max(self._task_num - 1, 1), - slice_id=max(self._task_index - 1, 0), + csv_delimiter=self._data_config.separator, + selected_cols=selected_cols, + slice_count=max(self._task_num - 1, 1), + slice_id=max(self._task_index - 1, 0), ) else: reader = tf.TableRecordReader( - csv_delimiter=self._data_config.separator, - selected_cols=selected_cols, - slice_count=self._task_num, - slice_id=self._task_index, + csv_delimiter=self._data_config.separator, + selected_cols=selected_cols, + slice_count=self._task_num, + slice_id=self._task_index, ) if type(self._input_path) != list: self._input_path = self._input_path.split(',') - assert len(self._input_path) > 0, 'match no files with %s' % self._input_path + assert len( + self._input_path) > 0, 'match no files with %s' % self._input_path if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.pai_worker_queue: work_queue = pai.data.WorkQueue( - self._input_path, - num_epochs=self.num_epochs, - shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num, + self._input_path, + num_epochs=self.num_epochs, + shuffle=self._data_config.shuffle, + num_slices=self._data_config.pai_worker_slice_num * self._task_num, ) work_queue.add_summary() file_queue = work_queue.input_producer() reader = tf.TableRecordReader() else: file_queue = tf.train.string_input_producer( - self._input_path, - num_epochs=self.num_epochs, - capacity=1000, - shuffle=self._data_config.shuffle, + self._input_path, + num_epochs=self.num_epochs, + capacity=1000, + shuffle=self._data_config.shuffle, ) else: - file_queue = tf.train.string_input_producer(self._input_path, num_epochs=1, capacity=1000, shuffle=False) + file_queue = tf.train.string_input_producer( + self._input_path, num_epochs=1, capacity=1000, shuffle=False) key, value = reader.read_up_to(file_queue, self._batch_size) record_defaults = [ - self.get_type_defaults(t, v) for t, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(t, v) + for t, v in zip(self._input_field_types, self._input_field_defaults) ] fields = tf.decode_csv( - value, - record_defaults=record_defaults, - field_delim=self._data_config.separator, - name='decode_csv', + value, + record_defaults=record_defaults, + field_delim=self._data_config.separator, + name='decode_csv', ) inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids} diff --git a/easy_rec/python/input/odps_input_v2.py b/easy_rec/python/input/odps_input_v2.py index c1ec5f515..9cba3d0e2 100644 --- a/easy_rec/python/input/odps_input_v2.py +++ b/easy_rec/python/input/odps_input_v2.py @@ -14,24 +14,25 @@ class OdpsInputV2(Input): + def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(OdpsInputV2, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(OdpsInputV2, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) def _parse_table(self, *fields): @@ -44,48 +45,54 @@ def _parse_table(self, *fields): def _build(self, mode, params): if type(self._input_path) != list: self._input_path = self._input_path.split(',') - assert len(self._input_path) > 0, 'match no files with %s' % self._input_path + assert len( + self._input_path) > 0, 'match no files with %s' % self._input_path # check data_config are consistent with odps tables odps_util.check_input_field_and_types(self._data_config) selected_cols = ','.join(self._input_fields) record_defaults = [ - self.get_type_defaults(x, v) for x, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(x, v) + for x, v in zip(self._input_field_types, self._input_field_defaults) ] if self._data_config.pai_worker_queue and mode == tf.estimator.ModeKeys.TRAIN: - logging.info('pai_worker_slice_num = %d' % self._data_config.pai_worker_slice_num) + logging.info('pai_worker_slice_num = %d' % + self._data_config.pai_worker_slice_num) work_queue = pai.data.WorkQueue( - self._input_path, - num_epochs=self.num_epochs, - shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num, + self._input_path, + num_epochs=self.num_epochs, + shuffle=self._data_config.shuffle, + num_slices=self._data_config.pai_worker_slice_num * self._task_num, ) que_paths = work_queue.input_dataset() - dataset = tf.data.TableRecordDataset(que_paths, record_defaults=record_defaults, selected_cols=selected_cols) + dataset = tf.data.TableRecordDataset( + que_paths, + record_defaults=record_defaults, + selected_cols=selected_cols) elif self._data_config.chief_redundant and mode == tf.estimator.ModeKeys.TRAIN: dataset = tf.data.TableRecordDataset( - self._input_path, - record_defaults=record_defaults, - selected_cols=selected_cols, - slice_id=max(self._task_index - 1, 0), - slice_count=max(self._task_num - 1, 1), + self._input_path, + record_defaults=record_defaults, + selected_cols=selected_cols, + slice_id=max(self._task_index - 1, 0), + slice_count=max(self._task_num - 1, 1), ) else: dataset = tf.data.TableRecordDataset( - self._input_path, - record_defaults=record_defaults, - selected_cols=selected_cols, - slice_id=self._task_index, - slice_count=self._task_num, + self._input_path, + record_defaults=record_defaults, + selected_cols=selected_cols, + slice_id=self._task_index, + slice_count=self._task_num, ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: @@ -93,19 +100,22 @@ def _build(self, mode, params): dataset = dataset.batch(batch_size=self._data_config.batch_size) - dataset = dataset.map(self._parse_table, num_parallel_calls=self._data_config.num_parallel_calls) + dataset = dataset.map( + self._parse_table, + num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls, + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/odps_input_v3.py b/easy_rec/python/input/odps_input_v3.py index 649a9f3c0..ba3f5389c 100644 --- a/easy_rec/python/input/odps_input_v3.py +++ b/easy_rec/python/input/odps_input_v3.py @@ -20,31 +20,30 @@ class OdpsInputV3(Input): """Common IO based interface, could run at local or on data science.""" def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(OdpsInputV3, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(OdpsInputV3, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) self._num_epoch = 0 if common_io is None: - logging.error( - """ + logging.error(""" please install common_io pip install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.2%2Btunnel-py2.py3-none-any.whl""" - ) + ) sys.exit(1) def _parse_table(self, *fields): @@ -59,30 +58,35 @@ def _odps_read(self): self._num_epoch += 1 if type(self._input_path) != list: self._input_path = self._input_path.split(',') - assert len(self._input_path) > 0, 'match no files with %s' % self._input_path + assert len( + self._input_path) > 0, 'match no files with %s' % self._input_path # check data_config are consistent with odps tables odps_util.check_input_field_and_types(self._data_config) record_defaults = [ - self.get_type_defaults(x, v) for x, v in zip(self._input_field_types, self._input_field_defaults) + self.get_type_defaults(x, v) + for x, v in zip(self._input_field_types, self._input_field_defaults) ] selected_cols = ','.join(self._input_fields) for table_path in self._input_path: reader = common_io.table.TableReader( - table_path, - selected_cols=selected_cols, - slice_id=self._task_index, - slice_count=self._task_num, + table_path, + selected_cols=selected_cols, + slice_id=self._task_index, + slice_count=self._task_num, ) total_records_num = reader.get_row_count() batch_num = int(total_records_num / self._data_config.batch_size) res_num = total_records_num - batch_num * self._data_config.batch_size - batch_defaults = [[x] * self._data_config.batch_size for x in record_defaults] + batch_defaults = [ + [x] * self._data_config.batch_size for x in record_defaults + ] for batch_id in range(batch_num): batch_data_np = [x.copy() for x in batch_defaults] - for row_id, one_data in enumerate(reader.read(self._data_config.batch_size)): + for row_id, one_data in enumerate( + reader.read(self._data_config.batch_size)): for col_id in range(len(record_defaults)): if one_data[col_id] not in ['', 'NULL', None]: batch_data_np[col_id][row_id] = one_data[col_id] @@ -105,31 +109,35 @@ def _build(self, mode, params): list_shapes = tuple(list_shapes) # read odps tables - dataset = tf.data.Dataset.from_generator(self._odps_read, output_types=list_type, output_shapes=list_shapes) + dataset = tf.data.Dataset.from_generator( + self._odps_read, output_types=list_type, output_shapes=list_shapes) if mode == tf.estimator.ModeKeys.TRAIN: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) - dataset = dataset.map(self._parse_table, num_parallel_calls=self._data_config.num_parallel_calls) + dataset = dataset.map( + self._parse_table, + num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls, + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/odps_rtp_input.py b/easy_rec/python/input/odps_rtp_input.py index 70d047221..bcf6d0a7e 100644 --- a/easy_rec/python/input/odps_rtp_input.py +++ b/easy_rec/python/input/odps_rtp_input.py @@ -30,25 +30,26 @@ class OdpsRTPInput(Input): """ def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(OdpsRTPInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(OdpsRTPInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) - logging.info('input_fields: %s label_fields: %s' % (','.join(self._input_fields), ','.join(self._label_fields))) + logging.info('input_fields: %s label_fields: %s' % + (','.join(self._input_fields), ','.join(self._label_fields))) def _parse_table(self, *fields): fields = list(fields) @@ -60,35 +61,40 @@ def _parse_table(self, *fields): cols = [c.strip() for c in selected_cols.split(',')] non_feature_cols = cols[:-1] # only for features, labels and sample_weight excluded - record_types = [t for x, t in zip(self._input_fields, self._input_field_types) if x not in non_feature_cols] + record_types = [ + t for x, t in zip(self._input_fields, self._input_field_types) + if x not in non_feature_cols + ] record_defaults = [ - self.get_type_defaults(t, v) - for x, t, v in zip(self._input_fields, self._input_field_types, self._input_field_defaults) - if x not in non_feature_cols + self.get_type_defaults(t, v) + for x, t, v in zip(self._input_fields, self._input_field_types, + self._input_field_defaults) + if x not in non_feature_cols ] feature_num = len(record_types) # assume that the last field is the generated feature column - print('field_delim = %s, feature_num = %d' % (self._data_config.separator, feature_num)) - logging.info('field_delim = %s, input_field_name = %d' % (self._data_config.separator, len(record_types))) + print('field_delim = %s, feature_num = %d' % + (self._data_config.separator, feature_num)) + logging.info('field_delim = %s, input_field_name = %d' % + (self._data_config.separator, len(record_types))) - check_list = ( - [ + check_list = ([ tf.py_func( - check_split, - [fields[-1], self._data_config.separator, len(record_types)], - Tout=tf.bool, + check_split, + [fields[-1], self._data_config.separator, + len(record_types)], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): - fields = str_split_by_chr(fields[-1], self._data_config.separator, skip_empty=False) + fields = str_split_by_chr( + fields[-1], self._data_config.separator, skip_empty=False) tmp_fields = tf.reshape(fields.values, [-1, feature_num]) - fields = labels[len(self._label_fields) :] + fields = labels[len(self._label_fields):] for i in range(feature_num): - field = string_to_number(tmp_fields[:, i], record_types[i], record_defaults[i], i) + field = string_to_number(tmp_fields[:, i], record_types[i], + record_defaults[i], i) fields.append(field) field_keys = [x for x in self._input_fields if x not in self._label_fields] @@ -97,77 +103,75 @@ def _parse_table(self, *fields): for x in range(len(self._label_fields)): inputs[self._label_fields[x]] = labels[x] - print('effective field num = %d, input_num = %d' % (len(fields), len(inputs))) + print('effective field num = %d, input_num = %d' % + (len(fields), len(inputs))) return inputs def _build(self, mode, params): if type(self._input_path) != list: self._input_path = self._input_path.split(',') - assert len(self._input_path) > 0, 'match no files with %s' % self._input_path + assert len( + self._input_path) > 0, 'match no files with %s' % self._input_path selected_cols = self._data_config.selected_cols if self._data_config.selected_cols else None if selected_cols: cols = [c.strip() for c in selected_cols.split(',')] record_defaults = [ - self.get_type_defaults(t, v) - for x, t, v in zip( - self._input_fields, - self._input_field_types, - self._input_field_defaults, - ) - if x in cols[:-1] + self.get_type_defaults(t, v) for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) if x in cols[:-1] ] - print('selected_cols: %s; defaults num: %d' % (','.join(cols), len(record_defaults))) + print('selected_cols: %s; defaults num: %d' % + (','.join(cols), len(record_defaults))) else: record_defaults = [ - self.get_type_defaults(t, v) - for x, t, v in zip( - self._input_fields, - self._input_field_types, - self._input_field_defaults, - ) - if x in self._label_fields + self.get_type_defaults(t, v) for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) if x in self._label_fields ] # the actual features are in one single column record_defaults.append( - self._data_config.separator.join( - [ - str(self.get_type_defaults(t, v)) - for x, t, v in zip( - self._input_fields, - self._input_field_types, - self._input_field_defaults, - ) - if x not in self._label_fields - ] - ) - ) + self._data_config.separator.join([ + str(self.get_type_defaults(t, v)) for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) if x not in self._label_fields + ])) if self._data_config.pai_worker_queue and mode == tf.estimator.ModeKeys.TRAIN: - logging.info('pai_worker_slice_num = %d' % self._data_config.pai_worker_slice_num) + logging.info('pai_worker_slice_num = %d' % + self._data_config.pai_worker_slice_num) work_queue = pai.data.WorkQueue( - self._input_path, - num_epochs=self.num_epochs, - shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num, + self._input_path, + num_epochs=self.num_epochs, + shuffle=self._data_config.shuffle, + num_slices=self._data_config.pai_worker_slice_num * self._task_num, ) que_paths = work_queue.input_dataset() - dataset = tf.data.TableRecordDataset(que_paths, record_defaults=record_defaults, selected_cols=selected_cols) + dataset = tf.data.TableRecordDataset( + que_paths, + record_defaults=record_defaults, + selected_cols=selected_cols) else: dataset = tf.data.TableRecordDataset( - self._input_path, - record_defaults=record_defaults, - selected_cols=selected_cols, - slice_id=self._task_index, - slice_count=self._task_num, + self._input_path, + record_defaults=record_defaults, + selected_cols=selected_cols, + slice_id=self._task_index, + slice_count=self._task_num, ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: @@ -175,19 +179,22 @@ def _build(self, mode, params): dataset = dataset.batch(batch_size=self._data_config.batch_size) - dataset = dataset.map(self._parse_table, num_parallel_calls=self._data_config.num_parallel_calls) + dataset = dataset.map( + self._parse_table, + num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls, + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/odps_rtp_input_v2.py b/easy_rec/python/input/odps_rtp_input_v2.py index ba988d9d1..9516084a0 100644 --- a/easy_rec/python/input/odps_rtp_input_v2.py +++ b/easy_rec/python/input/odps_rtp_input_v2.py @@ -32,24 +32,24 @@ class OdpsRTPInputV2(OdpsRTPInput): """ def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - fg_json_path=None, - pipeline_config=None, - ): - super(OdpsRTPInputV2, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + fg_json_path=None, + pipeline_config=None, + ): + super(OdpsRTPInputV2, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) if fg_json_path.startswith('!'): fg_json_path = fg_json_path[1:] @@ -94,11 +94,13 @@ def create_placeholders(self, *args, **kwargs): def create_multi_placeholders(self, *args, **kwargs): """Create serving multi-placeholders with rtp_fg.""" - raise NotImplementedError('create_multi_placeholders is not supported for OdpsRTPInputV2') + raise NotImplementedError( + 'create_multi_placeholders is not supported for OdpsRTPInputV2') def check_rtp(self): if rtp_fg is None: - raise NotImplementedError('OdpsRTPInputV2 cannot run without rtp_fg, which is not installed') + raise NotImplementedError( + 'OdpsRTPInputV2 cannot run without rtp_fg, which is not installed') def _pre_build(self, mode, params): try: diff --git a/easy_rec/python/input/parquet_input.py b/easy_rec/python/input/parquet_input.py index 027ac6c6e..0d6342772 100644 --- a/easy_rec/python/input/parquet_input.py +++ b/easy_rec/python/input/parquet_input.py @@ -17,26 +17,27 @@ class ParquetInput(Input): + def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs, - ): - super(ParquetInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, **kwargs, + ): + super(ParquetInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + **kwargs, ) self._need_pack = True if input_path is None: @@ -45,12 +46,15 @@ def __init__( self._input_files = [] for sub_path in input_path.strip().split(','): self._input_files.extend(tf.gfile.Glob(sub_path)) - logging.info('parquet input_path=%s file_num=%d' % (input_path, len(self._input_files))) + logging.info('parquet input_path=%s file_num=%d' % + (input_path, len(self._input_files))) mp_ctxt = multiprocessing.get_context('spawn') - self._data_que = queues.Queue(name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size) + self._data_que = queues.Queue( + name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size) file_num = len(self._input_files) - logging.info('[task_index=%d] total_file_num=%d task_num=%d' % (task_index, file_num, task_num)) + logging.info('[task_index=%d] total_file_num=%d task_num=%d' % + (task_index, file_num, task_num)) self._my_files = [] for file_id in range(file_num): @@ -58,7 +62,8 @@ def __init__( self._my_files.append(self._input_files[file_id]) # self._my_files = self._input_files - logging.info('[task_index=%d] task_file_num=%d' % (task_index, len(self._my_files))) + logging.info('[task_index=%d] task_file_num=%d' % + (task_index, len(self._my_files))) self._file_que = queues.Queue(name='file_que', ctx=mp_ctxt) self._num_proc = 8 @@ -103,7 +108,8 @@ def __init__( def _rebuild_que(self): mp_ctxt = multiprocessing.get_context('spawn') - self._data_que = queues.Queue(name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size) + self._data_que = queues.Queue( + name='data_que', ctx=mp_ctxt, maxsize=self._data_config.prefetch_size) self._file_que = queues.Queue(name='file_que', ctx=mp_ctxt) self._proc_start_que = queues.Queue(name='proc_start_que', ctx=mp_ctxt) self._proc_stop_que = queues.Queue(name='proc_stop_que', ctx=mp_ctxt) @@ -113,7 +119,8 @@ def _sample_generator(self): self._proc_start = True for proc in self._proc_arr: self._proc_start_que.put(True) - logging.info('task[%s] data_proc=%s is_alive=%s' % (self._task_index, proc, proc.is_alive())) + logging.info('task[%s] data_proc=%s is_alive=%s' % + (self._task_index, proc, proc.is_alive())) done_proc_cnt = 0 fetch_timeout_cnt = 0 @@ -144,28 +151,30 @@ def _sample_generator(self): yield sample if fetch_good_cnt % 200 == 0: logging.info( - 'task[%d] fetch_batch_cnt=%d, fetch_timeout_cnt=%d, qsize=%d' - % ( - self._task_index, - fetch_good_cnt, - fetch_timeout_cnt, - self._data_que.qsize(), - ) - ) + 'task[%d] fetch_batch_cnt=%d, fetch_timeout_cnt=%d, qsize=%d' % ( + self._task_index, + fetch_good_cnt, + fetch_timeout_cnt, + self._data_que.qsize(), + )) except queue.Empty: fetch_timeout_cnt += 1 if done_proc_cnt >= len(self._proc_arr): - logging.info('all sample finished, fetch_timeout_cnt=%d' % fetch_timeout_cnt) + logging.info('all sample finished, fetch_timeout_cnt=%d' % + fetch_timeout_cnt) break except Exception as ex: - logging.warning('task[%d] get from data_que exception: %s' % (self._task_index, str(ex))) + logging.warning('task[%d] get from data_que exception: %s' % + (self._task_index, str(ex))) break - logging.info('task[%d] sample_generator: total_batches=%d' % (self._task_index, fetch_good_cnt)) + logging.info('task[%d] sample_generator: total_batches=%d' % + (self._task_index, fetch_good_cnt)) def stop(self): if self._proc_arr is None or len(self._proc_arr) == 0: return - logging.info('task[%d] will stop dataset procs, proc_num=%d' % (self._task_index, len(self._proc_arr))) + logging.info('task[%d] will stop dataset procs, proc_num=%d' % + (self._task_index, len(self._proc_arr))) self._file_que.close() if self._proc_start: logging.info('try close data que') @@ -207,15 +216,15 @@ def _to_fea_dict(self, input_dict): if len(self._sparse_fea_names) > 0: if self._has_ev: tmp_vals, tmp_lens = ( - input_dict['sparse_fea'][1], - input_dict['sparse_fea'][0], + input_dict['sparse_fea'][1], + input_dict['sparse_fea'][0], ) fea_dict['sparse_fea'] = (tmp_vals, tmp_lens) else: tmp_vals, tmp_lens = ( - input_dict['sparse_fea'][1], - input_dict['sparse_fea'][0], + input_dict['sparse_fea'][1], + input_dict['sparse_fea'][0], ) num_buckets = -1 for fc in self._feature_configs: @@ -224,8 +233,8 @@ def _to_fea_dict(self, input_dict): num_buckets = fc.num_buckets else: assert num_buckets == fc.num_buckets, 'all features must share the same buckets, but are %d and %s' % ( - num_buckets, - str(fc), + num_buckets, + str(fc), ) fea_dict['sparse_fea'] = (tmp_vals % num_buckets, tmp_lens) @@ -253,14 +262,17 @@ def add_fea_type_and_shape(self, out_types, out_shapes): # second field: field values if len(self._sparse_fea_names) > 0: out_types['sparse_fea'] = (tf.int32, tf.int64) - out_shapes['sparse_fea'] = (tf.TensorShape([None]), tf.TensorShape([None])) + out_shapes['sparse_fea'] = (tf.TensorShape([None]), tf.TensorShape([None + ])) if len(self._dense_fea_names) > 0: out_types['dense_fea'] = tf.float32 - out_shapes['dense_fea'] = tf.TensorShape([None, self._total_dense_fea_dim]) + out_shapes['dense_fea'] = tf.TensorShape( + [None, self._total_dense_fea_dim]) def _build(self, mode, params): if mode == tf.estimator.ModeKeys.TRAIN and self._data_config.num_epochs > 1: - logging.info('will repeat train data for %d epochs' % self._data_config.num_epochs) + logging.info('will repeat train data for %d epochs' % + self._data_config.num_epochs) my_files = self._my_files * self._data_config.num_epochs else: my_files = self._my_files @@ -274,22 +286,22 @@ def _build(self, mode, params): lbl_fields = None drop_remainder = False self._proc_arr = load_parquet.start_data_proc( - self._task_index, - self._task_num, - self._num_proc, - self._file_que, - self._data_que, - self._proc_start_que, - self._proc_stop_que, - self._batch_size, - lbl_fields, - # self._effective_fields, - self._sparse_fea_names, - self._dense_fea_names, - self._dense_fea_cfgs, - self._reserve_fields, - drop_remainder, - need_pack=self._need_pack, + self._task_index, + self._task_num, + self._num_proc, + self._file_que, + self._data_que, + self._proc_start_que, + self._proc_stop_que, + self._batch_size, + lbl_fields, + # self._effective_fields, + self._sparse_fea_names, + self._dense_fea_names, + self._dense_fea_cfgs, + self._reserve_fields, + drop_remainder, + need_pack=self._need_pack, ) for input_file in my_files: @@ -298,7 +310,8 @@ def _build(self, mode, params): # add end signal for proc in self._proc_arr: self._file_que.put(None) - logging.info('add input_files to file_que, qsize=%d' % self._file_que.qsize()) + logging.info('add input_files to file_que, qsize=%d' % + self._file_que.qsize()) out_types = {} out_shapes = {} @@ -317,9 +330,13 @@ def _build(self, mode, params): self.add_fea_type_and_shape(out_types, out_shapes) - dataset = tf.data.Dataset.from_generator(self._sample_generator, output_types=out_types, output_shapes=out_shapes) + dataset = tf.data.Dataset.from_generator( + self._sample_generator, + output_types=out_types, + output_shapes=out_shapes) num_parallel_calls = self._data_config.num_parallel_calls - dataset = dataset.map(self._to_fea_dict, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + self._to_fea_dict, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) # Note: Input._preprocess is currently not supported as all features @@ -328,7 +345,8 @@ def _build(self, mode, params): # map_func=self._preprocess, num_parallel_calls=num_parallel_calls) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) # initial test show that prefetch to gpu has no performance gain # dataset = dataset.apply(tf.data.experimental.prefetch_to_device('/gpu:0')) else: @@ -341,16 +359,17 @@ def _build(self, mode, params): def _get_for_predictor(self, fea_dict): out_dict = { - 'feature': { - 'ragged_ids': fea_dict['feature']['sparse_fea'][0], - 'ragged_lens': fea_dict['feature']['sparse_fea'][1], - } + 'feature': { + 'ragged_ids': fea_dict['feature']['sparse_fea'][0], + 'ragged_lens': fea_dict['feature']['sparse_fea'][1], + } } if self._is_predictor and self._reserve_fields is not None: out_dict['reserve'] = fea_dict['reserve'] return out_dict def create_input(self, export_config=None): + def _input_fn(mode=None, params=None, config=None): """Build input_fn for estimator. @@ -367,9 +386,9 @@ def _input_fn(mode=None, params=None, config=None): tf.estimator.export.ServingInputReceiver instance """ if mode in ( - tf.estimator.ModeKeys.TRAIN, - tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT, + tf.estimator.ModeKeys.TRAIN, + tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT, ): # build dataset from self._config.input_path self._mode = mode @@ -378,21 +397,24 @@ def _input_fn(mode=None, params=None, config=None): elif mode is None: # serving_input_receiver_fn for export SavedModel inputs, features = {}, {} if len(self._sparse_fea_names) > 0: - ragged_ids = array_ops.placeholder(tf.int64, [None], name='ragged_ids') - ragged_lens = array_ops.placeholder(tf.int32, [None], name='ragged_lens') + ragged_ids = array_ops.placeholder( + tf.int64, [None], name='ragged_ids') + ragged_lens = array_ops.placeholder( + tf.int32, [None], name='ragged_lens') inputs = {'ragged_ids': ragged_ids, 'ragged_lens': ragged_lens} if self._has_ev: features = { - 'ragged_ids': ragged_ids, - 'ragged_lens': ragged_lens, + 'ragged_ids': ragged_ids, + 'ragged_lens': ragged_lens, } else: features = { - 'ragged_ids': ragged_ids % self._feature_configs[0].num_buckets, - 'ragged_lens': ragged_lens, + 'ragged_ids': ragged_ids % self._feature_configs[0].num_buckets, + 'ragged_lens': ragged_lens, } if len(self._dense_fea_names) > 0: - inputs['dense_fea'] = array_ops.placeholder(tf.float32, [None, self._total_dense_fea_dim], name='dense_fea') + inputs['dense_fea'] = array_ops.placeholder( + tf.float32, [None, self._total_dense_fea_dim], name='dense_fea') features['dense_fea'] = inputs['dense_fea'] return tf.estimator.export.ServingInputReceiver(features, inputs) diff --git a/easy_rec/python/input/parquet_input_v2.py b/easy_rec/python/input/parquet_input_v2.py index cf488dbb5..c9d963b11 100644 --- a/easy_rec/python/input/parquet_input_v2.py +++ b/easy_rec/python/input/parquet_input_v2.py @@ -6,11 +6,12 @@ # import numpy as np # import pandas as pd import tensorflow as tf -from tensorflow.python.framework import dtypes, ops - +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops # from tensorflow.python.ops import math_ops # from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops, string_ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import string_ops from easy_rec.python.input.parquet_input import ParquetInput from easy_rec.python.utils import conditional @@ -19,26 +20,27 @@ class ParquetInputV2(ParquetInput): + def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs, - ): - super(ParquetInputV2, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, **kwargs, + ): + super(ParquetInputV2, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + **kwargs, ) self._need_pack = False @@ -90,7 +92,8 @@ def _preprocess(self, inputs=None): features = {} placeholders = {} for fc in self._feature_configs: - feature_name = fc.feature_name if fc.feature_name != '' else fc.input_names[0] + feature_name = fc.feature_name if fc.feature_name != '' else fc.input_names[ + 0] feature_type = fc.feature_type if feature_type in [fc.IdFeature, fc.TagFeature]: input_name0 = fc.input_names[0] @@ -100,15 +103,18 @@ def _preprocess(self, inputs=None): if input_name0 in placeholders: input_lens, input_vals = placeholders[input_name0] else: - input_vals = array_ops.placeholder(dtypes.int64, [None], name=input_name0 + '/ids') - input_lens = array_ops.placeholder(dtypes.int64, [None], name=input_name0 + '/lens') + input_vals = array_ops.placeholder( + dtypes.int64, [None], name=input_name0 + '/ids') + input_lens = array_ops.placeholder( + dtypes.int64, [None], name=input_name0 + '/lens') placeholders[input_name0] = (input_lens, input_vals) if not self._has_ev: if fc.num_buckets > 0: input_vals = input_vals % fc.num_buckets else: input_vals = string_ops.as_string(input_vals) - features[feature_name] = tf.RaggedTensor.from_row_lengths(values=input_vals, row_lengths=input_lens) + features[feature_name] = tf.RaggedTensor.from_row_lengths( + values=input_vals, row_lengths=input_lens) elif feature_type in [fc.RawFeature]: input_name0 = fc.input_names[0] if inputs is not None: @@ -119,12 +125,13 @@ def _preprocess(self, inputs=None): else: if fc.raw_input_dim > 1: input_vals = array_ops.placeholder( - dtypes.float32, - [None, fc.raw_input_dim], - name=input_name0, + dtypes.float32, + [None, fc.raw_input_dim], + name=input_name0, ) else: - input_vals = array_ops.placeholder(dtypes.float32, [None], name=input_name0) + input_vals = array_ops.placeholder( + dtypes.float32, [None], name=input_name0) placeholders[input_name0] = input_vals features[feature_name] = input_vals else: @@ -152,6 +159,7 @@ def _get_for_predictor(self, fea_dict): return fea_dict def create_input(self, export_config=None): + def _input_fn(mode=None, params=None, config=None): """Build input_fn for estimator. @@ -168,9 +176,9 @@ def _input_fn(mode=None, params=None, config=None): tf.estimator.export.ServingInputReceiver instance """ if mode in ( - tf.estimator.ModeKeys.TRAIN, - tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT, + tf.estimator.ModeKeys.TRAIN, + tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT, ): # build dataset from self._config.input_path self._mode = mode diff --git a/easy_rec/python/input/parquet_input_v3.py b/easy_rec/python/input/parquet_input_v3.py index b18dac429..9431daed0 100644 --- a/easy_rec/python/input/parquet_input_v3.py +++ b/easy_rec/python/input/parquet_input_v3.py @@ -9,9 +9,7 @@ try: from tensorflow.python.data.experimental.ops import ( # NOQA - dataframe, - parquet_dataset_ops, - parquet_pybind, + dataframe, parquet_dataset_ops, parquet_pybind, ) from tensorflow.python.ops import gen_ragged_conversion_ops from tensorflow.python.ops.work_queue import WorkQueue @@ -25,38 +23,41 @@ class ParquetInputV3(Input): + def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs, + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + **kwargs, ): if not _has_deep_rec: raise RuntimeError('You should install DeepRec first.') super(ParquetInputV3, self).__init__( - data_config, - feature_config, - input_path, - task_index, - task_num, - check_mode, - pipeline_config, + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) self._ignore_val_dict = {} for f in data_config.input_fields: if f.HasField('ignore_val'): - self._ignore_val_dict[f.input_name] = get_type_defaults(f.input_type, f.ignore_val) + self._ignore_val_dict[f.input_name] = get_type_defaults( + f.input_type, f.ignore_val) self._true_type_dict = {} for fc in self._feature_configs: if fc.feature_type in [fc.IdFeature, fc.TagFeature, fc.SequenceFeature]: - if fc.hash_bucket_size > 0 or len(fc.vocab_list) > 0 or fc.HasField('vocab_file'): + if fc.hash_bucket_size > 0 or len( + fc.vocab_list) > 0 or fc.HasField('vocab_file'): self._true_type_dict[fc.input_names[0]] = tf.string else: self._true_type_dict[fc.input_names[0]] = tf.int64 @@ -80,16 +81,16 @@ def _ignore_and_cast(self, name, value): if isinstance(value, tf.SparseTensor): indices = tf.where(tf.equal(value.values, ignore_value)) value = tf.SparseTensor( - tf.gather_nd(value.indices, indices), - tf.gather_nd(value.values, indices), - value.dense_shape, + tf.gather_nd(value.indices, indices), + tf.gather_nd(value.values, indices), + value.dense_shape, ) elif isinstance(value, tf.Tensor): indices = tf.where(tf.not_equal(value, ignore_value), name='indices') value = tf.SparseTensor( - indices=indices, - values=tf.gather_nd(value, indices), - dense_shape=tf.shape(value, out_type=tf.int64), + indices=indices, + values=tf.gather_nd(value, indices), + dense_shape=tf.shape(value, out_type=tf.int64), ) dtype = self._true_type_dict.get(name, None) if dtype: @@ -100,11 +101,12 @@ def _parse_dataframe_value(self, value): if len(value.nested_row_splits) == 0: return value.values value.values.set_shape([None]) - sparse_value = gen_ragged_conversion_ops.ragged_tensor_to_sparse(value.nested_row_splits, value.values) + sparse_value = gen_ragged_conversion_ops.ragged_tensor_to_sparse( + value.nested_row_splits, value.values) return tf.SparseTensor( - sparse_value.sparse_indices, - sparse_value.sparse_values, - sparse_value.sparse_dense_shape, + sparse_value.sparse_indices, + sparse_value.sparse_values, + sparse_value.sparse_dense_shape, ) def _parse_dataframe(self, df): @@ -129,7 +131,8 @@ def _build(self, mode, params): for sub_path in self._input_path.strip().split(','): input_files.extend(tf.gfile.Glob(sub_path)) file_num = len(input_files) - logging.info('[task_index=%d] total_file_num=%d task_num=%d' % (self._task_index, file_num, self._task_num)) + logging.info('[task_index=%d] total_file_num=%d task_num=%d' % + (self._task_index, file_num, self._task_num)) task_index = self._task_index task_num = self._task_num @@ -139,9 +142,9 @@ def _build(self, mode, params): if self._data_config.pai_worker_queue and mode == tf.estimator.ModeKeys.TRAIN: work_queue = WorkQueue( - input_files, - num_epochs=self.num_epochs, - shuffle=self._data_config.shuffle, + input_files, + num_epochs=self.num_epochs, + shuffle=self._data_config.shuffle, ) my_files = work_queue.input_dataset() else: @@ -167,13 +170,14 @@ def _build(self, mode, params): if f.name in all_fields: selected_fields.append(f) - num_parallel_reads = min(self._data_config.num_parallel_calls, len(input_files) // task_num) + num_parallel_reads = min(self._data_config.num_parallel_calls, + len(input_files) // task_num) dataset = parquet_dataset_ops.ParquetDataset( - my_files, - batch_size=self._batch_size, - fields=selected_fields, - drop_remainder=self._data_config.drop_remainder, - num_parallel_reads=num_parallel_reads, + my_files, + batch_size=self._batch_size, + fields=selected_fields, + drop_remainder=self._data_config.drop_remainder, + num_parallel_reads=num_parallel_reads, ) # partition_count=task_num, # partition_index=task_index) @@ -181,30 +185,31 @@ def _build(self, mode, params): if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) dataset = dataset.map( - self._parse_dataframe, - num_parallel_calls=self._data_config.num_parallel_calls, + self._parse_dataframe, + num_parallel_calls=self._data_config.num_parallel_calls, ) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls, + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/rtp_input.py b/easy_rec/python/input/rtp_input.py index d6da5539a..1d8feb016 100644 --- a/easy_rec/python/input/rtp_input.py +++ b/easy_rec/python/input/rtp_input.py @@ -6,13 +6,13 @@ from easy_rec.python.input.input import Input from easy_rec.python.ops.gen_str_avx_op import str_split_by_chr -from easy_rec.python.utils.check_utils import ( # NOQA - check_split, - check_string_to_number, -) from easy_rec.python.utils.input_utils import string_to_number from easy_rec.python.utils.tf_utils import get_tf_type +from easy_rec.python.utils.check_utils import ( # NOQA + check_split, check_string_to_number, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -33,29 +33,32 @@ class RTPInput(Input): """ def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(RTPInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(RTPInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) - logging.info('input_fields: %s label_fields: %s' % (','.join(self._input_fields), ','.join(self._label_fields))) + logging.info('input_fields: %s label_fields: %s' % + (','.join(self._input_fields), ','.join(self._label_fields))) self._rtp_separator = self._data_config.rtp_separator if not isinstance(self._rtp_separator, str): self._rtp_separator = self._rtp_separator.encode('utf-8') - self._selected_cols = [int(x) for x in self._data_config.selected_cols.split(',')] + self._selected_cols = [ + int(x) for x in self._data_config.selected_cols.split(',') + ] self._num_cols = -1 self._feature_col_id = self._selected_cols[-1] logging.info('rtp separator = %s' % self._rtp_separator) @@ -64,29 +67,22 @@ def _parse_csv(self, line): record_defaults = ['' for i in range(self._num_cols)] # the actual features are in one single column - record_defaults[self._feature_col_id] = self._data_config.separator.join( - [ - str(self.get_type_defaults(t, v)) - for x, t, v in zip( - self._input_fields, - self._input_field_types, - self._input_field_defaults, - ) - if x not in self._label_fields - ] - ) + record_defaults[self._feature_col_id] = self._data_config.separator.join([ + str(self.get_type_defaults(t, v)) for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) if x not in self._label_fields + ]) - check_list = ( - [ + check_list = ([ tf.py_func( - check_split, - [line, self._rtp_separator, len(record_defaults)], - Tout=tf.bool, + check_split, + [line, self._rtp_separator, + len(record_defaults)], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): fields = tf.string_split(line, self._rtp_separator, skip_empty=False) @@ -99,38 +95,43 @@ def _parse_csv(self, line): ftype = self._input_field_types[idx] tf_type = get_tf_type(ftype) if field.dtype in [tf.string]: - check_list = [tf.py_func(check_string_to_number, [field, fname], Tout=tf.bool)] if self._check_mode else [] + check_list = [ + tf.py_func(check_string_to_number, [field, fname], Tout=tf.bool) + ] if self._check_mode else [] with tf.control_dependencies(check_list): field = tf.string_to_number(field, tf_type) labels.append(field) # only for features, labels excluded - record_types = [t for x, t in zip(self._input_fields, self._input_field_types) if x not in self._label_fields] + record_types = [ + t for x, t in zip(self._input_fields, self._input_field_types) + if x not in self._label_fields + ] # assume that the last field is the generated feature column print('field_delim = %s' % self._data_config.separator) feature_str = fields[:, self._feature_col_id] - check_list = ( - [ + check_list = ([ tf.py_func( - check_split, - [feature_str, self._data_config.separator, len(record_types)], - Tout=tf.bool, + check_split, + [feature_str, self._data_config.separator, + len(record_types)], + Tout=tf.bool, ) - ] - if self._check_mode - else [] - ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): - fields = str_split_by_chr(feature_str, self._data_config.separator, skip_empty=False) + fields = str_split_by_chr( + feature_str, self._data_config.separator, skip_empty=False) tmp_fields = tf.reshape(fields.values, [-1, len(record_types)]) rtp_record_defaults = [ - str(self.get_type_defaults(t, v)) - for x, t, v in zip(self._input_fields, self._input_field_types, self._input_field_defaults) - if x not in self._label_fields + str(self.get_type_defaults(t, v)) + for x, t, v in zip(self._input_fields, self._input_field_types, + self._input_field_defaults) + if x not in self._label_fields ] fields = [] for i in range(len(record_types)): - field = string_to_number(tmp_fields[:, i], record_types[i], rtp_record_defaults[i], i) + field = string_to_number(tmp_fields[:, i], record_types[i], + rtp_record_defaults[i], i) fields.append(field) field_keys = [x for x in self._input_fields if x not in self._label_fields] @@ -156,8 +157,9 @@ def _build(self, mode, params): line_tok = line_str.strip().split(self._rtp_separator) if self._num_cols != -1: assert self._num_cols == len(line_tok), ( - 'num selected cols is %d, not equal to %d, current line is: %s,' ' please check rtp_separator and data.' - ) % (self._num_cols, len(line_tok), line_str) + 'num selected cols is %d, not equal to %d, current line is: %s,' + ' please check rtp_separator and data.') % ( + self._num_cols, len(line_tok), line_str) self._num_cols = len(line_tok) num_lines += 1 if num_lines > 10: @@ -165,29 +167,26 @@ def _build(self, mode, params): logging.info('num selected cols = %d' % self._num_cols) record_defaults = [ - self.get_type_defaults(t, v) - for x, t, v in zip(self._input_fields, self._input_field_types, self._input_field_defaults) - if x in self._label_fields + self.get_type_defaults(t, v) + for x, t, v in zip(self._input_fields, self._input_field_types, + self._input_field_defaults) + if x in self._label_fields ] # the features are in one single column record_defaults.append( - self._data_config.separator.join( - [ - str(self.get_type_defaults(t, v)) - for x, t, v in zip( - self._input_fields, - self._input_field_types, - self._input_field_defaults, - ) - if x not in self._label_fields - ] - ) - ) + self._data_config.separator.join([ + str(self.get_type_defaults(t, v)) for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) if x not in self._label_fields + ])) num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('train files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -201,9 +200,9 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - tf.data.TextLineDataset, - cycle_length=parallel_num, - num_parallel_calls=parallel_num, + tf.data.TextLineDataset, + cycle_length=parallel_num, + num_parallel_calls=parallel_num, ) if not self._data_config.file_shard: @@ -211,31 +210,35 @@ def _build(self, mode, params): if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('eval files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(batch_size=self._data_config.batch_size) - dataset = dataset.map(self._parse_csv, num_parallel_calls=self._data_config.num_parallel_calls) + dataset = dataset.map( + self._parse_csv, + num_parallel_calls=self._data_config.num_parallel_calls) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( - map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls, + map_func=self._preprocess, + num_parallel_calls=self._data_config.num_parallel_calls, ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/rtp_input_v2.py b/easy_rec/python/input/rtp_input_v2.py index 3ad32f2ce..0987810b6 100644 --- a/easy_rec/python/input/rtp_input_v2.py +++ b/easy_rec/python/input/rtp_input_v2.py @@ -18,23 +18,23 @@ class RTPInputV2(Input): """ def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(RTPInputV2, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(RTPInputV2, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) def _parse_rtp(self, lines): @@ -48,40 +48,47 @@ def _parse_one_line_tf(line): keys = field_vals[:, 0] vals = field_vals[:, 1] temp_vals = [ - str(self.get_type_defaults(self._input_field_types[i], self._input_field_defaults[i])) - for i in range(len(self._input_fields)) + str( + self.get_type_defaults(self._input_field_types[i], + self._input_field_defaults[i])) + for i in range(len(self._input_fields)) ] for i, key in enumerate(self._input_fields): msk = tf.equal(key, keys) val = tf.boolean_mask(vals, msk) - def_val = self.get_type_defaults(self._input_field_types[i], self._input_field_defaults[i]) + def_val = self.get_type_defaults(self._input_field_types[i], + self._input_field_defaults[i]) temp_vals[i] = tf.cond( - tf.reduce_any(msk), - lambda: tf.reduce_join(val, separator=','), - lambda: tf.constant(str(def_val)), + tf.reduce_any(msk), + lambda: tf.reduce_join(val, separator=','), + lambda: tf.constant(str(def_val)), ) return temp_vals fields = tf.map_fn( - _parse_one_line_tf, - lines, - tf_types, - parallel_iterations=64, - name='parse_one_line_tf_map_fn', + _parse_one_line_tf, + lines, + tf_types, + parallel_iterations=64, + name='parse_one_line_tf_map_fn', ) def _convert(x, target_type, name): if target_type in [DatasetConfig.FLOAT, DatasetConfig.DOUBLE]: - return tf.string_to_number(x, tf.float32, name='convert_input_flt32/%s' % name) + return tf.string_to_number( + x, tf.float32, name='convert_input_flt32/%s' % name) elif target_type == DatasetConfig.INT32: - return tf.string_to_number(x, tf.int32, name='convert_input_int32/%s' % name) + return tf.string_to_number( + x, tf.int32, name='convert_input_int32/%s' % name) elif target_type == DatasetConfig.INT64: - return tf.string_to_number(x, tf.int64, name='convert_input_int64/%s' % name) + return tf.string_to_number( + x, tf.int64, name='convert_input_int64/%s' % name) return x inputs = { - self._input_fields[x]: _convert(fields[x], self._input_field_types[x], self._input_fields[x]) - for x in self._effective_fids + self._input_fields[x]: _convert(fields[x], self._input_field_types[x], + self._input_fields[x]) + for x in self._effective_fids } for x in self._label_fids: @@ -98,7 +105,8 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('train files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.file_shard: @@ -112,9 +120,9 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - tf.data.TextLineDataset, - cycle_length=parallel_num, - num_parallel_calls=parallel_num, + tf.data.TextLineDataset, + cycle_length=parallel_num, + num_parallel_calls=parallel_num, ) if not self._data_config.file_shard: @@ -122,25 +130,29 @@ def _build(self, mode, params): if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('eval files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.TextLineDataset(file_paths) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) - dataset = dataset.map(self._parse_rtp, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + self._parse_rtp, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/input/tfrecord_input.py b/easy_rec/python/input/tfrecord_input.py index 9d778fb99..ae7b9cf72 100644 --- a/easy_rec/python/input/tfrecord_input.py +++ b/easy_rec/python/input/tfrecord_input.py @@ -12,31 +12,34 @@ class TFRecordInput(Input): + def __init__( - self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - ): - super(TFRecordInput, self).__init__( + self, data_config, feature_config, input_path, - task_index, - task_num, - check_mode, - pipeline_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(TFRecordInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, ) self.feature_desc = {} - for x, t, d in zip(self._input_fields, self._input_field_types, self._input_field_defaults): + for x, t, d in zip(self._input_fields, self._input_field_types, + self._input_field_defaults): d = self.get_type_defaults(t, d) t = get_tf_type(t) - self.feature_desc[x] = tf.FixedLenFeature(dtype=t, shape=1, default_value=d) + self.feature_desc[x] = tf.FixedLenFeature( + dtype=t, shape=1, default_value=d) def _parse_tfrecord(self, example): try: @@ -56,7 +59,8 @@ def _build(self, mode, params): num_parallel_calls = self._data_config.num_parallel_calls data_compression_type = self._data_config.data_compression_type if mode == tf.estimator.ModeKeys.TRAIN: - logging.info('train files[%d]: %s' % (len(file_paths), ','.join(file_paths))) + logging.info('train files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) dataset = tf.data.Dataset.from_tensor_slices(file_paths) if self._data_config.shuffle: # shuffle input files @@ -65,32 +69,38 @@ def _build(self, mode, params): # as the same data will be read multiple times parallel_num = min(num_parallel_calls, len(file_paths)) dataset = dataset.interleave( - lambda x: tf.data.TFRecordDataset(x, compression_type=data_compression_type), - cycle_length=parallel_num, - num_parallel_calls=parallel_num, + lambda x: tf.data.TFRecordDataset( + x, compression_type=data_compression_type), + cycle_length=parallel_num, + num_parallel_calls=parallel_num, ) dataset = dataset.shard(self._task_num, self._task_index) if self._data_config.shuffle: dataset = dataset.shuffle( - self._data_config.shuffle_buffer_size, - seed=2020, - reshuffle_each_iteration=True, + self._data_config.shuffle_buffer_size, + seed=2020, + reshuffle_each_iteration=True, ) dataset = dataset.repeat(self.num_epochs) else: - logging.info('eval files[%d]: %s' % (len(file_paths), ','.join(file_paths))) - dataset = tf.data.TFRecordDataset(file_paths, compression_type=data_compression_type) + logging.info('eval files[%d]: %s' % + (len(file_paths), ','.join(file_paths))) + dataset = tf.data.TFRecordDataset( + file_paths, compression_type=data_compression_type) dataset = dataset.repeat(1) - dataset = dataset.map(self._parse_tfrecord, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + self._parse_tfrecord, num_parallel_calls=num_parallel_calls) dataset = dataset.batch(self._data_config.batch_size) dataset = dataset.prefetch(buffer_size=self._prefetch_size) - dataset = dataset.map(map_func=self._preprocess, num_parallel_calls=num_parallel_calls) + dataset = dataset.map( + map_func=self._preprocess, num_parallel_calls=num_parallel_calls) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: - dataset = dataset.map(lambda x: (self._get_features(x), self._get_labels(x))) + dataset = dataset.map(lambda x: + (self._get_features(x), self._get_labels(x))) else: dataset = dataset.map(lambda x: (self._get_features(x))) return dataset diff --git a/easy_rec/python/layers/backbone.py b/easy_rec/python/layers/backbone.py index 404d3fd0e..2e644f26c 100644 --- a/easy_rec/python/layers/backbone.py +++ b/easy_rec/python/layers/backbone.py @@ -7,7 +7,8 @@ from google.protobuf import struct_pb2 from easy_rec.python.layers.common_layers import EnhancedInputLayer -from easy_rec.python.layers.keras import MLP, EmbeddingLayer +from easy_rec.python.layers.keras import MLP +from easy_rec.python.layers.keras import EmbeddingLayer from easy_rec.python.layers.utils import Parameter from easy_rec.python.protos import backbone_pb2 from easy_rec.python.utils.dag import DAG @@ -64,7 +65,9 @@ def __init__(self, config, features, input_layer, l2_reg=None): one_input = block.inputs[0] name = one_input.WhichOneof('name') if name != 'feature_group_name': - raise KeyError('`feature_group_name` should be set for input layer: ' + block.name) + raise KeyError( + '`feature_group_name` should be set for input layer: ' + + block.name) group = one_input.feature_group_name if not input_layer.has_group(group): raise KeyError('invalid feature group name: ' + group) @@ -82,25 +85,27 @@ def __init__(self, config, features, input_layer, l2_reg=None): self._name_to_layer[block.name] = input_fn else: if layer == 'input_layer': - input_fn = EnhancedInputLayer(self._input_layer, self._features, group, reuse) + input_fn = EnhancedInputLayer(self._input_layer, self._features, + group, reuse) input_feature_groups[group] = input_fn elif layer == 'raw_input': input_fn = self._input_layer.get_raw_features(self._features, group) input_feature_groups[group] = input_fn else: # embedding_layer ( - inputs, - vocab, - weights, - ) = self._input_layer.get_bucketized_features(self._features, group) + inputs, + vocab, + weights, + ) = self._input_layer.get_bucketized_features( + self._features, group) block.embedding_layer.vocab_size = vocab params = Parameter.make_from_pb(block.embedding_layer) input_fn = EmbeddingLayer(params, block.name) input_feature_groups[group] = (inputs, vocab, weights) logging.info( - 'add an embedding layer %s with vocab size %d', - block.name, - vocab, + 'add an embedding layer %s with vocab size %d', + block.name, + vocab, ) self._name_to_layer[block.name] = input_fn else: @@ -170,21 +175,21 @@ def __init__(self, config, features, input_layer, l2_reg=None): num_pkg_input += 1 else: raise KeyError( - 'invalid input name `%s`, must be the name of either a feature group or an another block' % iname - ) + 'invalid input name `%s`, must be the name of either a feature group or an another block' + % iname) num_groups = len(input_feature_groups) assert num_pkg_input > 0 or num_groups > 0, 'there must be at least one input layer/feature group' if len(config.concat_blocks) == 0 and len(config.output_blocks) == 0: leaf = self._dag.all_leaves() logging.warning( - '%s has no `concat_blocks` or `output_blocks`, try to concat all leaf blocks: %s' - % (config.name, ','.join(leaf)) - ) + '%s has no `concat_blocks` or `output_blocks`, try to concat all leaf blocks: %s' + % (config.name, ','.join(leaf))) self._config.concat_blocks.extend(leaf) Package.__packages[self._config.name] = self - logging.info('%s layers: %s' % (config.name, ','.join(self._name_to_layer.keys()))) + logging.info('%s layers: %s' % + (config.name, ','.join(self._name_to_layer.keys()))) def define_layers(self, layer, layer_cnf, name, reuse): if layer == 'keras_layer': @@ -235,7 +240,8 @@ def block_input(self, config, block_outputs, training=None, **kwargs): pkg_input = block_outputs[pkg_input_name] else: if pkg_input_name not in Package.__packages: - raise KeyError('package name `%s` does not exists' % pkg_input_name) + raise KeyError('package name `%s` does not exists' % + pkg_input_name) inner_package = Package.__packages[pkg_input_name] pkg_input = inner_package(training) if input_node.HasField('package_input_fn'): @@ -320,7 +326,8 @@ def call(self, is_training, **kwargs): block_outputs[block] = input_fn([inputs, weights], is_training) else: with tf.name_scope(block + '_input'): - inputs = self.block_input(config, block_outputs, is_training, **kwargs) + inputs = self.block_input(config, block_outputs, is_training, + **kwargs) output = self.call_layer(inputs, config, block, is_training, **kwargs) block_outputs[block] = output @@ -351,7 +358,8 @@ def call(self, is_training, **kwargs): def load_keras_layer(self, layer_conf, name, reuse=None): layer_cls, customize = load_keras_layer(layer_conf.class_name) if layer_cls is None: - raise ValueError('Invalid keras layer class name: ' + layer_conf.class_name) + raise ValueError('Invalid keras layer class name: ' + + layer_conf.class_name) param_type = layer_conf.WhichOneof('params') if customize: @@ -388,12 +396,14 @@ def load_keras_layer(self, layer_conf, name, reuse=None): assert param_type == 'st_params', 'internal keras layer only support st_params' try: kwargs = convert_to_dict(layer_conf.st_params) - logging.info('call %s layer with params %r' % (layer_conf.class_name, kwargs)) + logging.info('call %s layer with params %r' % + (layer_conf.class_name, kwargs)) layer = layer_cls(name=name, **kwargs) except TypeError as e: logging.warning(e) args = map(format_value, layer_conf.st_params.values()) - logging.info('try to call %s layer with params %r' % (layer_conf.class_name, args)) + logging.info('try to call %s layer with params %r' % + (layer_conf.class_name, args)) layer = layer_cls(*args, name=name) return layer, customize @@ -543,7 +553,8 @@ def merge_inputs(inputs, axis=-1, msg=''): if any(map(lambda x: type(x) == list, inputs)): logging.warning('%s: try to merge inputs into list' % msg) - return reduce(lambda x, y: x + y, [e if type(e) == list else [e] for e in inputs]) + return reduce(lambda x, y: x + y, + [e if type(e) == list else [e] for e in inputs]) if axis != -1: logging.info('concat inputs %s axis=%d' % (msg, axis)) diff --git a/easy_rec/python/layers/capsule_layer.py b/easy_rec/python/layers/capsule_layer.py index 82cb0e46c..8519ae07a 100644 --- a/easy_rec/python/layers/capsule_layer.py +++ b/easy_rec/python/layers/capsule_layer.py @@ -10,6 +10,7 @@ class CapsuleLayer: + def __init__(self, capsule_config, is_training): # max_seq_len: max behaviour sequence length(history length) self._max_seq_len = capsule_config.max_seq_len @@ -35,13 +36,14 @@ def squash(self, inputs): input_norm = tf.reduce_sum(tf.square(inputs), keep_dims=True, axis=-1) input_norm_eps = tf.maximum(input_norm, 1e-8) scale_factor = ( - tf.pow(input_norm_eps / (1 + input_norm_eps), self._squash_pow) * self._scale_ratio / tf.sqrt(input_norm_eps) - ) + tf.pow(input_norm_eps / (1 + input_norm_eps), self._squash_pow) * + self._scale_ratio / tf.sqrt(input_norm_eps)) tf.summary.histogram('capsule/squash_scale_factor', scale_factor) return scale_factor * inputs def _build_capsule_simi(self, high_capsules, capsule_num): - high_capsule_mask = tf.sequence_mask(capsule_num, tf.shape(high_capsules)[1]) + high_capsule_mask = tf.sequence_mask(capsule_num, + tf.shape(high_capsules)[1]) high_capsules = high_capsules * tf.to_float(high_capsule_mask[:, :, None]) high_capsules = tf.nn.l2_normalize(high_capsules, axis=-1) sum_sqr = tf.square(tf.reduce_sum(high_capsules, axis=1)) @@ -52,7 +54,8 @@ def _build_capsule_simi(self, high_capsules, capsule_num): simi = tf.reduce_sum(simi, axis=1) / div is_multi = tf.to_float(capsule_num > 1) - avg_simi = tf.reduce_sum((simi + 1) * is_multi) / (2.0 * tf.reduce_sum(is_multi)) + avg_simi = tf.reduce_sum( + (simi + 1) * is_multi) / (2.0 * tf.reduce_sum(is_multi)) return avg_simi def __call__(self, seq_feas, seq_lens): @@ -67,16 +70,17 @@ def __call__(self, seq_feas, seq_lens): """ # pad or clip to max_seq_len seq_feas = tf.cond( - tf.greater(tf.shape(seq_feas)[1], self._max_seq_len), - lambda: seq_feas[:, : self._max_seq_len, :], - lambda: tf.cond( - tf.less(tf.shape(seq_feas)[1], self._max_seq_len), - lambda: tf.pad( - seq_feas, - [[0, 0], [0, self._max_seq_len - tf.shape(seq_feas)[1]], [0, 0]], + tf.greater(tf.shape(seq_feas)[1], self._max_seq_len), + lambda: seq_feas[:, :self._max_seq_len, :], + lambda: tf.cond( + tf.less(tf.shape(seq_feas)[1], self._max_seq_len), + lambda: tf.pad( + seq_feas, + [[0, 0], [0, self._max_seq_len - tf.shape(seq_feas)[1]], [0, 0] + ], + ), + lambda: seq_feas, ), - lambda: seq_feas, - ), ) seq_lens = tf.minimum(seq_lens, self._max_seq_len) @@ -84,17 +88,17 @@ def __call__(self, seq_feas, seq_lens): # max_seq_len x max_num_high_capsule(sh) if self._is_training: routing_logits = tf.truncated_normal( - [batch_size, self._max_seq_len, self._max_k], - stddev=self._routing_logits_stddev, + [batch_size, self._max_seq_len, self._max_k], + stddev=self._routing_logits_stddev, ) else: np.random.seed(28) routing_logits = tf.constant( - np.random.uniform( - high=self._routing_logits_stddev, - size=[self._max_seq_len, self._max_k], - ), - dtype=tf.float32, + np.random.uniform( + high=self._routing_logits_stddev, + size=[self._max_seq_len, self._max_k], + ), + dtype=tf.float32, ) routing_logits = tf.tile(routing_logits[None, :, :], [batch_size, 1, 1]) routing_logits = tf.stop_gradient(routing_logits) @@ -102,7 +106,8 @@ def __call__(self, seq_feas, seq_lens): low_fea_dim = seq_feas.get_shape()[-1] # map low capsule features to high capsule features: # low_fea_dim x high_dim(de) - bilinear_matrix = tf.get_variable(dtype=tf.float32, shape=[low_fea_dim, self._high_dim], name='capsule/S') + bilinear_matrix = tf.get_variable( + dtype=tf.float32, shape=[low_fea_dim, self._high_dim], name='capsule/S') # map sequence feature to high dimensional space seq_feas_high = tf.tensordot(seq_feas, bilinear_matrix, axes=1) seq_feas_high_stop = tf.stop_gradient(seq_feas_high) @@ -112,8 +117,12 @@ def __call__(self, seq_feas, seq_lens): logging.info('will use constant number of capsules: %d' % self._max_k) num_high_capsules = tf.zeros_like(seq_lens, dtype=tf.int32) + self._max_k else: - logging.info('will use log(seq_len) number of capsules, max_capsules: %d' % self._max_k) - num_high_capsules = tf.maximum(1, tf.minimum(self._max_k, tf.to_int32(tf.log(tf.to_float(seq_lens))))) + logging.info( + 'will use log(seq_len) number of capsules, max_capsules: %d' % + self._max_k) + num_high_capsules = tf.maximum( + 1, tf.minimum(self._max_k, + tf.to_int32(tf.log(tf.to_float(seq_lens))))) # batch_size x max_seq_len(bs) mask = tf.sequence_mask(seq_lens, self._max_seq_len) @@ -140,23 +149,26 @@ def __call__(self, seq_feas, seq_lens): # batch_size x max_k x high_dim(bse,bsh->bhe) high_capsules = tf.einsum( - 'bse, bsh->bhe', - seq_feas_high_stop if iter_id + 1 < self._num_iters else seq_feas_high, - routing_logits, + 'bse, bsh->bhe', + seq_feas_high_stop + if iter_id + 1 < self._num_iters else seq_feas_high, + routing_logits, ) if iter_id + 1 == self._num_iters: - capsule_simi = self._build_capsule_simi(high_capsules, num_high_capsules) + capsule_simi = self._build_capsule_simi(high_capsules, + num_high_capsules) tf.summary.scalar('caspule/simi_%d' % iter_id, capsule_simi) tf.summary.scalar( - 'capsule/before_squash', - tf.reduce_mean(tf.norm(high_capsules, axis=-1)), + 'capsule/before_squash', + tf.reduce_mean(tf.norm(high_capsules, axis=-1)), ) high_capsules = self.squash(high_capsules) tf.summary.scalar( - 'capsule/after_squash', - tf.reduce_mean(tf.norm(high_capsules, axis=-1)), + 'capsule/after_squash', + tf.reduce_mean(tf.norm(high_capsules, axis=-1)), ) - capsule_simi_final = self._build_capsule_simi(high_capsules, num_high_capsules) + capsule_simi_final = self._build_capsule_simi(high_capsules, + num_high_capsules) tf.summary.scalar('caspule/simi_final', capsule_simi_final) break @@ -167,10 +179,13 @@ def __call__(self, seq_feas, seq_lens): # batch_size x max_seq_len x max_k(bse, bhe->bsh) if self._routing_logits_scale > 0: if iter_id == 0: - logging.info('routing_logits_scale = %.2f' % self._routing_logits_scale) - routing_logits = tf.einsum('bse, bhe->bsh', seq_feas_high_norm, high_capsules) * self._routing_logits_scale + logging.info('routing_logits_scale = %.2f' % + self._routing_logits_scale) + routing_logits = tf.einsum('bse, bhe->bsh', seq_feas_high_norm, + high_capsules) * self._routing_logits_scale else: - routing_logits = tf.einsum('bse, bhe->bsh', seq_feas_high_stop, high_capsules) + routing_logits = tf.einsum('bse, bhe->bsh', seq_feas_high_stop, + high_capsules) # zero paddings high_capsule_mask = tf.sequence_mask(num_high_capsules, self._max_k) diff --git a/easy_rec/python/layers/cmbf.py b/easy_rec/python/layers/cmbf.py index 4ba8f886a..3faf1dca2 100644 --- a/easy_rec/python/layers/cmbf.py +++ b/easy_rec/python/layers/cmbf.py @@ -2,7 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import dnn, multihead_cross_attention +from easy_rec.python.layers import dnn +from easy_rec.python.layers import multihead_cross_attention from easy_rec.python.utils.shape_utils import get_shape_list if tf.__version__ >= '2.0': @@ -17,7 +18,8 @@ class CMBF(object): https://www.mdpi.com/1424-8220/21/16/5275 """ - def __init__(self, model_config, feature_configs, features, cmbf_config, input_layer): + def __init__(self, model_config, feature_configs, features, cmbf_config, + input_layer): self._model_config = cmbf_config has_feature = False @@ -31,7 +33,8 @@ def __init__(self, model_config, feature_configs, features, cmbf_config, input_l has_feature = True self._txt_seq_features = None if input_layer.has_group('text'): - self._txt_seq_features, _, _ = input_layer(features, 'text', is_combine=False) + self._txt_seq_features, _, _ = input_layer( + features, 'text', is_combine=False) has_feature = True self._other_features = None if input_layer.has_group('other'): # e.g. statistical feature @@ -48,17 +51,19 @@ def __init__(self, model_config, feature_configs, features, cmbf_config, input_l self._general_feature_num = len(fea_group.feature_names) general_feature_names = set(fea_group.feature_names) assert self._general_feature_num == len( - general_feature_names + general_feature_names ), 'there are duplicate features in `general` feature group' elif fea_group.group_name == 'image': self._img_feature_num = len(fea_group.feature_names) img_feature_names = set(fea_group.feature_names) - assert self._img_feature_num == len(img_feature_names), 'there are duplicate features in `image` feature group' + assert self._img_feature_num == len( + img_feature_names + ), 'there are duplicate features in `image` feature group' elif fea_group.group_name == 'text': txt_seq_feature_names = set(fea_group.feature_names) self._txt_feature_num = len(fea_group.feature_names) assert self._txt_feature_num == len( - txt_seq_feature_names + txt_seq_feature_names ), 'there are duplicate features in `text` feature group' max_seq_len = 0 @@ -77,38 +82,40 @@ def __init__(self, model_config, feature_configs, features, cmbf_config, input_l txt_fea_emb_dim_list.append(feature_config.embedding_dim) if feature_config.HasField('max_seq_len'): assert feature_config.max_seq_len > 0, ( - 'feature config `max_seq_len` must be greater than 0 for feature: ' + fea_name - ) + 'feature config `max_seq_len` must be greater than 0 for feature: ' + + fea_name) if feature_config.max_seq_len > max_seq_len: max_seq_len = feature_config.max_seq_len unique_dim_num = len(set(txt_fea_emb_dim_list)) assert ( - unique_dim_num <= 1 and len(txt_fea_emb_dim_list) == self._txt_feature_num + unique_dim_num <= 1 and + len(txt_fea_emb_dim_list) == self._txt_feature_num ), 'CMBF requires that all `text` feature dimensions must be consistent.' unique_dim_num = len(set(general_emb_dim_list)) assert ( - unique_dim_num <= 1 and len(general_emb_dim_list) == self._general_feature_num + unique_dim_num <= 1 and + len(general_emb_dim_list) == self._general_feature_num ), 'CMBF requires that all `general` feature dimensions must be consistent.' unique_dim_num = len(set(img_fea_emb_dim_list)) assert ( - unique_dim_num <= 1 and len(img_fea_emb_dim_list) == self._img_feature_num + unique_dim_num <= 1 and + len(img_fea_emb_dim_list) == self._img_feature_num ), 'CMBF requires that all `image` feature dimensions must be consistent.' if cmbf_config.use_position_embeddings: assert cmbf_config.max_position_embeddings > 0, ( - 'model config `max_position_embeddings` must be greater than 0. ' - 'It must be set when `use_position_embeddings` is true (default)' - ) + 'model config `max_position_embeddings` must be greater than 0. ' + 'It must be set when `use_position_embeddings` is true (default)') assert cmbf_config.max_position_embeddings >= max_seq_len, ( - 'model config `max_position_embeddings` must be greater than' - ' or equal to the maximum of all feature config ' - '`max_seq_len`, which is %d' - ) % max_seq_len + 'model config `max_position_embeddings` must be greater than' + ' or equal to the maximum of all feature config ' + '`max_seq_len`, which is %d') % max_seq_len self._img_emb_size = img_fea_emb_dim_list[0] if img_fea_emb_dim_list else 0 self._txt_emb_size = txt_fea_emb_dim_list[0] if txt_fea_emb_dim_list else 0 - self._general_emb_size = general_emb_dim_list[0] if general_emb_dim_list else 0 + self._general_emb_size = general_emb_dim_list[ + 0] if general_emb_dim_list else 0 self._head_num = cmbf_config.multi_head_num self._img_head_num = cmbf_config.image_multi_head_num self._txt_head_num = cmbf_config.text_multi_head_num @@ -118,14 +125,14 @@ def __init__(self, model_config, feature_configs, features, cmbf_config, input_l self._img_self_attention_layer_num = cmbf_config.image_self_attention_layer_num self._txt_self_attention_layer_num = cmbf_config.text_self_attention_layer_num self._cross_modal_layer_num = cmbf_config.cross_modal_layer_num - print( - 'txt_feature_num: {0}, img_feature_num: {1}, txt_seq_feature_num: {2}'.format( - self._general_feature_num, - self._img_feature_num, - len(self._txt_seq_features) if self._txt_seq_features else 0, - ) - ) - print('txt_embedding_size: {0}, img_embedding_size: {1}'.format(self._txt_emb_size, self._img_emb_size)) + print('txt_feature_num: {0}, img_feature_num: {1}, txt_seq_feature_num: {2}' + .format( + self._general_feature_num, + self._img_feature_num, + len(self._txt_seq_features) if self._txt_seq_features else 0, + )) + print('txt_embedding_size: {0}, img_embedding_size: {1}'.format( + self._txt_emb_size, self._img_emb_size)) if self._img_features is not None: assert self._img_emb_size > 0, '`image` feature dimensions must be greater than 0, set by `raw_input_dim`' @@ -147,48 +154,60 @@ def image_self_attention_tower(self): hidden_size = self._model_config.multi_head_num * self._model_config.image_cross_head_size if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` - image_features = tf.reshape(self._img_features, shape=[-1, self._img_emb_size]) - image_features = tf.layers.dense(image_features, hidden_size, name='img_projection') - image_features = tf.reshape(image_features, shape=[-1, img_fea_num, hidden_size]) + image_features = tf.reshape( + self._img_features, shape=[-1, self._img_emb_size]) + image_features = tf.layers.dense( + image_features, hidden_size, name='img_projection') + image_features = tf.reshape( + image_features, shape=[-1, img_fea_num, hidden_size]) return image_features hidden_size = self._img_head_size * self._img_head_num if img_fea_num > 1: # in case of video frames or ROIs (Region Of Interest) if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` - image_features = tf.reshape(self._img_features, shape=[-1, self._img_emb_size]) - image_features = tf.layers.dense(image_features, hidden_size, name='img_projection') - image_features = tf.reshape(image_features, shape=[-1, img_fea_num, hidden_size]) + image_features = tf.reshape( + self._img_features, shape=[-1, self._img_emb_size]) + image_features = tf.layers.dense( + image_features, hidden_size, name='img_projection') + image_features = tf.reshape( + image_features, shape=[-1, img_fea_num, hidden_size]) elif img_fea_num == 1: if self._img_patch_num > 1: # image feature dimension: patch_num * emb_size img_fea_num = self._img_patch_num img_emb_size = self._img_emb_size // self._img_patch_num assert ( - img_emb_size * self._img_patch_num == self._img_emb_size + img_emb_size * self._img_patch_num == self._img_emb_size ), 'image feature dimension must equal to `image_feature_slice_num * embedding_size_per_region`' self._img_emb_size = img_emb_size if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` - image_features = tf.reshape(self._img_features, shape=[-1, self._img_emb_size]) - image_features = tf.layers.dense(image_features, hidden_size, name='img_projection') - image_features = tf.reshape(image_features, shape=[-1, img_fea_num, hidden_size]) + image_features = tf.reshape( + self._img_features, shape=[-1, self._img_emb_size]) + image_features = tf.layers.dense( + image_features, hidden_size, name='img_projection') + image_features = tf.reshape( + image_features, shape=[-1, img_fea_num, hidden_size]) else: img_fea_num = self._model_config.image_feature_dim if img_fea_num != self._img_emb_size: - image_features = tf.layers.dense(image_features, img_fea_num, name='img_projection') + image_features = tf.layers.dense( + image_features, img_fea_num, name='img_projection') # convert each element of image feature to a feature vector - img_mapping_matrix = tf.get_variable('img_map_matrix', [1, img_fea_num, hidden_size], dtype=tf.float32) + img_mapping_matrix = tf.get_variable( + 'img_map_matrix', [1, img_fea_num, hidden_size], dtype=tf.float32) image_features = tf.expand_dims(image_features, -1) * img_mapping_matrix img_attention_fea = multihead_cross_attention.transformer_encoder( - image_features, - hidden_size=hidden_size, # head_num * size_per_head - num_hidden_layers=self._img_self_attention_layer_num, - num_attention_heads=self._head_num, - intermediate_size=hidden_size * 4, - hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config.attention_probs_dropout_prob, - name='image_self_attention', + image_features, + hidden_size=hidden_size, # head_num * size_per_head + num_hidden_layers=self._img_self_attention_layer_num, + num_attention_heads=self._head_num, + intermediate_size=hidden_size * 4, + hidden_dropout_prob=self._model_config.hidden_dropout_prob, + attention_probs_dropout_prob=self._model_config + .attention_probs_dropout_prob, + name='image_self_attention', ) # shape: [batch_size, image_seq_num/image_feature_dim, hidden_size] # print('img_attention_fea:', img_attention_fea.shape) return img_attention_fea @@ -203,13 +222,18 @@ def text_self_attention_tower(self): general_features = self._general_features if self._general_emb_size != hidden_size: # Run a linear projection of `hidden_size` - general_features = tf.reshape(general_features, shape=[-1, self._general_emb_size]) - general_features = tf.layers.dense(general_features, hidden_size, name='txt_projection') - txt_features = tf.reshape(general_features, shape=[-1, self._general_feature_num, hidden_size]) + general_features = tf.reshape( + general_features, shape=[-1, self._general_emb_size]) + general_features = tf.layers.dense( + general_features, hidden_size, name='txt_projection') + txt_features = tf.reshape( + general_features, shape=[-1, self._general_feature_num, hidden_size]) all_txt_features.append(txt_features) batch_size = tf.shape(txt_features)[0] - mask = tf.ones(shape=tf.stack([batch_size, self._general_feature_num]), dtype=tf.int32) + mask = tf.ones( + shape=tf.stack([batch_size, self._general_feature_num]), + dtype=tf.int32) input_masks.append(mask) input_mask = None @@ -226,47 +250,49 @@ def dynamic_mask(x, max_len): batch_size, max_seq_len, emb_size = get_shape_list(seq_fea, 3) if emb_size != hidden_size: seq_fea = tf.reshape(seq_fea, shape=[-1, emb_size]) - seq_fea = tf.layers.dense(seq_fea, hidden_size, name='txt_seq_projection_%d' % i) + seq_fea = tf.layers.dense( + seq_fea, hidden_size, name='txt_seq_projection_%d' % i) seq_fea = tf.reshape(seq_fea, shape=[-1, max_seq_len, hidden_size]) seq_fea = multihead_cross_attention.embedding_postprocessor( - seq_fea, - use_token_type=self._model_config.use_token_type, - token_type_ids=tf.ones(shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * i, - token_type_vocab_size=token_type_vocab_size, - reuse_token_type=tf.AUTO_REUSE, - use_position_embeddings=self._model_config.use_position_embeddings, - max_position_embeddings=self._model_config.max_position_embeddings, - position_embedding_name='position_embeddings_%d' % i, - dropout_prob=self._model_config.text_seq_emb_dropout_prob, + seq_fea, + use_token_type=self._model_config.use_token_type, + token_type_ids=tf.ones( + shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * i, + token_type_vocab_size=token_type_vocab_size, + reuse_token_type=tf.AUTO_REUSE, + use_position_embeddings=self._model_config.use_position_embeddings, + max_position_embeddings=self._model_config.max_position_embeddings, + position_embedding_name='position_embeddings_%d' % i, + dropout_prob=self._model_config.text_seq_emb_dropout_prob, ) all_txt_features.append(seq_fea) input_mask = tf.map_fn( - fn=lambda t: dynamic_mask(t, max_seq_len), - elems=tf.to_int32(seq_len), + fn=lambda t: dynamic_mask(t, max_seq_len), + elems=tf.to_int32(seq_len), ) input_masks.append(input_mask) txt_features = tf.concat(all_txt_features, axis=1) input_mask = tf.concat(input_masks, axis=1) attention_mask = multihead_cross_attention.create_attention_mask_from_input_mask( - from_tensor=txt_features, to_mask=input_mask - ) + from_tensor=txt_features, to_mask=input_mask) if txt_features is None: return None, None, None txt_attention_fea = multihead_cross_attention.transformer_encoder( - txt_features, - hidden_size=hidden_size, - num_hidden_layers=self._txt_self_attention_layer_num, - num_attention_heads=self._head_num, - attention_mask=attention_mask, - intermediate_size=hidden_size * 4, - hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config.attention_probs_dropout_prob, - name='text_self_attention', + txt_features, + hidden_size=hidden_size, + num_hidden_layers=self._txt_self_attention_layer_num, + num_attention_heads=self._head_num, + attention_mask=attention_mask, + intermediate_size=hidden_size * 4, + hidden_dropout_prob=self._model_config.hidden_dropout_prob, + attention_probs_dropout_prob=self._model_config + .attention_probs_dropout_prob, + name='text_self_attention', ) # shape: [batch_size, txt_seq_length, hidden_size] print('txt_attention_fea:', txt_attention_fea.shape) return txt_attention_fea, input_mask, input_masks @@ -279,21 +305,25 @@ def merge_text_embedding(self, txt_embeddings, input_masks): text_seq_emb = [] if self._general_feature_num > 0: text_emb = tf.slice( - txt_embeddings, - [0, 0, 0], - [shape[0], self._general_feature_num, shape[2]], + txt_embeddings, + [0, 0, 0], + [shape[0], self._general_feature_num, shape[2]], ) text_seq_emb.append(text_emb) begin = self._general_feature_num for i in range(len(text_seq_emb), len(input_masks)): size = tf.shape(input_masks[i])[1] - temp_emb = tf.slice(txt_embeddings, [0, begin, 0], [shape[0], size, shape[2]]) + temp_emb = tf.slice(txt_embeddings, [0, begin, 0], + [shape[0], size, shape[2]]) mask = tf.expand_dims(tf.to_float(input_masks[i]), -1) temp_emb = temp_emb * mask # avg pooling - emb_sum = tf.reduce_sum(temp_emb, axis=1, keepdims=True) # shape: [batch_size, 1, hidden_size] - count = tf.reduce_sum(mask, axis=1, keepdims=True) # shape: [batch_size, 1, 1] + emb_sum = tf.reduce_sum( + temp_emb, axis=1, + keepdims=True) # shape: [batch_size, 1, hidden_size] + count = tf.reduce_sum( + mask, axis=1, keepdims=True) # shape: [batch_size, 1, 1] seq_emb = emb_sum / count # shape: [batch_size, 1, hidden_size] text_seq_emb.append(seq_emb) @@ -316,25 +346,29 @@ def __call__(self, is_training, *args, **kwargs): img_attention_fea = self.image_self_attention_tower() # shape: [batch_size, txt_seq_length, hidden_size] - txt_attention_fea, input_mask, input_masks = self.text_self_attention_tower() + txt_attention_fea, input_mask, input_masks = self.text_self_attention_tower( + ) all_fea = [] if None not in [img_attention_fea, txt_attention_fea]: ( - img_embeddings, - txt_embeddings, + img_embeddings, + txt_embeddings, ) = multihead_cross_attention.cross_attention_tower( - img_attention_fea, - txt_attention_fea, - num_hidden_layers=self._cross_modal_layer_num, - num_attention_heads=self._head_num, - right_input_mask=input_mask, - left_size_per_head=self._model_config.image_cross_head_size, - left_intermediate_size=4 * self._model_config.image_cross_head_size * self._head_num, - right_size_per_head=self._model_config.text_cross_head_size, - right_intermediate_size=4 * self._model_config.text_cross_head_size * self._head_num, - hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config.attention_probs_dropout_prob, + img_attention_fea, + txt_attention_fea, + num_hidden_layers=self._cross_modal_layer_num, + num_attention_heads=self._head_num, + right_input_mask=input_mask, + left_size_per_head=self._model_config.image_cross_head_size, + left_intermediate_size=4 * self._model_config.image_cross_head_size * + self._head_num, + right_size_per_head=self._model_config.text_cross_head_size, + right_intermediate_size=4 * self._model_config.text_cross_head_size * + self._head_num, + hidden_dropout_prob=self._model_config.hidden_dropout_prob, + attention_probs_dropout_prob=self._model_config + .attention_probs_dropout_prob, ) # img_emb shape: [batch_size, image_(region_)num/image_feature_dim, multi_head_num * image_cross_head_size] print('img_embeddings:', img_embeddings.shape) @@ -362,10 +396,10 @@ def __call__(self, is_training, *args, **kwargs): if self._model_config.HasField('other_feature_dnn'): l2_reg = kwargs['l2_reg'] if 'l2_reg' in kwargs else 0 other_dnn_layer = dnn.DNN( - self._model_config.other_feature_dnn, - l2_reg, - 'other_dnn', - is_training, + self._model_config.other_feature_dnn, + l2_reg, + 'other_dnn', + is_training, ) other_fea = other_dnn_layer(self._other_features) all_fea.append(other_fea) # e.g. statistical features diff --git a/easy_rec/python/layers/common_layers.py b/easy_rec/python/layers/common_layers.py index 1138298da..dfe57ddfc 100644 --- a/easy_rec/python/layers/common_layers.py +++ b/easy_rec/python/layers/common_layers.py @@ -12,14 +12,14 @@ def highway( - x, - size=None, - activation=None, - num_layers=1, - scope='highway', - dropout=0.0, - init_gate_bias=-1.0, - reuse=None, + x, + size=None, + activation=None, + num_layers=1, + scope='highway', + dropout=0.0, + init_gate_bias=-1.0, + reuse=None, ): if isinstance(activation, six.string_types): activation = get_activation(activation) @@ -32,14 +32,15 @@ def highway( initializer = tf.constant_initializer(init_gate_bias) for i in range(num_layers): T = tf.layers.dense( - x, - size, - activation=tf.sigmoid, - bias_initializer=initializer, - name='gate_%d' % i, - reuse=reuse, + x, + size, + activation=tf.sigmoid, + bias_initializer=initializer, + name='gate_%d' % i, + reuse=reuse, ) - H = tf.layers.dense(x, size, activation=activation, name='activation_%d' % i, reuse=reuse) + H = tf.layers.dense( + x, size, activation=activation, name='activation_%d' % i, reuse=reuse) if dropout > 0.0: H = tf.nn.dropout(H, 1.0 - dropout) x = H * T + x * (1.0 - T) @@ -47,11 +48,11 @@ def highway( def text_cnn( - x, - filter_sizes=(3, 4, 5), - num_filters=(128, 64, 64), - scope_name='textcnn', - reuse=False, + x, + filter_sizes=(3, 4, 5), + num_filters=(128, 64, 64), + scope_name='textcnn', + reuse=False, ): # x: None * step_dim * embed_dim assert len(filter_sizes) == len(num_filters) @@ -64,29 +65,31 @@ def text_cnn( with tf.variable_scope(scope_name_i, reuse=reuse): # conv shape: (batch_size, seq_len - filter_size + 1, num_filters) conv = tf.layers.conv1d( - x, - filters=int(num_filter), - kernel_size=int(filter_size), - activation=tf.nn.relu, - name='conv_layer', - reuse=reuse, - kernel_initializer=initializer, - padding='same', + x, + filters=int(num_filter), + kernel_size=int(filter_size), + activation=tf.nn.relu, + name='conv_layer', + reuse=reuse, + kernel_initializer=initializer, + padding='same', ) - pool = tf.reduce_max(conv, axis=1) # max pooling, shape: (batch_size, num_filters) + pool = tf.reduce_max( + conv, axis=1) # max pooling, shape: (batch_size, num_filters) pooled_outputs.append(pool) - pool_flat = tf.concat(pooled_outputs, 1) # shape: (batch_size, num_filters * len(filter_sizes)) + pool_flat = tf.concat( + pooled_outputs, 1) # shape: (batch_size, num_filters * len(filter_sizes)) return pool_flat def layer_norm(input_tensor, name=None, reuse=None): """Run layer normalization on the last dimension of the tensor.""" return tf_layer_norm( - inputs=input_tensor, - begin_norm_axis=-1, - begin_params_axis=-1, - reuse=reuse, - scope=name, + inputs=input_tensor, + begin_norm_axis=-1, + begin_params_axis=-1, + reuse=reuse, + scope=name, ) @@ -109,14 +112,17 @@ def __call__(self, config, is_training, **kwargs): return self.inputs if config.do_batch_norm and config.do_layer_norm: - raise ValueError('can not do batch norm and layer norm for input layer at the same time') + raise ValueError( + 'can not do batch norm and layer norm for input layer at the same time' + ) with tf.name_scope(self.name): return self.call(config, is_training) def build(self, config, training): self.built = True combine = not config.output_seq_and_normal_feature - self.inputs = self._input_layer(self._feature_dict, self._group_name, is_combine=combine) + self.inputs = self._input_layer( + self._feature_dict, self._group_name, is_combine=combine) if config.output_seq_and_normal_feature: seq_feature_and_len, _, target_features = self.inputs seq_len = seq_feature_and_len[0][1] @@ -126,7 +132,8 @@ def build(self, config, training): target_features = tf.concat(target_features, axis=-1) else: target_features = None - assert len(seq_features) > 0, '[%s] sequence feature is empty' % self.name + assert len( + seq_features) > 0, '[%s] sequence feature is empty' % self.name seq_features = tf.concat(seq_features, axis=-1) self.inputs = seq_features, seq_len, target_features self.reset(config, training) @@ -150,9 +157,11 @@ def call(self, config, training): keep_prob = 1.0 - config.feature_dropout_rate mask = self.bern.sample(num_features) elif do_bn: - features = tf.layers.batch_normalization(features, training=training, reuse=self._reuse) + features = tf.layers.batch_normalization( + features, training=training, reuse=self._reuse) elif do_ln: - features = layer_norm(features, name=self._group_name + '_features', reuse=self._reuse) + features = layer_norm( + features, name=self._group_name + '_features', reuse=self._reuse) output_feature_list = config.output_2d_tensor_and_feature_list output_feature_list = output_feature_list or config.only_output_feature_list @@ -163,7 +172,8 @@ def call(self, config, training): for i in range(num_features): fea = feature_list[i] if do_bn: - fea = tf.layers.batch_normalization(fea, training=training, reuse=self._reuse) + fea = tf.layers.batch_normalization( + fea, training=training, reuse=self._reuse) elif do_ln: ln_name = self._group_name + 'f_%d' % i fea = layer_norm(fea, name=ln_name, reuse=self._reuse) diff --git a/easy_rec/python/layers/dnn.py b/easy_rec/python/layers/dnn.py index 01555822e..9587fb358 100644 --- a/easy_rec/python/layers/dnn.py +++ b/easy_rec/python/layers/dnn.py @@ -11,14 +11,15 @@ class DNN: + def __init__( - self, - dnn_config, - l2_reg, - name='dnn', - is_training=False, - last_layer_no_activation=False, - last_layer_no_batch_norm=False, + self, + dnn_config, + l2_reg, + name='dnn', + is_training=False, + last_layer_no_activation=False, + last_layer_no_batch_norm=False, ): """Initializes a `DNN` Layer. @@ -35,7 +36,8 @@ def __init__( self._name = name self._is_training = is_training logging.info('dnn activation function = %s' % self._config.activation) - self.activation = get_activation(self._config.activation, training=is_training) + self.activation = get_activation( + self._config.activation, training=is_training) self._last_layer_no_activation = last_layer_no_activation self._last_layer_no_batch_norm = last_layer_no_batch_norm @@ -55,27 +57,30 @@ def __call__(self, deep_fea, hidden_layer_feature_output=False): hidden_feature_dict = {} for i, unit in enumerate(self.hidden_units): deep_fea = tf.layers.dense( - inputs=deep_fea, - units=unit, - kernel_regularizer=self._l2_reg, - activation=None, - name='%s/dnn_%d' % (self._name, i), + inputs=deep_fea, + units=unit, + kernel_regularizer=self._l2_reg, + activation=None, + name='%s/dnn_%d' % (self._name, i), ) - if self._config.use_bn and ((i + 1 < hidden_units_len) or not self._last_layer_no_batch_norm): + if self._config.use_bn and ((i + 1 < hidden_units_len) or + not self._last_layer_no_batch_norm): deep_fea = tf.layers.batch_normalization( - deep_fea, - training=self._is_training, - trainable=True, - name='%s/dnn_%d/bn' % (self._name, i), + deep_fea, + training=self._is_training, + trainable=True, + name='%s/dnn_%d/bn' % (self._name, i), ) if (i + 1 < hidden_units_len) or not self._last_layer_no_activation: - deep_fea = self.activation(deep_fea, name='%s/dnn_%d/act' % (self._name, i)) + deep_fea = self.activation( + deep_fea, name='%s/dnn_%d/act' % (self._name, i)) if len(self.dropout_ratio) > 0 and self._is_training: - assert self.dropout_ratio[i] < 1, 'invalid dropout_ratio: %.3f' % self.dropout_ratio[i] + assert self.dropout_ratio[ + i] < 1, 'invalid dropout_ratio: %.3f' % self.dropout_ratio[i] deep_fea = tf.nn.dropout( - deep_fea, - keep_prob=1 - self.dropout_ratio[i], - name='%s/%d/dropout' % (self._name, i), + deep_fea, + keep_prob=1 - self.dropout_ratio[i], + name='%s/%d/dropout' % (self._name, i), ) if hidden_layer_feature_output: diff --git a/easy_rec/python/layers/embed_input_layer.py b/easy_rec/python/layers/embed_input_layer.py index 8c325f687..63250de4b 100644 --- a/easy_rec/python/layers/embed_input_layer.py +++ b/easy_rec/python/layers/embed_input_layer.py @@ -7,14 +7,16 @@ class EmbedInputLayer(object): + def __init__(self, feature_groups_config, dump_dir=None): - self._feature_groups = {x.group_name: FeatureGroup(x) for x in feature_groups_config} + self._feature_groups = { + x.group_name: FeatureGroup(x) for x in feature_groups_config + } self._dump_dir = dump_dir def __call__(self, features, group_name): assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ','.join( - [x for x in self._feature_groups] - ) + [x for x in self._feature_groups]) feature_group = self._feature_groups[group_name] group_features = [] for feature_name in feature_group.feature_names: diff --git a/easy_rec/python/layers/fm.py b/easy_rec/python/layers/fm.py index 342204c9a..1929e00aa 100644 --- a/easy_rec/python/layers/fm.py +++ b/easy_rec/python/layers/fm.py @@ -8,6 +8,7 @@ class FM: + def __init__(self, name='fm'): """Initializes a `FM` Layer. diff --git a/easy_rec/python/layers/input_layer.py b/easy_rec/python/layers/input_layer.py index e1f145fdd..19f6e4202 100644 --- a/easy_rec/python/layers/input_layer.py +++ b/easy_rec/python/layers/input_layer.py @@ -6,23 +6,24 @@ import tensorflow as tf from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops, variable_scope +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import variable_scope from easy_rec.python.compat import regularizers from easy_rec.python.compat.feature_column import feature_column -from easy_rec.python.compat.feature_column.feature_column_v2 import ( # NOQA - is_embedding_column, -) from easy_rec.python.feature_column.feature_column import FeatureColumnParser from easy_rec.python.feature_column.feature_group import FeatureGroup -from easy_rec.python.layers import ( # NOQA - sequence_feature_layer, - variational_dropout_layer, -) from easy_rec.python.layers.keras import TextCNN from easy_rec.python.layers.utils import Parameter from easy_rec.python.protos.feature_config_pb2 import WideOrDeep -from easy_rec.python.utils import conditional, shape_utils +from easy_rec.python.utils import conditional +from easy_rec.python.utils import shape_utils + +from easy_rec.python.compat.feature_column.feature_column_v2 import ( # NOQA + is_embedding_column,) +from easy_rec.python.layers import ( # NOQA + sequence_feature_layer, variational_dropout_layer, +) class InputLayer(object): @@ -32,36 +33,44 @@ class InputLayer(object): """ def __init__( - self, - feature_configs, - feature_groups_config, - variational_dropout_config=None, - wide_output_dim=-1, - ev_params=None, - embedding_regularizer=None, - kernel_regularizer=None, - is_training=False, - is_predicting=False, - ): - self._feature_groups = {x.group_name: FeatureGroup(x) for x in feature_groups_config} - self.sequence_feature_layer = sequence_feature_layer.SequenceFeatureLayer( + self, feature_configs, feature_groups_config, - ev_params, - embedding_regularizer, - kernel_regularizer, - is_training, - is_predicting, + variational_dropout_config=None, + wide_output_dim=-1, + ev_params=None, + embedding_regularizer=None, + kernel_regularizer=None, + is_training=False, + is_predicting=False, + ): + self._feature_groups = { + x.group_name: FeatureGroup(x) for x in feature_groups_config + } + self.sequence_feature_layer = sequence_feature_layer.SequenceFeatureLayer( + feature_configs, + feature_groups_config, + ev_params, + embedding_regularizer, + kernel_regularizer, + is_training, + is_predicting, ) self._seq_feature_groups_config = [] for x in feature_groups_config: for y in x.sequence_features: self._seq_feature_groups_config.append(y) self._group_name_to_seq_features = { - x.group_name: x.sequence_features for x in feature_groups_config if len(x.sequence_features) > 0 + x.group_name: x.sequence_features + for x in feature_groups_config + if len(x.sequence_features) > 0 } wide_and_deep_dict = self.get_wide_deep_dict() - self._fc_parser = FeatureColumnParser(feature_configs, wide_and_deep_dict, wide_output_dim, ev_params=ev_params) + self._fc_parser = FeatureColumnParser( + feature_configs, + wide_and_deep_dict, + wide_output_dim, + ev_params=ev_params) self._embedding_regularizer = embedding_regularizer self._kernel_regularizer = kernel_regularizer @@ -90,26 +99,27 @@ def get_combined_feature(self, features, group_name, is_dict=False): place_on_cpu = os.getenv('place_embedding_on_cpu') place_on_cpu = eval(place_on_cpu) if place_on_cpu else False - with conditional(self._is_predicting and place_on_cpu, ops.device('/CPU:0')): + with conditional(self._is_predicting and place_on_cpu, + ops.device('/CPU:0')): concat_features, group_features = self.single_call_input_layer( - features, group_name, feature_name_to_output_tensors - ) + features, group_name, feature_name_to_output_tensors) if group_name in self._group_name_to_seq_features: # for target attention group_seq_arr = self._group_name_to_seq_features[group_name] concat_features, all_seq_fea = self.sequence_feature_layer( - features, - concat_features, - group_seq_arr, - feature_name_to_output_tensors, - negative_sampler=negative_sampler, - scope_name=group_name, + features, + concat_features, + group_seq_arr, + feature_name_to_output_tensors, + negative_sampler=negative_sampler, + scope_name=group_name, ) group_features.extend(all_seq_fea) for col, fea in zip(group_seq_arr, all_seq_fea): feature_name_to_output_tensors['seq_fea/' + col.group_name] = fea all_seq_fea = array_ops.concat(all_seq_fea, axis=-1) - concat_features = array_ops.concat([concat_features, all_seq_fea], axis=-1) + concat_features = array_ops.concat([concat_features, all_seq_fea], + axis=-1) if is_dict: return concat_features, group_features, feature_name_to_output_tensors else: @@ -127,8 +137,8 @@ def get_plain_feature(self, features, group_name): group_features: list of features """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, - ','.join([x for x in self._feature_groups]), + group_name, + ','.join([x for x in self._feature_groups]), ) feature_group = self._feature_groups[group_name] @@ -138,10 +148,10 @@ def get_plain_feature(self, features, group_name): cols_to_output_tensors = OrderedDict() output_features = feature_column.input_layer( - features, - group_columns, - cols_to_output_tensors=cols_to_output_tensors, - is_training=self._is_training, + features, + group_columns, + cols_to_output_tensors=cols_to_output_tensors, + is_training=self._is_training, ) group_features = [cols_to_output_tensors[x] for x in group_columns] @@ -151,7 +161,8 @@ def get_plain_feature(self, features, group_name): embedding_reg_lst.append(val) if self._embedding_regularizer is not None and len(embedding_reg_lst) > 0: - regularizers.apply_regularization(self._embedding_regularizer, weights_list=embedding_reg_lst) + regularizers.apply_regularization( + self._embedding_regularizer, weights_list=embedding_reg_lst) return output_features, group_features def get_sequence_feature(self, features, group_name): @@ -167,12 +178,13 @@ def get_sequence_feature(self, features, group_name): 1d sequence length tensor. """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, - ','.join([x for x in self._feature_groups]), + group_name, + ','.join([x for x in self._feature_groups]), ) if self._variational_dropout_config is not None: - raise ValueError('variational dropout is not supported in not combined mode now.') + raise ValueError( + 'variational dropout is not supported in not combined mode now.') feature_group = self._feature_groups[group_name] _, group_seq_columns = feature_group.select_columns(self._fc_parser) @@ -181,15 +193,18 @@ def get_sequence_feature(self, features, group_name): builder = feature_column._LazyBuilder(features) seq_features = [] for fc in group_seq_columns: - with variable_scope.variable_scope('input_layer/' + fc.categorical_column.name): + with variable_scope.variable_scope('input_layer/' + + fc.categorical_column.name): tmp_embedding, tmp_seq_len = fc._get_sequence_dense_tensor(builder) if fc.max_seq_length > 0: - tmp_embedding, tmp_seq_len = shape_utils.truncate_sequence(tmp_embedding, tmp_seq_len, fc.max_seq_length) + tmp_embedding, tmp_seq_len = shape_utils.truncate_sequence( + tmp_embedding, tmp_seq_len, fc.max_seq_length) seq_features.append((tmp_embedding, tmp_seq_len)) embedding_reg_lst.append(tmp_embedding) if self._embedding_regularizer is not None and len(embedding_reg_lst) > 0: - regularizers.apply_regularization(self._embedding_regularizer, weights_list=embedding_reg_lst) + regularizers.apply_regularization( + self._embedding_regularizer, weights_list=embedding_reg_lst) return seq_features def get_raw_features(self, features, group_name): @@ -203,8 +218,8 @@ def get_raw_features(self, features, group_name): features: all raw features in list """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, - ','.join([x for x in self._feature_groups]), + group_name, + ','.join([x for x in self._feature_groups]), ) feature_group = self._feature_groups[group_name] return [features[x] for x in feature_group.feature_names] @@ -220,8 +235,8 @@ def get_bucketized_features(self, features, group_name): features: all raw features in list, added feature offset """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, - ','.join([x for x in self._feature_groups]), + group_name, + ','.join([x for x in self._feature_groups]), ) feature_group = self._feature_groups[group_name] offset = 0 @@ -268,8 +283,8 @@ def __call__(self, features, group_name, is_combine=True, is_dict=False): 1 dimension sequence length tensor. """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, - ','.join([x for x in self._feature_groups]), + group_name, + ','.join([x for x in self._feature_groups]), ) if is_combine: return self.get_combined_feature(features, group_name, is_dict) @@ -277,12 +292,17 @@ def __call__(self, features, group_name, is_combine=True, is_dict=False): # return sequence feature in raw format instead of combine them place_on_cpu = os.getenv('place_embedding_on_cpu') place_on_cpu = eval(place_on_cpu) if place_on_cpu else False - with conditional(self._is_predicting and place_on_cpu, ops.device('/CPU:0')): + with conditional(self._is_predicting and place_on_cpu, + ops.device('/CPU:0')): seq_features = self.get_sequence_feature(features, group_name) - plain_features, feature_list = self.get_plain_feature(features, group_name) + plain_features, feature_list = self.get_plain_feature( + features, group_name) return seq_features, plain_features, feature_list - def single_call_input_layer(self, features, group_name, feature_name_to_output_tensors=None): + def single_call_input_layer(self, + features, + group_name, + feature_name_to_output_tensors=None): """Get features by group_name. Args: @@ -296,45 +316,51 @@ def single_call_input_layer(self, features, group_name, feature_name_to_output_t group_features: list of features """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, - ','.join([x for x in self._feature_groups]), + group_name, + ','.join([x for x in self._feature_groups]), ) feature_group = self._feature_groups[group_name] - group_columns, group_seq_columns = feature_group.select_columns(self._fc_parser) + group_columns, group_seq_columns = feature_group.select_columns( + self._fc_parser) cols_to_output_tensors = OrderedDict() output_features = feature_column.input_layer( - features, - group_columns, - cols_to_output_tensors=cols_to_output_tensors, - feature_name_to_output_tensors=feature_name_to_output_tensors, - is_training=self._is_training, + features, + group_columns, + cols_to_output_tensors=cols_to_output_tensors, + feature_name_to_output_tensors=feature_name_to_output_tensors, + is_training=self._is_training, ) embedding_reg_lst = [] builder = feature_column._LazyBuilder(features) seq_features = [] for column in sorted(group_seq_columns, key=lambda x: x.name): - with variable_scope.variable_scope(None, default_name=column._var_scope_name): + with variable_scope.variable_scope( + None, default_name=column._var_scope_name): seq_feature, seq_len = column._get_sequence_dense_tensor(builder) embedding_reg_lst.append(seq_feature) sequence_combiner = column.sequence_combiner if sequence_combiner is None: - raise ValueError('sequence_combiner is none, please set sequence_combiner or use TagFeature') + raise ValueError( + 'sequence_combiner is none, please set sequence_combiner or use TagFeature' + ) if sequence_combiner.WhichOneof('combiner') == 'attention': attn_logits = tf.layers.dense( - inputs=seq_feature, - units=1, - kernel_regularizer=self._kernel_regularizer, - use_bias=False, - activation=None, - name='attention', + inputs=seq_feature, + units=1, + kernel_regularizer=self._kernel_regularizer, + use_bias=False, + activation=None, + name='attention', ) attn_logits = tf.squeeze(attn_logits, axis=-1) attn_logits_padding = tf.ones_like(attn_logits) * (-(2**32) + 1) seq_mask = tf.sequence_mask(seq_len) - attn_score = tf.nn.softmax(tf.where(seq_mask, attn_logits, attn_logits_padding)) - seq_feature = tf.reduce_sum(attn_score[:, :, tf.newaxis] * seq_feature, axis=1) + attn_score = tf.nn.softmax( + tf.where(seq_mask, attn_logits, attn_logits_padding)) + seq_feature = tf.reduce_sum( + attn_score[:, :, tf.newaxis] * seq_feature, axis=1) seq_features.append(seq_feature) cols_to_output_tensors[column] = seq_feature elif sequence_combiner.WhichOneof('combiner') == 'text_cnn': @@ -346,20 +372,26 @@ def single_call_input_layer(self, features, group_name, feature_name_to_output_t else: raise NotImplementedError if self._variational_dropout_config is not None: - features_dimension = OrderedDict([(k.raw_name, int(v.shape[-1])) for k, v in cols_to_output_tensors.items()]) - concat_features = array_ops.concat([output_features] + seq_features, axis=-1) + features_dimension = OrderedDict([ + (k.raw_name, int(v.shape[-1])) + for k, v in cols_to_output_tensors.items() + ]) + concat_features = array_ops.concat( + [output_features] + seq_features, axis=-1) variational_dropout = variational_dropout_layer.VariationalDropoutLayer( - self._variational_dropout_config, - features_dimension, - self._is_training, - name=group_name, + self._variational_dropout_config, + features_dimension, + self._is_training, + name=group_name, ) concat_features = variational_dropout(concat_features) - group_features = tf.split(concat_features, list(features_dimension.values()), axis=-1) + group_features = tf.split( + concat_features, list(features_dimension.values()), axis=-1) else: - concat_features = array_ops.concat([output_features] + seq_features, axis=-1) + concat_features = array_ops.concat( + [output_features] + seq_features, axis=-1) group_features = [cols_to_output_tensors[x] for x in group_columns] + [ - cols_to_output_tensors[x] for x in group_seq_columns + cols_to_output_tensors[x] for x in group_seq_columns ] if self._embedding_regularizer is not None: @@ -367,7 +399,8 @@ def single_call_input_layer(self, features, group_name, feature_name_to_output_t if is_embedding_column(fc): embedding_reg_lst.append(val) if embedding_reg_lst: - regularizers.apply_regularization(self._embedding_regularizer, weights_list=embedding_reg_lst) + regularizers.apply_regularization( + self._embedding_regularizer, weights_list=embedding_reg_lst) return concat_features, group_features def get_wide_deep_dict(self): diff --git a/easy_rec/python/layers/keras/__init__.py b/easy_rec/python/layers/keras/__init__.py index 64ed4cad5..7afbc8ebb 100644 --- a/easy_rec/python/layers/keras/__init__.py +++ b/easy_rec/python/layers/keras/__init__.py @@ -1,26 +1,34 @@ from .attention import Attention from .auxiliary_loss import AuxiliaryLoss -from .blocks import MLP, Gate, Highway, TextCNN +from .blocks import MLP +from .blocks import Gate +from .blocks import Highway +from .blocks import TextCNN from .bst import BST -from .custom_ops import ( # NOQA - EditDistance, - MappedDotProduct, - OverlapFeature, - SeqAugmentOps, - TextNormalize, -) from .data_augment import SeqAugment from .din import DIN from .embedding import EmbeddingLayer -from .fibinet import BiLinear, FiBiNet, SENet -from .interaction import CIN, FM, Cross, DotInteraction -from .mask_net import MaskBlock, MaskNet +from .fibinet import BiLinear +from .fibinet import FiBiNet +from .fibinet import SENet +from .interaction import CIN +from .interaction import FM +from .interaction import Cross +from .interaction import DotInteraction +from .mask_net import MaskBlock +from .mask_net import MaskNet from .multi_head_attention import MultiHeadAttention -from .multi_task import AITMTower, MMoE +from .multi_task import AITMTower +from .multi_task import MMoE +from .ppnet import PPNet +from .transformer import TextEncoder +from .transformer import TransformerBlock +from .transformer import TransformerEncoder + +from .custom_ops import ( # NOQA + EditDistance, MappedDotProduct, OverlapFeature, SeqAugmentOps, + TextNormalize, +) from .numerical_embedding import ( # NOQA - AutoDisEmbedding, - NaryDisEmbedding, - PeriodicEmbedding, + AutoDisEmbedding, NaryDisEmbedding, PeriodicEmbedding, ) -from .ppnet import PPNet -from .transformer import TextEncoder, TransformerBlock, TransformerEncoder diff --git a/easy_rec/python/layers/keras/activation.py b/easy_rec/python/layers/keras/activation.py index 32c9f0e37..d83351638 100644 --- a/easy_rec/python/layers/keras/activation.py +++ b/easy_rec/python/layers/keras/activation.py @@ -1,7 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from tensorflow.python.keras.layers import Activation, Layer +from tensorflow.python.keras.layers import Activation +from tensorflow.python.keras.layers import Layer import easy_rec.python.utils.activation @@ -49,12 +50,13 @@ def __init__(self, axis=-1, epsilon=1e-9, **kwargs): super(Dice, self).__init__(**kwargs) def build(self, input_shape): - self.bn = BatchNormalization(axis=self.axis, epsilon=self.epsilon, center=False, scale=False) + self.bn = BatchNormalization( + axis=self.axis, epsilon=self.epsilon, center=False, scale=False) self.alphas = self.add_weight( - shape=(input_shape[-1],), - initializer=Zeros(), - dtype=tf.float32, - name='dice_alpha', + shape=(input_shape[-1],), + initializer=Zeros(), + dtype=tf.float32, + name='dice_alpha', ) # name='alpha_'+self.name super(Dice, self).build(input_shape) # Be sure to call this somewhere! self.uses_learning_phase = True @@ -73,15 +75,14 @@ def compute_output_shape(self, input_shape): def updates(self): return self.bn.updates - def get_config( - self, - ): + def get_config(self,): config = {'axis': self.axis, 'epsilon': self.epsilon} base_config = super(Dice, self).get_config() return dict(list(base_config.items()) + list(config.items())) class MaskedSoftmax(Layer): + def __init__(self, axis=-1, **kwargs): super(MaskedSoftmax, self).__init__(**kwargs) self.axis = axis @@ -108,5 +109,7 @@ def activation_layer(activation, name=None): elif issubclass(activation, Layer): act_layer = activation(name=name) else: - raise ValueError('Invalid activation,found %s.You should use a str or a Activation Layer Class.' % (activation)) + raise ValueError( + 'Invalid activation,found %s.You should use a str or a Activation Layer Class.' + % (activation)) return act_layer diff --git a/easy_rec/python/layers/keras/attention.py b/easy_rec/python/layers/keras/attention.py index eb44e29d1..08e867eb7 100644 --- a/easy_rec/python/layers/keras/attention.py +++ b/easy_rec/python/layers/keras/attention.py @@ -73,16 +73,15 @@ def __init__(self, params, name='attention', reuse=None, **kwargs): self.scale_by_dim = params.get_or_default('scale_by_dim', False) self.score_mode = params.get_or_default('score_mode', 'dot') if self.score_mode not in ['dot', 'concat']: - raise ValueError( - 'Invalid value for argument score_mode. ' - "Expected one of {'dot', 'concat'}. " - 'Received: score_mode=%s' % self.score_mode - ) + raise ValueError('Invalid value for argument score_mode. ' + "Expected one of {'dot', 'concat'}. " + 'Received: score_mode=%s' % self.score_mode) self.dropout = params.get_or_default('dropout', 0.0) self.seed = params.get_or_default('seed', None) self.scale = None self.concat_score_weight = None - self._return_attention_scores = params.get_or_default('return_attention_scores', False) + self._return_attention_scores = params.get_or_default( + 'return_attention_scores', False) self.use_causal_mask = params.get_or_default('use_causal_mask', False) @property @@ -93,19 +92,19 @@ def build(self, input_shape): self._validate_inputs(input_shape) if self.use_scale: self.scale = self.add_weight( - name='scale', - shape=(), - initializer='ones', - dtype=self.dtype, - trainable=True, + name='scale', + shape=(), + initializer='ones', + dtype=self.dtype, + trainable=True, ) if self.score_mode == 'concat': self.concat_score_weight = self.add_weight( - name='concat_score_weight', - shape=(), - initializer='ones', - dtype=self.dtype, - trainable=True, + name='concat_score_weight', + shape=(), + initializer='ones', + dtype=self.dtype, + trainable=True, ) super(Attention, self).build(input_shape) # Be sure to call this somewhere! @@ -133,9 +132,11 @@ def _calculate_scores(self, query, key): # Reshape into [batch_size, 1, Tv, dim]. k_reshaped = tf.expand_dims(key, axis=-3) if self.scale is not None: - scores = self.concat_score_weight * tf.reduce_sum(tf.tanh(self.scale * (q_reshaped + k_reshaped)), axis=-1) + scores = self.concat_score_weight * tf.reduce_sum( + tf.tanh(self.scale * (q_reshaped + k_reshaped)), axis=-1) else: - scores = self.concat_score_weight * tf.reduce_sum(tf.tanh(q_reshaped + k_reshaped), axis=-1) + scores = self.concat_score_weight * tf.reduce_sum( + tf.tanh(q_reshaped + k_reshaped), axis=-1) return scores def _apply_scores(self, scores, value, scores_mask=None, training=False): @@ -211,8 +212,10 @@ def call(self, inputs, mask=None, training=False, **kwargs): q_mask = mask[0] if mask else None v_mask = mask[1] if mask else None scores = self._calculate_scores(query=q, key=k) - scores_mask = self._calculate_score_mask(scores, v_mask, self.use_causal_mask) - result, attention_scores = self._apply_scores(scores=scores, value=v, scores_mask=scores_mask, training=training) + scores_mask = self._calculate_score_mask(scores, v_mask, + self.use_causal_mask) + result, attention_scores = self._apply_scores( + scores=scores, value=v, scores_mask=scores_mask, training=training) if q_mask is not None: # Mask of shape [batch_size, Tq, 1]. q_mask = tf.expand_dims(q_mask, axis=-1) @@ -235,36 +238,31 @@ def _validate_inputs(self, inputs, mask=None): """Validates arguments of the call method.""" class_name = self.__class__.__name__ if not isinstance(inputs, list): - raise ValueError( - '{class_name} layer must be called on a list of inputs, ' - 'namely [query, value] or [query, value, key]. ' - 'Received: inputs={inputs}.'.format(class_name=class_name, inputs=inputs) - ) + raise ValueError('{class_name} layer must be called on a list of inputs, ' + 'namely [query, value] or [query, value, key]. ' + 'Received: inputs={inputs}.'.format( + class_name=class_name, inputs=inputs)) if len(inputs) < 2 or len(inputs) > 3: - raise ValueError( - '%s layer accepts inputs list of length 2 or 3, ' - 'namely [query, value] or [query, value, key]. ' - 'Received length: %d.' % (class_name, len(inputs)) - ) + raise ValueError('%s layer accepts inputs list of length 2 or 3, ' + 'namely [query, value] or [query, value, key]. ' + 'Received length: %d.' % (class_name, len(inputs))) if mask is not None: if not isinstance(mask, list): raise ValueError( - '{class_name} layer mask must be a list, ' 'namely [query_mask, value_mask]. Received: mask={mask}.'.format( - class_name=class_name, mask=mask - ) - ) + '{class_name} layer mask must be a list, ' + 'namely [query_mask, value_mask]. Received: mask={mask}.'.format( + class_name=class_name, mask=mask)) if len(mask) < 2 or len(mask) > 3: raise ValueError( - '{class_name} layer accepts mask list of length 2 or 3. ' 'Received: inputs={inputs}, mask={mask}.'.format( - class_name=class_name, inputs=inputs, mask=mask - ) - ) + '{class_name} layer accepts mask list of length 2 or 3. ' + 'Received: inputs={inputs}, mask={mask}.'.format( + class_name=class_name, inputs=inputs, mask=mask)) def get_config(self): base_config = super(Attention, self).get_config() config = { - 'use_scale': self.use_scale, - 'score_mode': self.score_mode, - 'dropout': self.dropout, + 'use_scale': self.use_scale, + 'score_mode': self.score_mode, + 'dropout': self.dropout, } return dict(list(base_config.items()) + list(config.items())) diff --git a/easy_rec/python/layers/keras/auxiliary_loss.py b/easy_rec/python/layers/keras/auxiliary_loss.py index 6ad614985..6be248872 100644 --- a/easy_rec/python/layers/keras/auxiliary_loss.py +++ b/easy_rec/python/layers/keras/auxiliary_loss.py @@ -15,7 +15,8 @@ def __init__(self, params, name='auxiliary_loss', reuse=None, **kwargs): params.check_required('loss_type') self.loss_type = params.get_or_default('loss_type', None) self.loss_weight = params.get_or_default('loss_weight', 1.0) - logging.info('init layer `%s` with loss type: %s and weight: %f' % (self.name, self.loss_type, self.loss_weight)) + logging.info('init layer `%s` with loss type: %s and weight: %f' % + (self.name, self.loss_type, self.loss_weight)) self.temperature = params.get_or_default('temperature', 0.1) def call(self, inputs, training=None, **kwargs): @@ -33,7 +34,8 @@ def call(self, inputs, training=None, **kwargs): loss_dict['%s_l2_loss' % self.name] = loss_value elif self.loss_type == 'info_nce': query, positive = inputs - loss = contrastive_loss.info_nce_loss(query, positive, temperature=self.temperature) + loss = contrastive_loss.info_nce_loss( + query, positive, temperature=self.temperature) loss_value = loss if self.loss_weight == 1.0 else loss * self.loss_weight loss_dict['%s_info_nce_loss' % self.name] = loss_value elif self.loss_type == 'nce_loss': diff --git a/easy_rec/python/layers/keras/blocks.py b/easy_rec/python/layers/keras/blocks.py index a89e3898d..23ca4f445 100644 --- a/easy_rec/python/layers/keras/blocks.py +++ b/easy_rec/python/layers/keras/blocks.py @@ -6,7 +6,10 @@ import tensorflow as tf from tensorflow.python.keras.initializers import Constant -from tensorflow.python.keras.layers import Dense, Dropout, Lambda, Layer +from tensorflow.python.keras.layers import Dense +from tensorflow.python.keras.layers import Dropout +from tensorflow.python.keras.layers import Lambda +from tensorflow.python.keras.layers import Layer from easy_rec.python.layers.keras.activation import activation_layer from easy_rec.python.layers.utils import Parameter @@ -43,21 +46,20 @@ def __init__(self, params, name='mlp', reuse=None, **kwargs): use_bn_after_act = params.get_or_default('use_bn_after_activation', False) units = list(params.hidden_units) logging.info( - 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' - ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' - % ( - name, - units, - dropout_rate, - activation, - use_bn, - use_final_bn, - final_activation, - use_bias, - initializer, - use_bn_after_act, - ) - ) + 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' + ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' % + ( + name, + units, + dropout_rate, + activation, + use_bn, + use_final_bn, + final_activation, + use_bias, + initializer, + use_bn_after_act, + )) assert len(units) > 0, 'MLP(%s) takes at least one hidden units' % name self.reuse = reuse self.add_to_outputs = params.get_or_default('add_to_outputs', False) @@ -68,64 +70,65 @@ def __init__(self, params, name='mlp', reuse=None, **kwargs): name = 'layer_%d' % i drop_rate = dropout_rate[i] if i < num_dropout else 0.0 self.add_rich_layer( - num_units, - use_bn, - drop_rate, - activation, - initializer, - use_bias, - use_bn_after_act, - name, - params.l2_regularizer, + num_units, + use_bn, + drop_rate, + activation, + initializer, + use_bias, + use_bn_after_act, + name, + params.l2_regularizer, ) n = len(units) - 1 drop_rate = dropout_rate[n] if num_dropout > n else 0.0 name = 'layer_%d' % n self.add_rich_layer( - units[-1], - use_final_bn, - drop_rate, - final_activation, - initializer, - use_final_bias, - use_bn_after_act, - name, - params.l2_regularizer, + units[-1], + use_final_bn, + drop_rate, + final_activation, + initializer, + use_final_bias, + use_bn_after_act, + name, + params.l2_regularizer, ) def add_rich_layer( - self, - num_units, - use_bn, - dropout_rate, - activation, - initializer, - use_bias, - use_bn_after_activation, - name, - l2_reg=None, + self, + num_units, + use_bn, + dropout_rate, + activation, + initializer, + use_bias, + use_bn_after_activation, + name, + l2_reg=None, ): act_layer = activation_layer(activation, name='%s/act' % name) if use_bn and not use_bn_after_activation: dense = Dense( - units=num_units, - use_bias=use_bias, - kernel_initializer=initializer, - kernel_regularizer=l2_reg, - name='%s/dense' % name, + units=num_units, + use_bias=use_bias, + kernel_initializer=initializer, + kernel_regularizer=l2_reg, + name='%s/dense' % name, ) self._sub_layers.append(dense) - bn = tf.keras.layers.BatchNormalization(name='%s/bn' % name, trainable=True) + bn = tf.keras.layers.BatchNormalization( + name='%s/bn' % name, trainable=True) self._sub_layers.append(bn) self._sub_layers.append(act_layer) else: dense = Dense( - num_units, - use_bias=use_bias, - kernel_initializer=initializer, - kernel_regularizer=l2_reg, - name='%s/dense' % name, + num_units, + use_bias=use_bias, + kernel_initializer=initializer, + kernel_regularizer=l2_reg, + name='%s/dense' % name, ) self._sub_layers.append(dense) self._sub_layers.append(act_layer) @@ -157,6 +160,7 @@ def call(self, x, training=None, **kwargs): class Highway(Layer): + def __init__(self, params, name='highway', reuse=None, **kwargs): super(Highway, self).__init__(name=name, **kwargs) self.emb_size = params.get_or_default('emb_size', None) @@ -165,7 +169,8 @@ def __init__(self, params, name='highway', reuse=None, **kwargs): self.dropout_rate = params.get_or_default('dropout_rate', 0.0) self.init_gate_bias = params.get_or_default('init_gate_bias', -3.0) self.act_layer = activation_layer(self.activation) - self.dropout_layer = Dropout(self.dropout_rate) if self.dropout_rate > 0.0 else None + self.dropout_layer = Dropout( + self.dropout_rate) if self.dropout_rate > 0.0 else None self.project_layer = None self.gate_bias_initializer = Constant(self.init_gate_bias) self.gates = [] # T @@ -181,10 +186,10 @@ def build(self, input_shape): self.carry_gate = Lambda(lambda x: 1.0 - x, output_shape=(dim,)) for i in range(self.num_layers): gate = Dense( - units=dim, - bias_initializer=self.gate_bias_initializer, - activation='sigmoid', - name='gate_%d' % i, + units=dim, + bias_initializer=self.gate_bias_initializer, + activation='sigmoid', + name='gate_%d' % i, ) self.gates.append(gate) self.transforms.append(Dense(units=dim)) @@ -218,7 +223,9 @@ def __init__(self, params, name='gate', reuse=None, **kwargs): self.top_mlp = None def call(self, inputs, training=None, **kwargs): - assert len(inputs) > 1, 'input of Gate layer must be a list containing at least 2 elements' + assert len( + inputs + ) > 1, 'input of Gate layer must be a list containing at least 2 elements' weights = inputs[self.weight_index] j = 0 for i, x in enumerate(inputs): @@ -246,15 +253,17 @@ def __init__(self, params, name='text_cnn', reuse=None, **kwargs): self.config = params.get_pb_config() self.pad_seq_length = self.config.pad_sequence_length if self.pad_seq_length <= 0: - logging.warning('run text cnn with pad_sequence_length <= 0, the predict of model may be unstable') + logging.warning( + 'run text cnn with pad_sequence_length <= 0, the predict of model may be unstable' + ) self.conv_layers = [] self.pool_layer = tf.keras.layers.GlobalMaxPool1D() self.concat_layer = tf.keras.layers.Concatenate(axis=-1) for size, filters in zip(self.config.filter_sizes, self.config.num_filters): conv = tf.keras.layers.Conv1D( - filters=int(filters), - kernel_size=int(size), - activation=self.config.activation, + filters=int(filters), + kernel_size=int(size), + activation=self.config.activation, ) self.conv_layers.append(conv) if self.config.HasField('mlp'): @@ -271,7 +280,8 @@ def call(self, inputs, training=None, **kwargs): seq_emb, seq_len = inputs[:2] if self.pad_seq_length > 0: - seq_emb, seq_len = pad_or_truncate_sequence(seq_emb, seq_len, self.pad_seq_length) + seq_emb, seq_len = pad_or_truncate_sequence(seq_emb, seq_len, + self.pad_seq_length) pooled_outputs = [] for layer in self.conv_layers: conv = layer(seq_emb) diff --git a/easy_rec/python/layers/keras/bst.py b/easy_rec/python/layers/keras/bst.py index 5920ffa10..123f86e73 100644 --- a/easy_rec/python/layers/keras/bst.py +++ b/easy_rec/python/layers/keras/bst.py @@ -12,6 +12,7 @@ class BST(Layer): + def __init__(self, params, name='bst', reuse=None, **kwargs): super(BST, self).__init__(name=name, **kwargs) self.reuse = reuse @@ -20,37 +21,37 @@ def __init__(self, params, name='bst', reuse=None, **kwargs): def encode(self, seq_input, max_position): seq_fea = multihead_cross_attention.embedding_postprocessor( - seq_input, - position_embedding_name=self.name, - max_position_embeddings=max_position, - reuse_position_embedding=self.reuse, + seq_input, + position_embedding_name=self.name, + max_position_embeddings=max_position, + reuse_position_embedding=self.reuse, ) n = tf.count_nonzero(seq_input, axis=-1) seq_mask = tf.cast(n > 0, tf.int32) attention_mask = multihead_cross_attention.create_attention_mask_from_input_mask( - from_tensor=seq_fea, to_mask=seq_mask - ) + from_tensor=seq_fea, to_mask=seq_mask) hidden_act = get_activation(self.config.hidden_act) attention_fea = multihead_cross_attention.transformer_encoder( - seq_fea, - hidden_size=self.config.hidden_size, - num_hidden_layers=self.config.num_hidden_layers, - num_attention_heads=self.config.num_attention_heads, - attention_mask=attention_mask, - intermediate_size=self.config.intermediate_size, - intermediate_act_fn=hidden_act, - hidden_dropout_prob=self.config.hidden_dropout_prob, - attention_probs_dropout_prob=self.config.attention_probs_dropout_prob, - initializer_range=self.config.initializer_range, - name=self.name + '/transformer', - reuse=self.reuse, + seq_fea, + hidden_size=self.config.hidden_size, + num_hidden_layers=self.config.num_hidden_layers, + num_attention_heads=self.config.num_attention_heads, + attention_mask=attention_mask, + intermediate_size=self.config.intermediate_size, + intermediate_act_fn=hidden_act, + hidden_dropout_prob=self.config.hidden_dropout_prob, + attention_probs_dropout_prob=self.config.attention_probs_dropout_prob, + initializer_range=self.config.initializer_range, + name=self.name + '/transformer', + reuse=self.reuse, ) # attention_fea shape: [batch_size, seq_length, hidden_size] if self.config.output_all_token_embeddings: - out_fea = tf.reshape(attention_fea, [-1, max_position * self.config.hidden_size]) + out_fea = tf.reshape(attention_fea, + [-1, max_position * self.config.hidden_size]) else: out_fea = attention_fea[:, 0, :] # target feature print('bst output shape:', out_fea.shape) @@ -67,52 +68,51 @@ def call(self, inputs, training=None, **kwargs): target = inputs[2] if len(inputs) > 2 else None max_position = self.config.max_position_embeddings # max_seq_len: the max sequence length in current mini-batch, all sequences are padded to this length - batch_size, cur_batch_max_seq_len, seq_embed_size = get_shape_list(seq_input, 3) + batch_size, cur_batch_max_seq_len, seq_embed_size = get_shape_list( + seq_input, 3) valid_len = tf.assert_less_equal( - cur_batch_max_seq_len, - max_position, - message='sequence length is greater than `max_position_embeddings`:' - + str(max_position) - + ' in feature group:' - + self.name - + ', you should set `max_seq_len` in sequence feature configs', + cur_batch_max_seq_len, + max_position, + message='sequence length is greater than `max_position_embeddings`:' + + str(max_position) + ' in feature group:' + self.name + + ', you should set `max_seq_len` in sequence feature configs', ) if self.config.output_all_token_embeddings: seq_input = tf.cond( - tf.constant(max_position) > cur_batch_max_seq_len, - lambda: tf.pad( - seq_input, - [[0, 0], [0, max_position - cur_batch_max_seq_len], [0, 0]], - 'CONSTANT', - ), - lambda: tf.slice(seq_input, [0, 0, 0], [-1, max_position, -1]), + tf.constant(max_position) > cur_batch_max_seq_len, + lambda: tf.pad( + seq_input, + [[0, 0], [0, max_position - cur_batch_max_seq_len], [0, 0]], + 'CONSTANT', + ), + lambda: tf.slice(seq_input, [0, 0, 0], [-1, max_position, -1]), ) if seq_embed_size != self.config.hidden_size: seq_input = tf.layers.dense( - seq_input, - self.config.hidden_size, - activation=tf.nn.relu, - kernel_regularizer=self.l2_reg, - name=self.name + '/seq_project', - reuse=self.reuse, + seq_input, + self.config.hidden_size, + activation=tf.nn.relu, + kernel_regularizer=self.l2_reg, + name=self.name + '/seq_project', + reuse=self.reuse, ) keep_target = self.config.target_item_position in ('head', 'tail') if target is not None and keep_target: target_size = target.shape.as_list()[-1] assert seq_embed_size == target_size, ( - 'the embedding size of sequence and target item is not equal' ' in feature group:' + self.name - ) + 'the embedding size of sequence and target item is not equal' + ' in feature group:' + self.name) if target_size != self.config.hidden_size: target = tf.layers.dense( - target, - self.config.hidden_size, - activation=tf.nn.relu, - kernel_regularizer=self.l2_reg, - name=self.name + '/target_project', - reuse=self.reuse, + target, + self.config.hidden_size, + activation=tf.nn.relu, + kernel_regularizer=self.l2_reg, + name=self.name + '/target_project', + reuse=self.reuse, ) # target_feature: [batch_size, 1, embed_size] target = tf.expand_dims(target, 1) diff --git a/easy_rec/python/layers/keras/custom_ops.py b/easy_rec/python/layers/keras/custom_ops.py index 2e04c6334..2b8cf0c7d 100644 --- a/easy_rec/python/layers/keras/custom_ops.py +++ b/easy_rec/python/layers/keras/custom_ops.py @@ -31,7 +31,8 @@ custom_ops = tf.load_op_library(custom_op_path) logging.info('load custom op from %s succeed' % custom_op_path) except Exception as ex: - logging.warning('load custom op from %s failed: %s' % (custom_op_path, str(ex))) + logging.warning('load custom op from %s failed: %s' % + (custom_op_path, str(ex))) custom_ops = None # if tf.__version__ >= '2.0': @@ -48,26 +49,30 @@ def __init__(self, params, name='sequence_aug', reuse=None, **kwargs): self.seq_augment = custom_ops.my_seq_augment def call(self, inputs, training=None, **kwargs): - assert isinstance(inputs, (list, tuple)), 'the inputs of SeqAugmentOps must be type of list/tuple' + assert isinstance( + inputs, + (list, tuple)), 'the inputs of SeqAugmentOps must be type of list/tuple' assert len(inputs) >= 2, 'SeqAugmentOps must have at least 2 inputs' seq_input, seq_len = inputs[:2] embedding_dim = int(seq_input.shape[-1]) with tf.variable_scope(self.name, reuse=self.reuse): - mask_emb = tf.get_variable('mask', (embedding_dim,), dtype=tf.float32, trainable=True) + mask_emb = tf.get_variable( + 'mask', (embedding_dim,), dtype=tf.float32, trainable=True) seq_len = tf.to_int32(seq_len) with ops.device('/CPU:0'): aug_seq, aug_len = self.seq_augment( - seq_input, - seq_len, - mask_emb, - self.seq_aug_params.crop_rate, - self.seq_aug_params.reorder_rate, - self.seq_aug_params.mask_rate, + seq_input, + seq_len, + mask_emb, + self.seq_aug_params.crop_rate, + self.seq_aug_params.reorder_rate, + self.seq_aug_params.mask_rate, ) return aug_seq, aug_len class TextNormalize(Layer): + def __init__(self, params, name='text_normalize', reuse=None, **kwargs): super(TextNormalize, self).__init__(name=name, **kwargs) self.txt_normalizer = custom_ops.text_normalize_op @@ -78,7 +83,10 @@ def call(self, inputs, training=None, **kwargs): inputs = inputs if type(inputs) in (tuple, list) else [inputs] with ops.device('/CPU:0'): result = [ - self.txt_normalizer(txt, parameter=self.norm_parameter, remove_space=self.remove_space) for txt in inputs + self.txt_normalizer( + txt, + parameter=self.norm_parameter, + remove_space=self.remove_space) for txt in inputs ] if len(result) == 1: return result[0] @@ -86,6 +94,7 @@ def call(self, inputs, training=None, **kwargs): class MappedDotProduct(Layer): + def __init__(self, params, name='mapped_dot_product', reuse=None, **kwargs): super(MappedDotProduct, self).__init__(name=name, **kwargs) self.mapped_dot_product = custom_ops.mapped_dot_product @@ -101,31 +110,31 @@ def __init__(self, params, name='mapped_dot_product', reuse=None, **kwargs): vocab_size = len(self.boundaries) + 1 with tf.variable_scope(self.name, reuse=reuse): self.embedding_table = tf.get_variable( - name='dot_product_emb_table', - shape=[vocab_size, self.emb_dim], - dtype=tf.float32, + name='dot_product_emb_table', + shape=[vocab_size, self.emb_dim], + dtype=tf.float32, ) def call(self, inputs, training=None, **kwargs): query, doc = inputs[:2] with ops.device('/CPU:0'): feature = self.mapped_dot_product( - query=query, - document=doc, - feature_name=self.name, - separator=self.separator, - default_value=self.default_value, + query=query, + document=doc, + feature_name=self.name, + separator=self.separator, + default_value=self.default_value, ) tf.summary.scalar(self.name, tf.reduce_mean(feature)) if self.print_first_n: encode_q = tf.regex_replace(query, self.separator, ' ') encode_t = tf.regex_replace(query, self.separator, ' ') feature = tf.Print( - feature, - [encode_q, encode_t, feature], - message=self.name, - first_n=self.print_first_n, - summarize=self.summarize, + feature, + [encode_q, encode_t, feature], + message=self.name, + first_n=self.print_first_n, + summarize=self.summarize, ) if self.norm_fn is not None: fn = eval(self.norm_fn) @@ -133,11 +142,11 @@ def call(self, inputs, training=None, **kwargs): tf.summary.scalar('normalized_%s' % self.name, tf.reduce_mean(feature)) if self.print_first_n: feature = tf.Print( - feature, - [feature], - message='normalized %s' % self.name, - first_n=self.print_first_n, - summarize=self.summarize, + feature, + [feature], + message='normalized %s' % self.name, + first_n=self.print_first_n, + summarize=self.summarize, ) if self.boundaries: feature = self.bucketize(feature, boundaries=self.boundaries) @@ -150,6 +159,7 @@ def call(self, inputs, training=None, **kwargs): class OverlapFeature(Layer): + def __init__(self, params, name='overlap_feature', reuse=None, **kwargs): super(OverlapFeature, self).__init__(name=name, **kwargs) self.overlap_feature = custom_ops.overlap_fg_op @@ -168,23 +178,23 @@ def __init__(self, params, name='overlap_feature', reuse=None, **kwargs): vocab_size *= len(self.methods) with tf.variable_scope(self.name, reuse=reuse): self.embedding_table = tf.get_variable( - name='overlap_emb_table', - shape=[vocab_size, self.emb_dim], - dtype=tf.float32, + name='overlap_emb_table', + shape=[vocab_size, self.emb_dim], + dtype=tf.float32, ) def call(self, inputs, training=None, **kwargs): query, title = inputs[:2] with ops.device('/CPU:0'): feature = self.overlap_feature( - query=query, - title=title, - feature_name=self.name, - separator=self.separator, - default_value=self.default_value, - boundaries=self.boundaries, - methods=self.methods, - dtype=tf.int32 if self.boundaries else tf.float32, + query=query, + title=title, + feature_name=self.name, + separator=self.separator, + default_value=self.default_value, + boundaries=self.boundaries, + methods=self.methods, + dtype=tf.int32 if self.boundaries else tf.float32, ) for i, method in enumerate(self.methods): @@ -197,11 +207,11 @@ def call(self, inputs, training=None, **kwargs): encode_q = tf.regex_replace(query, self.separator, ' ') encode_t = tf.regex_replace(query, self.separator, ' ') feature = tf.Print( - feature, - [encode_q, encode_t, feature], - message=self.name, - first_n=self.print_first_n, - summarize=self.summarize, + feature, + [encode_q, encode_t, feature], + message=self.name, + first_n=self.print_first_n, + summarize=self.summarize, ) if self.norm_fn is not None: fn = eval(self.norm_fn) @@ -216,17 +226,20 @@ def call(self, inputs, training=None, **kwargs): # Compute offsets, add to every column indices offsets = tf.range(num_indices) * vocab_size # Shape: [3] offsets = tf.reshape(offsets, [1, num_indices]) # Shape: [1, 3] - offsets = tf.tile(offsets, [batch_size, 1]) # Shape: [batch_size, num_indices] + offsets = tf.tile(offsets, + [batch_size, 1]) # Shape: [batch_size, num_indices] shifted_indices = feature + offsets # Shape: [batch_size, num_indices] flat_feature_ids = tf.reshape(shifted_indices, [-1]) one_hot_ids = tf.one_hot(flat_feature_ids, depth=vocab_size * num_indices) feature_embeddings = tf.matmul(one_hot_ids, self.embedding_table) - feature_embeddings = tf.reshape(feature_embeddings, [batch_size, num_indices * self.emb_dim]) + feature_embeddings = tf.reshape(feature_embeddings, + [batch_size, num_indices * self.emb_dim]) return feature_embeddings return feature class EditDistance(Layer): + def __init__(self, params, name='edit_distance', reuse=None, **kwargs): super(EditDistance, self).__init__(name=name, **kwargs) self.edit_distance = custom_ops.my_edit_distance @@ -234,17 +247,19 @@ def __init__(self, params, name='edit_distance', reuse=None, **kwargs): self.emb_size = params.get_or_default('embedding_size', 512) emb_dim = params.get_or_default('embedding_dim', 4) with tf.variable_scope(self.name, reuse=reuse): - self.embedding_table = tf.get_variable('embedding_table', [self.emb_size, emb_dim], tf.float32) + self.embedding_table = tf.get_variable('embedding_table', + [self.emb_size, emb_dim], + tf.float32) def call(self, inputs, training=None, **kwargs): input1, input2 = inputs[:2] with ops.device('/CPU:0'): dist = self.edit_distance( - input1, - input2, - normalize=False, - dtype=tf.int32, - encoding=self.txt_encoding, + input1, + input2, + normalize=False, + dtype=tf.int32, + encoding=self.txt_encoding, ) ids = tf.clip_by_value(dist, 0, self.emb_size - 1) embed = tf.nn.embedding_lookup(self.embedding_table, ids) diff --git a/easy_rec/python/layers/keras/data_augment.py b/easy_rec/python/layers/keras/data_augment.py index d10c60999..93674d130 100644 --- a/easy_rec/python/layers/keras/data_augment.py +++ b/easy_rec/python/layers/keras/data_augment.py @@ -30,15 +30,18 @@ def item_crop(aug_data, length, crop_rate): max_length = tf.cast(max_len, dtype=tf.int32) num_left = tf.cast(tf.math.floor(length1 * crop_rate), dtype=tf.int32) - crop_begin = tf.random.uniform([], minval=0, maxval=length - num_left, dtype=tf.int32) + crop_begin = tf.random.uniform([], + minval=0, + maxval=length - num_left, + dtype=tf.int32) zeros = tf.zeros_like(aug_data) - x = aug_data[crop_begin : crop_begin + num_left] - y = zeros[: max_length - num_left] + x = aug_data[crop_begin:crop_begin + num_left] + y = zeros[:max_length - num_left] cropped = tf.concat([x, y], axis=0) cropped_item_seq = tf.where( - crop_begin + num_left < max_length, - cropped, - tf.concat([aug_data[crop_begin:], zeros[:crop_begin]], axis=0), + crop_begin + num_left < max_length, + cropped, + tf.concat([aug_data[crop_begin:], zeros[:crop_begin]], axis=0), ) return cropped_item_seq, num_left @@ -46,14 +49,19 @@ def item_crop(aug_data, length, crop_rate): def item_reorder(aug_data, length, reorder_rate): length1 = tf.cast(length, dtype=tf.float32) num_reorder = tf.cast(tf.math.floor(length1 * reorder_rate), dtype=tf.int32) - reorder_begin = tf.random.uniform([], minval=0, maxval=length - num_reorder, dtype=tf.int32) + reorder_begin = tf.random.uniform([], + minval=0, + maxval=length - num_reorder, + dtype=tf.int32) shuffle_index = tf.range(reorder_begin, reorder_begin + num_reorder) shuffle_index = tf.random.shuffle(shuffle_index) x = tf.range(get_shape_list(aug_data)[0]) left = tf.slice(x, [0], [reorder_begin]) right = tf.slice(x, [reorder_begin + num_reorder], [-1]) reordered_item_index = tf.concat([left, shuffle_index, right], axis=0) - reordered_item_seq = tf.scatter_nd(tf.expand_dims(reordered_item_index, axis=1), aug_data, tf.shape(aug_data)) + reordered_item_seq = tf.scatter_nd( + tf.expand_dims(reordered_item_index, axis=1), aug_data, + tf.shape(aug_data)) return reordered_item_seq, length @@ -89,9 +97,9 @@ def reorder_fn(): return tf.cond(tf.equal(method, 0), trans_fn[0], trans_fn[1]) aug_seq, aug_len = tf.cond( - tf.equal(method, 0), - crop_fn, - lambda: tf.cond(tf.equal(method, 1), mask_fn, reorder_fn), + tf.equal(method, 0), + crop_fn, + lambda: tf.cond(tf.equal(method, 1), mask_fn, reorder_fn), ) return aug_seq, aug_len @@ -99,9 +107,9 @@ def reorder_fn(): def sequence_augment(seq_input, seq_len, mask, aug_param): lengths = tf.cast(seq_len, dtype=tf.int32) aug_seq, aug_len = tf.map_fn( - lambda elems: augment_fn(elems, aug_param, mask), - elems=(seq_input, lengths), - dtype=(tf.float32, tf.int32), + lambda elems: augment_fn(elems, aug_param, mask), + elems=(seq_input, lengths), + dtype=(tf.float32, tf.int32), ) aug_seq = tf.reshape(aug_seq, tf.shape(seq_input)) @@ -122,7 +130,9 @@ def call(self, inputs, training=None, **kwargs): embedding_size = int(seq_input.shape[-1]) with tf.variable_scope(self.name, reuse=self.reuse): - mask_emb = tf.get_variable('mask', [1, embedding_size], dtype=tf.float32, trainable=True) + mask_emb = tf.get_variable( + 'mask', [1, embedding_size], dtype=tf.float32, trainable=True) - aug_seq, aug_len = sequence_augment(seq_input, seq_len, mask_emb, self.seq_aug_params) + aug_seq, aug_len = sequence_augment(seq_input, seq_len, mask_emb, + self.seq_aug_params) return aug_seq, aug_len diff --git a/easy_rec/python/layers/keras/din.py b/easy_rec/python/layers/keras/din.py index 003c63f1a..1f671b80f 100644 --- a/easy_rec/python/layers/keras/din.py +++ b/easy_rec/python/layers/keras/din.py @@ -11,6 +11,7 @@ class DIN(Layer): + def __init__(self, params, name='din', reuse=None, **kwargs): super(DIN, self).__init__(name=name, **kwargs) self.reuse = reuse @@ -30,10 +31,11 @@ def call(self, inputs, training=None, **kwargs): seq_emb_size = keys.shape.as_list()[-1] if query_emb_size != seq_emb_size: logging.info( - ' the embedding size of sequence [%d] and target item [%d] is not equal' ' in feature group: %s', - seq_emb_size, - query_emb_size, - self.name, + ' the embedding size of sequence [%d] and target item [%d] is not equal' + ' in feature group: %s', + seq_emb_size, + query_emb_size, + self.name, ) if query_emb_size < seq_emb_size: query = tf.pad(query, [[0, 0], [0, seq_emb_size - query_emb_size]]) @@ -42,7 +44,8 @@ def call(self, inputs, training=None, **kwargs): batch_size, max_seq_len, _ = get_shape_list(keys, 3) queries = tf.tile(tf.expand_dims(query, 1), [1, max_seq_len, 1]) - din_all = tf.concat([queries, keys, queries - keys, queries * keys], axis=-1) + din_all = tf.concat([queries, keys, queries - keys, queries * keys], + axis=-1) output = self.din_layer(din_all, training) # [B, L, 1] scores = tf.transpose(output, [0, 2, 1]) # [B, 1, L] @@ -56,7 +59,8 @@ def call(self, inputs, training=None, **kwargs): scores = scores / (seq_emb_size**0.5) scores = tf.nn.sigmoid(scores) else: - raise ValueError('unsupported attention normalizer: ' + self.config.attention_normalizer) + raise ValueError('unsupported attention normalizer: ' + + self.config.attention_normalizer) if query_emb_size < seq_emb_size: keys = keys[:, :, :query_emb_size] # [B, L, E] diff --git a/easy_rec/python/layers/keras/einsum_dense.py b/easy_rec/python/layers/keras/einsum_dense.py index e22f312c4..eab9aabf3 100644 --- a/easy_rec/python/layers/keras/einsum_dense.py +++ b/easy_rec/python/layers/keras/einsum_dense.py @@ -4,13 +4,11 @@ import string import tensorflow as tf +from tensorflow.python.keras.layers import Layer + from tensorflow.python.keras import ( # NOQA - activations, - constraints, - initializers, - regularizers, + activations, constraints, initializers, regularizers, ) -from tensorflow.python.keras.layers import Layer class EinsumDense(Layer): @@ -108,19 +106,19 @@ class EinsumDense(Layer): """ def __init__( - self, - equation, - output_shape, - activation=None, - bias_axes=None, - kernel_initializer='glorot_uniform', - bias_initializer='zeros', - kernel_regularizer=None, - bias_regularizer=None, - kernel_constraint=None, - bias_constraint=None, - lora_rank=None, - **kwargs, + self, + equation, + output_shape, + activation=None, + bias_axes=None, + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + lora_rank=None, + **kwargs, ): super(EinsumDense, self).__init__(**kwargs) self.equation = equation @@ -141,10 +139,10 @@ def __init__( def build(self, input_shape): shape_data = _analyze_einsum_string( - self.equation, - self.bias_axes, - input_shape, - self.partial_output_shape, + self.equation, + self.bias_axes, + input_shape, + self.partial_output_shape, ) kernel_shape, bias_shape, full_output_shape = shape_data for i in range(len(kernel_shape)): @@ -161,23 +159,23 @@ def build(self, input_shape): full_output_shape[i] = dim.value self.full_output_shape = tuple(full_output_shape) self._kernel = self.add_weight( - name='kernel', - shape=tuple(kernel_shape), - initializer=self.kernel_initializer, - regularizer=self.kernel_regularizer, - constraint=self.kernel_constraint, - dtype=self.dtype, - trainable=True, + name='kernel', + shape=tuple(kernel_shape), + initializer=self.kernel_initializer, + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + dtype=self.dtype, + trainable=True, ) if bias_shape is not None: self.bias = self.add_weight( - name='bias', - shape=tuple(bias_shape), - initializer=self.bias_initializer, - regularizer=self.bias_regularizer, - constraint=self.bias_constraint, - dtype=self.dtype, - trainable=True, + name='bias', + shape=tuple(bias_shape), + initializer=self.bias_initializer, + regularizer=self.bias_regularizer, + constraint=self.bias_constraint, + dtype=self.dtype, + trainable=True, ) else: self.bias = None @@ -188,7 +186,8 @@ def build(self, input_shape): @property def kernel(self): if not self.built: - raise AttributeError('You must build the layer before accessing `kernel`.') + raise AttributeError( + 'You must build the layer before accessing `kernel`.') if self.lora_enabled: return self._kernel + tf.matmul(self.lora_kernel_a, self.lora_kernel_b) return self._kernel @@ -204,29 +203,31 @@ def call(self, inputs, training=None): x = self.activation(x) return x - def enable_lora(self, rank, a_initializer='he_uniform', b_initializer='zeros'): + def enable_lora(self, + rank, + a_initializer='he_uniform', + b_initializer='zeros'): if self.kernel_constraint: - raise ValueError( - 'Lora is incompatible with kernel constraints. ' - 'In order to enable lora on this layer, remove the ' - '`kernel_constraint` argument.' - ) + raise ValueError('Lora is incompatible with kernel constraints. ' + 'In order to enable lora on this layer, remove the ' + '`kernel_constraint` argument.') if not self.built: raise ValueError("Cannot enable lora on a layer that isn't yet built.") if self.lora_enabled: - raise ValueError('lora is already enabled. ' 'This can only be done once per layer.') + raise ValueError('lora is already enabled. ' + 'This can only be done once per layer.') self._tracker.unlock() self.lora_kernel_a = self.add_weight( - name='lora_kernel_a', - shape=(self.kernel.shape[:-1] + (rank,)), - initializer=initializers.get(a_initializer), - regularizer=self.kernel_regularizer, + name='lora_kernel_a', + shape=(self.kernel.shape[:-1] + (rank,)), + initializer=initializers.get(a_initializer), + regularizer=self.kernel_regularizer, ) self.lora_kernel_b = self.add_weight( - name='lora_kernel_b', - shape=(rank, self.kernel.shape[-1]), - initializer=initializers.get(b_initializer), - regularizer=self.kernel_regularizer, + name='lora_kernel_b', + shape=(rank, self.kernel.shape[-1]), + initializer=initializers.get(b_initializer), + regularizer=self.kernel_regularizer, ) self._kernel.trainable = False self._tracker.lock() @@ -266,17 +267,28 @@ def load_own_variables(self, store): def get_config(self): base_config = super(EinsumDense, self).get_config() config = { - 'output_shape': self.partial_output_shape, - 'equation': self.equation, - 'activation': activations.serialize(self.activation), - 'bias_axes': self.bias_axes, - 'kernel_initializer': initializers.serialize(self.kernel_initializer), - 'bias_initializer': initializers.serialize(self.bias_initializer), - 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), - 'bias_regularizer': regularizers.serialize(self.bias_regularizer), - 'activity_regularizer': regularizers.serialize(self.activity_regularizer), - 'kernel_constraint': constraints.serialize(self.kernel_constraint), - 'bias_constraint': constraints.serialize(self.bias_constraint), + 'output_shape': + self.partial_output_shape, + 'equation': + self.equation, + 'activation': + activations.serialize(self.activation), + 'bias_axes': + self.bias_axes, + 'kernel_initializer': + initializers.serialize(self.kernel_initializer), + 'bias_initializer': + initializers.serialize(self.bias_initializer), + 'kernel_regularizer': + regularizers.serialize(self.kernel_regularizer), + 'bias_regularizer': + regularizers.serialize(self.bias_regularizer), + 'activity_regularizer': + regularizers.serialize(self.activity_regularizer), + 'kernel_constraint': + constraints.serialize(self.kernel_constraint), + 'bias_constraint': + constraints.serialize(self.bias_constraint), } if self.lora_rank: config['lora_rank'] = self.lora_rank @@ -288,38 +300,37 @@ def _check_load_own_variables(self, store): if len(store.keys()) != len(all_vars): if len(all_vars) == 0 and not self.built: raise ValueError( - "Layer '{name}' was never built " - "and thus it doesn't have any variables. " - 'However the weights file lists {num_keys} ' - 'variables for this layer.\n' - 'In most cases, this error indicates that either:\n\n' - '1. The layer is owned by a parent layer that ' - 'implements a `build()` method, but calling the ' - "parent's `build()` method did NOT create the state of " - "the child layer '{name}'. A `build()` method " - 'must create ALL state for the layer, including ' - 'the state of any children layers.\n\n' - '2. You need to implement ' - 'the `def build_from_config(self, config)` method ' - "on layer '{name}', to specify how to rebuild " - 'it during loading. ' - 'In this case, you might also want to implement the ' - 'method that generates the build config at saving time, ' - '`def get_build_config(self)`. ' - 'The method `build_from_config()` is meant ' - 'to create the state ' - 'of the layer (i.e. its variables) upon deserialization.'.format(name=self.name, num_keys=len(store.keys())) - ) + "Layer '{name}' was never built " + "and thus it doesn't have any variables. " + 'However the weights file lists {num_keys} ' + 'variables for this layer.\n' + 'In most cases, this error indicates that either:\n\n' + '1. The layer is owned by a parent layer that ' + 'implements a `build()` method, but calling the ' + "parent's `build()` method did NOT create the state of " + "the child layer '{name}'. A `build()` method " + 'must create ALL state for the layer, including ' + 'the state of any children layers.\n\n' + '2. You need to implement ' + 'the `def build_from_config(self, config)` method ' + "on layer '{name}', to specify how to rebuild " + 'it during loading. ' + 'In this case, you might also want to implement the ' + 'method that generates the build config at saving time, ' + '`def get_build_config(self)`. ' + 'The method `build_from_config()` is meant ' + 'to create the state ' + 'of the layer (i.e. its variables) upon deserialization.'.format( + name=self.name, num_keys=len(store.keys()))) raise ValueError( - "Layer '{name}' expected {num_var} variables, but received " - '{num_key} variables during loading. ' - 'Expected: {names}'.format( - name=self.name, - num_var=len(store.keys()), - num_key=len(store.keys()), - names=[v.name for v in all_vars], - ) - ) + "Layer '{name}' expected {num_var} variables, but received " + '{num_key} variables during loading. ' + 'Expected: {names}'.format( + name=self.name, + num_var=len(store.keys()), + num_key=len(store.keys()), + names=[v.name for v in all_vars], + )) def _get_kernel_with_merged_lora(self): kernel_value = self.kernel @@ -332,27 +343,37 @@ def _analyze_einsum_string(equation, bias_axes, input_shape, output_shape): dot_replaced_string = re.sub(r'\.\.\.', '0', equation) # This is the case where no ellipses are present in the string. - split_string = re.match('([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', dot_replaced_string) + split_string = re.match('([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', + dot_replaced_string) if split_string: - return _analyze_split_string(split_string, bias_axes, input_shape, output_shape) + return _analyze_split_string(split_string, bias_axes, input_shape, + output_shape) # This is the case where ellipses are present on the left. - split_string = re.match('0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', dot_replaced_string) + split_string = re.match('0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', + dot_replaced_string) if split_string: - return _analyze_split_string(split_string, bias_axes, input_shape, output_shape, left_elided=True) + return _analyze_split_string( + split_string, bias_axes, input_shape, output_shape, left_elided=True) # This is the case where ellipses are present on the right. - split_string = re.match('([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', dot_replaced_string) + split_string = re.match('([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', + dot_replaced_string) if split_string: - return _analyze_split_string(split_string, bias_axes, input_shape, output_shape) + return _analyze_split_string(split_string, bias_axes, input_shape, + output_shape) raise ValueError( - "Invalid einsum equation '{equation}'. Equations must be in the form " - '[X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format(equation=equation) - ) + "Invalid einsum equation '{equation}'. Equations must be in the form " + '[X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format( + equation=equation)) -def _analyze_split_string(split_string, bias_axes, input_shape, output_shape, left_elided=False): +def _analyze_split_string(split_string, + bias_axes, + input_shape, + output_shape, + left_elided=False): """Analyze an pre-split einsum string to find the weight shape.""" input_spec = split_string.group(1) weight_spec = split_string.group(2) @@ -377,7 +398,9 @@ def _analyze_split_string(split_string, bias_axes, input_shape, output_shape, le if left_elided: # If we have beginning dimensions elided, we need to use negative # indexing to determine where in the input dimension our values are. - input_dim_map = {dim: (i + elided) - len(input_shape) for i, dim in enumerate(input_spec)} + input_dim_map = { + dim: (i + elided) - len(input_shape) for i, dim in enumerate(input_spec) + } # Because we've constructed the full output shape already, we don't need # to do negative indexing. output_dim_map = {dim: (i + elided) for i, dim in enumerate(output_spec)} @@ -391,24 +414,21 @@ def _analyze_split_string(split_string, bias_axes, input_shape, output_shape, le output_shape_at_dim = output_shape[output_dim_map[dim]] if output_shape_at_dim is not None and output_shape_at_dim != input_shape_at_dim: raise ValueError( - 'Input shape and output shape do not match at shared ' - "dimension '{dim}'. Input shape is {input_shape_at_dim}, " - 'and output shape is {output_shape}.'.format( - dim=dim, - input_shape_at_dim=input_shape_at_dim, - output_shape=output_shape[output_dim_map[dim]], - ) - ) + 'Input shape and output shape do not match at shared ' + "dimension '{dim}'. Input shape is {input_shape_at_dim}, " + 'and output shape is {output_shape}.'.format( + dim=dim, + input_shape_at_dim=input_shape_at_dim, + output_shape=output_shape[output_dim_map[dim]], + )) for dim in output_spec: if dim not in input_spec and dim not in weight_spec: raise ValueError( - "Dimension '{dim}' was specified in the output " - "'{output_spec}' but has no corresponding dim in the input " - "spec '{input_spec}' or weight spec '{output_spec}'".format( - dim=dim, output_spec=output_spec, input_spec=input_spec - ) - ) + "Dimension '{dim}' was specified in the output " + "'{output_spec}' but has no corresponding dim in the input " + "spec '{input_spec}' or weight spec '{output_spec}'".format( + dim=dim, output_spec=output_spec, input_spec=input_spec)) weight_shape = [] for dim in weight_spec: @@ -418,28 +438,32 @@ def _analyze_split_string(split_string, bias_axes, input_shape, output_shape, le weight_shape.append(output_shape[output_dim_map[dim]]) else: raise ValueError( - "Weight dimension '{dim}' did not have a match in either " - "the input spec '{input_spec}' or the output " - "spec '{output_spec}'. For this layer, the weight must " - 'be fully specified.'.format(dim=dim, input_spec=input_spec, output_spec=output_spec) - ) + "Weight dimension '{dim}' did not have a match in either " + "the input spec '{input_spec}' or the output " + "spec '{output_spec}'. For this layer, the weight must " + 'be fully specified.'.format( + dim=dim, input_spec=input_spec, output_spec=output_spec)) if bias_axes is not None: num_left_elided = elided if left_elided else 0 - idx_map = {char: output_shape[i + num_left_elided] for i, char in enumerate(output_spec)} + idx_map = { + char: output_shape[i + num_left_elided] + for i, char in enumerate(output_spec) + } for char in bias_axes: if char not in output_spec: raise ValueError( - "Bias dimension '{char}' was requested, but is not part " "of the output spec '{output_spec}'".format( - char=char, output_spec=output_spec - ) - ) + "Bias dimension '{char}' was requested, but is not part " + "of the output spec '{output_spec}'".format( + char=char, output_spec=output_spec)) first_bias_location = min([output_spec.find(char) for char in bias_axes]) bias_output_spec = output_spec[first_bias_location:] - bias_shape = [idx_map[char] if char in bias_axes else 1 for char in bias_output_spec] + bias_shape = [ + idx_map[char] if char in bias_axes else 1 for char in bias_output_spec + ] if not left_elided: for _ in range(elided): @@ -451,12 +475,14 @@ def _analyze_split_string(split_string, bias_axes, input_shape, output_shape, le def _analyze_quantization_info(equation, input_shape): + def get_specs(equation, input_shape): possible_labels = string.ascii_letters dot_replaced_string = re.sub(r'\.\.\.', '0', equation) # This is the case where no ellipses are present in the string. - split_string = re.match('([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', dot_replaced_string) + split_string = re.match('([a-zA-Z]+),([a-zA-Z]+)->([a-zA-Z]+)', + dot_replaced_string) if split_string is not None: input_spec = split_string.group(1) weight_spec = split_string.group(2) @@ -464,13 +490,16 @@ def get_specs(equation, input_shape): return input_spec, weight_spec, output_spec # This is the case where ellipses are present on the left. - split_string = re.match('0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', dot_replaced_string) + split_string = re.match('0([a-zA-Z]+),([a-zA-Z]+)->0([a-zA-Z]+)', + dot_replaced_string) if split_string is not None: input_spec = split_string.group(1) weight_spec = split_string.group(2) output_spec = split_string.group(3) elided = len(input_shape) - len(input_spec) - possible_labels = sorted(set(possible_labels) - set(input_spec) - set(weight_spec) - set(output_spec)) + possible_labels = sorted( + set(possible_labels) - set(input_spec) - set(weight_spec) - + set(output_spec)) # Pad labels on the left to `input_spec` and `output_spec` for i in range(elided): input_spec = possible_labels[i] + input_spec @@ -478,13 +507,16 @@ def get_specs(equation, input_shape): return input_spec, weight_spec, output_spec # This is the case where ellipses are present on the right. - split_string = re.match('([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', dot_replaced_string) + split_string = re.match('([a-zA-Z]{2,})0,([a-zA-Z]+)->([a-zA-Z]+)0', + dot_replaced_string) if split_string is not None: input_spec = split_string.group(1) weight_spec = split_string.group(2) output_spec = split_string.group(3) elided = len(input_shape) - len(input_spec) - possible_labels = sorted(set(possible_labels) - set(input_spec) - set(weight_spec) - set(output_spec)) + possible_labels = sorted( + set(possible_labels) - set(input_spec) - set(weight_spec) - + set(output_spec)) # Pad labels on the right to `input_spec` and `output_spec` for i in range(elided): input_spec = input_spec + possible_labels[i] @@ -492,9 +524,9 @@ def get_specs(equation, input_shape): return input_spec, weight_spec, output_spec raise ValueError( - "Invalid einsum equation '{equation}'. Equations must be in the " - 'form [X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format(equation=equation) - ) + "Invalid einsum equation '{equation}'. Equations must be in the " + 'form [X],[Y]->[Z], ...[X],[Y]->...[Z], or [X]...,[Y]->[Z]....'.format( + equation=equation)) input_spec, weight_spec, output_spec = get_specs(equation, input_shape) @@ -550,18 +582,20 @@ def get_specs(equation, input_shape): weight_transpose_axes.insert(index, ori_index) # Prepare equation for `einsum_with_inputs_gradient` custom_gradient_equation = '{output_spec},{weight_spec}->{input_spec}'.format( - output_spec=output_spec, input_spec=input_spec, weight_spec=weight_spec - ) - weight_reverse_transpose_axes = [i for (_, i) in sorted((v, i) for (i, v) in enumerate(weight_transpose_axes))] + output_spec=output_spec, input_spec=input_spec, weight_spec=weight_spec) + weight_reverse_transpose_axes = [ + i for (_, i) in sorted((v, i) + for (i, v) in enumerate(weight_transpose_axes)) + ] return ( - input_reduced_axes, - weight_reduced_axes, - input_transpose_axes, - weight_transpose_axes, - input_expand_axes, - weight_expand_axes, - input_squeeze_axes, - weight_squeeze_axes, - custom_gradient_equation, - weight_reverse_transpose_axes, + input_reduced_axes, + weight_reduced_axes, + input_transpose_axes, + weight_transpose_axes, + input_expand_axes, + weight_expand_axes, + input_squeeze_axes, + weight_squeeze_axes, + custom_gradient_equation, + weight_reverse_transpose_axes, ) diff --git a/easy_rec/python/layers/keras/embedding.py b/easy_rec/python/layers/keras/embedding.py index 661036f82..5ce61fa4a 100644 --- a/easy_rec/python/layers/keras/embedding.py +++ b/easy_rec/python/layers/keras/embedding.py @@ -3,7 +3,8 @@ """Fused embedding layer.""" import tensorflow as tf -from tensorflow.python.keras.layers import Embedding, Layer +from tensorflow.python.keras.layers import Embedding +from tensorflow.python.keras.layers import Layer def _combine(embeddings, weights, comb_fn): @@ -24,6 +25,7 @@ def _combine(embeddings, weights, comb_fn): class EmbeddingLayer(Layer): + def __init__(self, params, name='embedding_layer', reuse=None, **kwargs): super(EmbeddingLayer, self).__init__(name=name, **kwargs) params.check_required(['vocab_size', 'embedding_dim']) @@ -70,13 +72,13 @@ def call(self, inputs, training=None, **kwargs): if is_multi[i]: batch_size = tf.shape(inputs[i])[0] embeddings[i] = tf.cond( - tf.equal(tf.size(embeddings[i]), 0), - lambda: tf.zeros([batch_size, self.embed_dim]), - lambda: _combine( - tf.reshape(embeddings[i], [batch_size, -1, self.embed_dim]), - weights[i], - self.combine_fn, - ), + tf.equal(tf.size(embeddings[i]), 0), + lambda: tf.zeros([batch_size, self.embed_dim]), + lambda: _combine( + tf.reshape(embeddings[i], [batch_size, -1, self.embed_dim]), + weights[i], + self.combine_fn, + ), ) if self.do_concat: embeddings = tf.concat(embeddings, axis=-1) diff --git a/easy_rec/python/layers/keras/fibinet.py b/easy_rec/python/layers/keras/fibinet.py index 72cf64383..23c4b6729 100644 --- a/easy_rec/python/layers/keras/fibinet.py +++ b/easy_rec/python/layers/keras/fibinet.py @@ -4,7 +4,8 @@ import logging import tensorflow as tf -from tensorflow.python.keras.layers import Dense, Layer +from tensorflow.python.keras.layers import Dense +from tensorflow.python.keras.layers import Layer from easy_rec.python.layers.keras.blocks import MLP from easy_rec.python.layers.keras.layer_norm import LayerNormalization @@ -43,19 +44,21 @@ def build(self, input_shape): for shape in input_shape: assert shape.ndims == 2, 'field embeddings must be rank 2 tensors' dim = int(shape[-1]) - assert dim >= g and dim % g == 0, 'field embedding dimension %d must be divisible by %d' % (dim, g) + assert dim >= g and dim % g == 0, 'field embedding dimension %d must be divisible by %d' % ( + dim, g) emb_size += dim r = self.config.reduction_ratio field_size = len(input_shape) reduction_size = max(1, field_size * g * 2 // r) self.reduce_layer = Dense( - units=reduction_size, - activation='relu', - kernel_initializer='he_normal', - name='W1', + units=reduction_size, + activation='relu', + kernel_initializer='he_normal', + name='W1', ) - self.excite_layer = Dense(units=emb_size, kernel_initializer='glorot_normal', name='W2') + self.excite_layer = Dense( + units=emb_size, kernel_initializer='glorot_normal', name='W2') super(SENet, self).build(input_shape) # Be sure to call this somewhere! def call(self, inputs, **kwargs): @@ -63,7 +66,9 @@ def call(self, inputs, **kwargs): # Squeeze # embedding dimension 必须能被 g 整除 - group_embs = [tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs] + group_embs = [ + tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs + ] squeezed = [] for emb in group_embs: @@ -91,7 +96,8 @@ def call(self, inputs, **kwargs): def _full_interaction(v_i, v_j): # [bs, 1, dim] x [bs, dim, 1] = [bs, 1] - interaction = tf.matmul(tf.expand_dims(v_i, axis=1), tf.expand_dims(v_j, axis=-1)) + interaction = tf.matmul( + tf.expand_dims(v_i, axis=1), tf.expand_dims(v_j, axis=-1)) return tf.squeeze(interaction, axis=1) @@ -126,7 +132,8 @@ def __init__(self, params, name='bilinear', reuse=None, **kwargs): self.output_size = params.num_output_units self.bilinear_type = params.get_or_default('type', 'interaction').lower() if self.bilinear_type not in ['all', 'each', 'interaction']: - raise NotImplementedError("bilinear_type only support: ['all', 'each', 'interaction']") + raise NotImplementedError( + "bilinear_type only support: ['all', 'each', 'interaction']") if bilinear_plus: self.func = _full_interaction else: @@ -147,17 +154,22 @@ def build(self, input_shape): if shape[-1] != _dim: equal_dim = False if not equal_dim and self.bilinear_type != 'interaction': - raise ValueError('all embedding dimensions must be same when not use bilinear type: interaction') + raise ValueError( + 'all embedding dimensions must be same when not use bilinear type: interaction' + ) dim = int(_dim) if self.bilinear_type == 'all': self.dot_layer = Dense(dim, name='all') elif self.bilinear_type == 'each': - self.dot_layers = [Dense(dim, name='each_%d' % i) for i in range(field_num - 1)] + self.dot_layers = [ + Dense(dim, name='each_%d' % i) for i in range(field_num - 1) + ] else: # interaction self.dot_layers = [ - Dense(units=int(input_shape[j][-1]), name='interaction_%d_%d' % (i, j)) - for i, j in itertools.combinations(range(field_num), 2) + Dense( + units=int(input_shape[j][-1]), name='interaction_%d_%d' % (i, j)) + for i, j in itertools.combinations(range(field_num), 2) ] super(BiLinear, self).build(input_shape) # Be sure to call this somewhere! @@ -172,14 +184,21 @@ def call(self, inputs, **kwargs): # - where k_i is the embedding size of the ith field if self.bilinear_type == 'all': v_dot = [self.dot_layer(v_i) for v_i in embeddings[:-1]] - p = [self.func(v_dot[i], embeddings[j]) for i, j in itertools.combinations(range(field_num), 2)] + p = [ + self.func(v_dot[i], embeddings[j]) + for i, j in itertools.combinations(range(field_num), 2) + ] elif self.bilinear_type == 'each': v_dot = [self.dot_layers[i](v_i) for i, v_i in enumerate(embeddings[:-1])] - p = [self.func(v_dot[i], embeddings[j]) for i, j in itertools.combinations(range(field_num), 2)] + p = [ + self.func(v_dot[i], embeddings[j]) + for i, j in itertools.combinations(range(field_num), 2) + ] else: # interaction p = [ - self.func(self.dot_layers[i * field_num + j](embeddings[i]), embeddings[j]) - for i, j in itertools.combinations(range(field_num), 2) + self.func(self.dot_layers[i * field_num + j](embeddings[i]), + embeddings[j]) + for i, j in itertools.combinations(range(field_num), 2) ] return self.output_layer(tf.concat(p, axis=-1)) @@ -199,11 +218,13 @@ def __init__(self, params, name='fibinet', reuse=None, **kwargs): self._config = params.get_pb_config() se_params = Parameter.make_from_pb(self._config.senet) - self.senet_layer = SENet(se_params, name=self.name + '/senet', reuse=self.reuse) + self.senet_layer = SENet( + se_params, name=self.name + '/senet', reuse=self.reuse) if self._config.HasField('bilinear'): bi_params = Parameter.make_from_pb(self._config.bilinear) - self.bilinear_layer = BiLinear(bi_params, name=self.name + '/bilinear', reuse=self.reuse) + self.bilinear_layer = BiLinear( + bi_params, name=self.name + '/bilinear', reuse=self.reuse) if self._config.HasField('mlp'): p = Parameter.make_from_pb(self._config.mlp) diff --git a/easy_rec/python/layers/keras/interaction.py b/easy_rec/python/layers/keras/interaction.py index b3d53ade7..6f177e0db 100644 --- a/easy_rec/python/layers/keras/interaction.py +++ b/easy_rec/python/layers/keras/interaction.py @@ -92,7 +92,8 @@ def call(self, inputs, **kwargs): try: concat_features = tf.stack(inputs, axis=1) except (ValueError, tf.errors.InvalidArgumentError) as e: - raise ValueError('Input tensors` dimensions must be equal, original' 'error message: {}'.format(e)) + raise ValueError('Input tensors` dimensions must be equal, original' + 'error message: {}'.format(e)) else: assert inputs.shape.ndims == 3, 'input of dot func must be a 3D tensor or a list of 2D tensors' concat_features = inputs @@ -117,9 +118,9 @@ def call(self, inputs, **kwargs): if self._skip_gather: # Setting upper triangle part of the interaction matrix to zeros. activations = tf.where( - condition=tf.cast(upper_tri_mask, tf.bool), - x=tf.zeros_like(xactions), - y=xactions, + condition=tf.cast(upper_tri_mask, tf.bool), + x=tf.zeros_like(xactions), + y=xactions, ) out_dim = num_features * num_features else: @@ -195,7 +196,8 @@ def __init__(self, params, name='cross', reuse=None, **kwargs): preactivation = params.get_or_default('preactivation', None) preact = get_activation(preactivation) self._preactivation = tf.keras.activations.get(preact) - kernel_initializer = params.get_or_default('kernel_initializer', 'truncated_normal') + kernel_initializer = params.get_or_default('kernel_initializer', + 'truncated_normal') self._kernel_initializer = tf.keras.initializers.get(kernel_initializer) bias_initializer = params.get_or_default('bias_initializer', 'zeros') self._bias_initializer = tf.keras.initializers.get(bias_initializer) @@ -207,39 +209,41 @@ def __init__(self, params, name='cross', reuse=None, **kwargs): self._supports_masking = True if self._diag_scale < 0: # pytype: disable=unsupported-operands - raise ValueError('`diag_scale` should be non-negative. Got `diag_scale` = {}'.format(self._diag_scale)) + raise ValueError( + '`diag_scale` should be non-negative. Got `diag_scale` = {}'.format( + self._diag_scale)) def build(self, input_shape): last_dim = input_shape[0][-1] if self._projection_dim is None: self._dense = tf.keras.layers.Dense( - last_dim, - kernel_initializer=_clone_initializer(self._kernel_initializer), - bias_initializer=self._bias_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer, - use_bias=self._use_bias, - dtype=self.dtype, - activation=self._preactivation, + last_dim, + kernel_initializer=_clone_initializer(self._kernel_initializer), + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + use_bias=self._use_bias, + dtype=self.dtype, + activation=self._preactivation, ) else: self._dense_u = tf.keras.layers.Dense( - self._projection_dim, - kernel_initializer=_clone_initializer(self._kernel_initializer), - kernel_regularizer=self._kernel_regularizer, - use_bias=False, - dtype=self.dtype, + self._projection_dim, + kernel_initializer=_clone_initializer(self._kernel_initializer), + kernel_regularizer=self._kernel_regularizer, + use_bias=False, + dtype=self.dtype, ) self._dense_v = tf.keras.layers.Dense( - last_dim, - kernel_initializer=_clone_initializer(self._kernel_initializer), - bias_initializer=self._bias_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer, - use_bias=self._use_bias, - dtype=self.dtype, - activation=self._preactivation, + last_dim, + kernel_initializer=_clone_initializer(self._kernel_initializer), + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + use_bias=self._use_bias, + dtype=self.dtype, + activation=self._preactivation, ) super(Cross, self).build(input_shape) # Be sure to call this somewhere! @@ -266,9 +270,9 @@ def call(self, inputs, **kwargs): if x0.shape[-1] != x.shape[-1]: raise ValueError( - '`x0` and `x` dimension mismatch! Got `x0` dimension {}, and x ' - 'dimension {}. This case is not supported yet.'.format(x0.shape[-1], x.shape[-1]) - ) + '`x0` and `x` dimension mismatch! Got `x0` dimension {}, and x ' + 'dimension {}. This case is not supported yet.'.format( + x0.shape[-1], x.shape[-1])) if self._projection_dim is None: prod_output = self._dense(x) @@ -284,14 +288,22 @@ def call(self, inputs, **kwargs): def get_config(self): config = { - 'projection_dim': self._projection_dim, - 'diag_scale': self._diag_scale, - 'use_bias': self._use_bias, - 'preactivation': tf.keras.activations.serialize(self._preactivation), - 'kernel_initializer': tf.keras.initializers.serialize(self._kernel_initializer), - 'bias_initializer': tf.keras.initializers.serialize(self._bias_initializer), - 'kernel_regularizer': tf.keras.regularizers.serialize(self._kernel_regularizer), - 'bias_regularizer': tf.keras.regularizers.serialize(self._bias_regularizer), + 'projection_dim': + self._projection_dim, + 'diag_scale': + self._diag_scale, + 'use_bias': + self._use_bias, + 'preactivation': + tf.keras.activations.serialize(self._preactivation), + 'kernel_initializer': + tf.keras.initializers.serialize(self._kernel_initializer), + 'bias_initializer': + tf.keras.initializers.serialize(self._bias_initializer), + 'kernel_regularizer': + tf.keras.regularizers.serialize(self._kernel_regularizer), + 'bias_regularizer': + tf.keras.regularizers.serialize(self._bias_regularizer), } base_config = super(Cross, self).get_config() return dict(list(base_config.items()) + list(config.items())) @@ -312,10 +324,12 @@ class CIN(tf.keras.layers.Layer): def __init__(self, params, name='cin', reuse=None, **kwargs): super(CIN, self).__init__(name=name, **kwargs) self._name = name - self._hidden_feature_sizes = list(params.get_or_default('hidden_feature_sizes', [])) + self._hidden_feature_sizes = list( + params.get_or_default('hidden_feature_sizes', [])) assert ( - isinstance(self._hidden_feature_sizes, list) and len(self._hidden_feature_sizes) > 0 + isinstance(self._hidden_feature_sizes, list) and + len(self._hidden_feature_sizes) > 0 ), 'parameter hidden_feature_sizes must be a list of int with length greater than 0' kernel_regularizer = params.get_or_default('kernel_regularizer', None) @@ -325,34 +339,35 @@ def __init__(self, params, name='cin', reuse=None, **kwargs): def build(self, input_shape): if len(input_shape) != 3: - raise ValueError('Unexpected inputs dimensions %d, expect to be 3 dimensions' % (len(input_shape))) + raise ValueError( + 'Unexpected inputs dimensions %d, expect to be 3 dimensions' % + (len(input_shape))) - hidden_feature_sizes = [input_shape[1]] + [h for h in self._hidden_feature_sizes] + hidden_feature_sizes = [input_shape[1] + ] + [h for h in self._hidden_feature_sizes] tfv1 = tf.compat.v1 if tf.__version__ >= '2.0' else tf with tfv1.variable_scope(self._name): self.kernel_list = [ - tfv1.get_variable( - name='cin_kernel_%d' % i, - shape=[ - hidden_feature_sizes[i + 1], - hidden_feature_sizes[i], - hidden_feature_sizes[0], - ], - initializer=tf.initializers.he_normal(), - regularizer=self._kernel_regularizer, - trainable=True, - ) - for i in range(len(self._hidden_feature_sizes)) + tfv1.get_variable( + name='cin_kernel_%d' % i, + shape=[ + hidden_feature_sizes[i + 1], + hidden_feature_sizes[i], + hidden_feature_sizes[0], + ], + initializer=tf.initializers.he_normal(), + regularizer=self._kernel_regularizer, + trainable=True, + ) for i in range(len(self._hidden_feature_sizes)) ] self.bias_list = [ - tfv1.get_variable( - name='cin_bias_%d' % i, - shape=[hidden_feature_sizes[i + 1]], - initializer=tf.keras.initializers.Zeros, - regularizer=self._bias_regularizer, - trainable=True, - ) - for i in range(len(self._hidden_feature_sizes)) + tfv1.get_variable( + name='cin_bias_%d' % i, + shape=[hidden_feature_sizes[i + 1]], + initializer=tf.keras.initializers.Zeros, + regularizer=self._bias_regularizer, + trainable=True, + ) for i in range(len(self._hidden_feature_sizes)) ] super(CIN, self).build(input_shape) @@ -379,23 +394,26 @@ def call(self, input, **kwargs): intermediate_tensor = tf.multiply(x_0_expanded, x_i_expanded) intermediate_tensor_expanded = tf.expand_dims(intermediate_tensor, 1) - intermediate_tensor_expanded = tf.tile(intermediate_tensor_expanded, [1, hk, 1, 1, 1]) + intermediate_tensor_expanded = tf.tile(intermediate_tensor_expanded, + [1, hk, 1, 1, 1]) feature_map_elementwise = tf.multiply( - intermediate_tensor_expanded, - tf.expand_dims(tf.expand_dims(self.kernel_list[i], -1), 0), + intermediate_tensor_expanded, + tf.expand_dims(tf.expand_dims(self.kernel_list[i], -1), 0), ) - feature_map = tf.reduce_sum(tf.reduce_sum(feature_map_elementwise, axis=3), axis=2) + feature_map = tf.reduce_sum( + tf.reduce_sum(feature_map_elementwise, axis=3), axis=2) feature_map = tf.add( - feature_map, - tf.expand_dims(tf.expand_dims(self.bias_list[i], axis=-1), axis=0), + feature_map, + tf.expand_dims(tf.expand_dims(self.bias_list[i], axis=-1), axis=0), ) feature_map = tf.nn.relu(feature_map) x_i = feature_map pooled_feature_map_list.append(tf.reduce_sum(feature_map, axis=-1)) - return tf.concat(pooled_feature_map_list, axis=-1) # shape = (b, h1 + ... + hk) + return tf.concat( + pooled_feature_map_list, axis=-1) # shape = (b, h1 + ... + hk) def get_config(self): pass diff --git a/easy_rec/python/layers/keras/layer_norm.py b/easy_rec/python/layers/keras/layer_norm.py index 51a1d8441..10491140f 100644 --- a/easy_rec/python/layers/keras/layer_norm.py +++ b/easy_rec/python/layers/keras/layer_norm.py @@ -1,7 +1,9 @@ """Layer Normalization layer.""" import tensorflow as tf -from tensorflow.python.keras import constraints, initializers, regularizers +from tensorflow.python.keras import constraints +from tensorflow.python.keras import initializers +from tensorflow.python.keras import regularizers from tensorflow.python.keras.layers import Layer @@ -19,7 +21,9 @@ def validate_axis(axis, input_shape): input_shape = tf.TensorShape(input_shape) rank = input_shape.ndims if not rank: - raise ValueError('Input has undefined rank. Received: input_shape={input_shape}'.format(input_shape=input_shape)) + raise ValueError( + 'Input has undefined rank. Received: input_shape={input_shape}'.format( + input_shape=input_shape)) # Convert axis to list and resolve negatives if isinstance(axis, int): @@ -33,11 +37,10 @@ def validate_axis(axis, input_shape): # Validate axes for x in axis: if x < 0 or x >= rank: - raise ValueError( - 'Invalid value for `axis` argument. ' - 'Expected 0 <= axis < inputs.rank (with ' - 'inputs.rank={rank}). Received: axis={axis}'.format(rank=rank, axis=tuple(axis)) - ) + raise ValueError('Invalid value for `axis` argument. ' + 'Expected 0 <= axis < inputs.rank (with ' + 'inputs.rank={rank}). Received: axis={axis}'.format( + rank=rank, axis=tuple(axis))) if len(axis) != len(set(axis)): raise ValueError('Duplicate axis: {axis}'.format(axis=tuple(axis))) return axis @@ -167,18 +170,18 @@ class LayerNormalization(Layer): """ def __init__( - self, - axis=-1, - epsilon=1e-3, - center=True, - scale=True, - beta_initializer='zeros', - gamma_initializer='ones', - beta_regularizer=None, - gamma_regularizer=None, - beta_constraint=None, - gamma_constraint=None, - **kwargs, + self, + axis=-1, + epsilon=1e-3, + center=True, + scale=True, + beta_initializer='zeros', + gamma_initializer='ones', + beta_regularizer=None, + gamma_regularizer=None, + beta_constraint=None, + gamma_constraint=None, + **kwargs, ): super(LayerNormalization, self).__init__(**kwargs) if isinstance(axis, (list, tuple)): @@ -186,7 +189,8 @@ def __init__( elif isinstance(axis, int): self.axis = axis else: - raise TypeError('Expected an int or a list/tuple of ints for the ' "argument 'axis', but received: %r" % axis) + raise TypeError('Expected an int or a list/tuple of ints for the ' + "argument 'axis', but received: %r" % axis) self.epsilon = epsilon self.center = center @@ -235,30 +239,31 @@ def build(self, input_shape): param_shape = [input_shape[dim] for dim in self.axis] if self.scale: self.gamma = self.add_weight( - name='gamma', - shape=param_shape, - initializer=self.gamma_initializer, - regularizer=self.gamma_regularizer, - constraint=self.gamma_constraint, - trainable=True, + name='gamma', + shape=param_shape, + initializer=self.gamma_initializer, + regularizer=self.gamma_regularizer, + constraint=self.gamma_constraint, + trainable=True, ) else: self.gamma = None if self.center: self.beta = self.add_weight( - name='beta', - shape=param_shape, - initializer=self.beta_initializer, - regularizer=self.beta_regularizer, - constraint=self.beta_constraint, - trainable=True, + name='beta', + shape=param_shape, + initializer=self.beta_initializer, + regularizer=self.beta_regularizer, + constraint=self.beta_constraint, + trainable=True, ) else: self.beta = None self._fused = self._fused_can_be_used(rank) - super(LayerNormalization, self).build(input_shape) # Be sure to call this somewhere! + super(LayerNormalization, + self).build(input_shape) # Be sure to call this somewhere! def call(self, inputs): # Compute the axes along which to reduce the mean / variance @@ -291,12 +296,12 @@ def _broadcast(v): # Compute layer normalization using the batch_normalization # function. outputs = tf.nn.batch_normalization( - inputs, - mean, - variance, - offset=offset, - scale=scale, - variance_epsilon=self.epsilon, + inputs, + mean, + variance, + offset=offset, + scale=scale, + variance_epsilon=self.epsilon, ) outputs = tf.cast(outputs, input_dtype) else: @@ -304,8 +309,8 @@ def _broadcast(v): axis = sorted(self.axis) tensor_shape = tf.shape(inputs) - pre_dim = tf.reduce_prod(tensor_shape[: axis[0]]) - in_dim = tf.reduce_prod(tensor_shape[axis[0] :]) + pre_dim = tf.reduce_prod(tensor_shape[:axis[0]]) + in_dim = tf.reduce_prod(tensor_shape[axis[0]:]) squeezed_shape = [1, pre_dim, in_dim, 1] # This fused operation requires reshaped inputs to be NCHW. data_format = 'NCHW' @@ -322,11 +327,11 @@ def _broadcast(v): # Compute layer normalization using the fused_batch_norm function. outputs, _, _ = tf.compat.v1.nn.fused_batch_norm( - inputs, - scale=scale, - offset=offset, - epsilon=self.epsilon, - data_format=data_format, + inputs, + scale=scale, + offset=offset, + epsilon=self.epsilon, + data_format=data_format, ) outputs = tf.reshape(outputs, tensor_shape) @@ -347,16 +352,16 @@ def compute_output_shape(self, input_shape): def get_config(self): config = { - 'axis': self.axis, - 'epsilon': self.epsilon, - 'center': self.center, - 'scale': self.scale, - 'beta_initializer': initializers.serialize(self.beta_initializer), - 'gamma_initializer': initializers.serialize(self.gamma_initializer), - 'beta_regularizer': regularizers.serialize(self.beta_regularizer), - 'gamma_regularizer': regularizers.serialize(self.gamma_regularizer), - 'beta_constraint': constraints.serialize(self.beta_constraint), - 'gamma_constraint': constraints.serialize(self.gamma_constraint), + 'axis': self.axis, + 'epsilon': self.epsilon, + 'center': self.center, + 'scale': self.scale, + 'beta_initializer': initializers.serialize(self.beta_initializer), + 'gamma_initializer': initializers.serialize(self.gamma_initializer), + 'beta_regularizer': regularizers.serialize(self.beta_regularizer), + 'gamma_regularizer': regularizers.serialize(self.gamma_regularizer), + 'beta_constraint': constraints.serialize(self.beta_constraint), + 'gamma_constraint': constraints.serialize(self.gamma_constraint), } base_config = super(LayerNormalization, self).get_config() return dict(list(base_config.items()) + list(config.items())) diff --git a/easy_rec/python/layers/keras/mask_net.py b/easy_rec/python/layers/keras/mask_net.py index 77a1375c9..96eb0fea5 100644 --- a/easy_rec/python/layers/keras/mask_net.py +++ b/easy_rec/python/layers/keras/mask_net.py @@ -3,7 +3,9 @@ import logging import tensorflow as tf -from tensorflow.python.keras.layers import Activation, Dense, Layer +from tensorflow.python.keras.layers import Activation +from tensorflow.python.keras.layers import Dense +from tensorflow.python.keras.layers import Layer from easy_rec.python.layers.keras.blocks import MLP from easy_rec.python.layers.keras.layer_norm import LayerNormalization @@ -44,35 +46,39 @@ def build(self, input_shape): elif self.config.HasField('aggregation_size') is not None: aggregation_size = self.config.aggregation_size else: - raise ValueError('Need one of reduction factor or aggregation size for MaskBlock.') + raise ValueError( + 'Need one of reduction factor or aggregation size for MaskBlock.') self.aggr_layer = Dense( - aggregation_size, - activation='relu', - kernel_initializer='he_uniform', - kernel_regularizer=self.l2_reg, - name='aggregation', + aggregation_size, + activation='relu', + kernel_initializer='he_uniform', + kernel_regularizer=self.l2_reg, + name='aggregation', ) self.weight_layer = Dense(input_dim, name='weights') if self._projection_dim is not None: logging.info('%s project dim is %d', self.name, self._projection_dim) self.project_layer = Dense( - self._projection_dim, - kernel_regularizer=self.l2_reg, - use_bias=False, - name='project', + self._projection_dim, + kernel_regularizer=self.l2_reg, + use_bias=False, + name='project', ) if self.config.input_layer_norm: # 推荐在调用MaskBlock之前做好 layer norm，否则每一次调用都需要对input做ln if tf.__version__ >= '2.0': - self.input_layer_norm = tf.keras.layers.LayerNormalization(name='input_ln') + self.input_layer_norm = tf.keras.layers.LayerNormalization( + name='input_ln') else: self.input_layer_norm = LayerNormalization(name='input_ln') if self.config.HasField('output_size'): - self.output_layer = Dense(self.config.output_size, use_bias=False, name='output') + self.output_layer = Dense( + self.config.output_size, use_bias=False, name='output') if tf.__version__ >= '2.0': - self.output_layer_norm = tf.keras.layers.LayerNormalization(name='output_ln') + self.output_layer_norm = tf.keras.layers.LayerNormalization( + name='output_ln') else: self.output_layer_norm = LayerNormalization(name='output_ln') super(MaskBlock, self).build(input_shape) @@ -130,7 +136,8 @@ def __init__(self, params, name='mask_net', reuse=None, **kwargs): if self.config.input_layer_norm: if tf.__version__ >= '2.0': - self.input_layer_norm = tf.keras.layers.LayerNormalization(name='input_ln') + self.input_layer_norm = tf.keras.layers.LayerNormalization( + name='input_ln') else: self.input_layer_norm = LayerNormalization(name='input_ln') @@ -139,7 +146,9 @@ def call(self, inputs, training=None, **kwargs): inputs = self.input_layer_norm(inputs) if self.config.use_parallel: - mask_outputs = [mask_layer((inputs, inputs)) for mask_layer in self.mask_layers] + mask_outputs = [ + mask_layer((inputs, inputs)) for mask_layer in self.mask_layers + ] all_mask_outputs = tf.concat(mask_outputs, axis=1) if self.mlp is not None: output = self.mlp(all_mask_outputs, training=training) diff --git a/easy_rec/python/layers/keras/multi_head_attention.py b/easy_rec/python/layers/keras/multi_head_attention.py index 7971eb714..5bbe0112a 100644 --- a/easy_rec/python/layers/keras/multi_head_attention.py +++ b/easy_rec/python/layers/keras/multi_head_attention.py @@ -5,8 +5,12 @@ import numpy as np import tensorflow as tf -from tensorflow.python.keras import constraints, initializers, regularizers -from tensorflow.python.keras.layers import Dropout, Layer, Softmax +from tensorflow.python.keras import constraints +from tensorflow.python.keras import initializers +from tensorflow.python.keras import regularizers +from tensorflow.python.keras.layers import Dropout +from tensorflow.python.keras.layers import Layer +from tensorflow.python.keras.layers import Softmax from easy_rec.python.layers.keras.activation import MaskedSoftmax from easy_rec.python.layers.keras.einsum_dense import EinsumDense @@ -101,16 +105,24 @@ def __init__(self, params, name='multi_head_attention', reuse=None, **kwargs): self._dropout = params.get_or_default('dropout', 0.0) self._use_bias = params.get_or_default('use_bias', True) self._output_shape = params.get_or_default('output_shape', None) - self._kernel_initializer = initializers.get(params.get_or_default('kernel_initializer', 'glorot_uniform')) - self._bias_initializer = initializers.get(params.get_or_default('bias_initializer', 'zeros')) - self._kernel_regularizer = regularizers.get(params.get_or_default('kernel_regularizer', None)) - self._bias_regularizer = regularizers.get(params.get_or_default('bias_regularizer', None)) - self._activity_regularizer = regularizers.get(params.get_or_default('activity_regularizer', None)) - self._kernel_constraint = constraints.get(params.get_or_default('kernel_constraint', None)) - self._bias_constraint = constraints.get(params.get_or_default('bias_constraint', None)) + self._kernel_initializer = initializers.get( + params.get_or_default('kernel_initializer', 'glorot_uniform')) + self._bias_initializer = initializers.get( + params.get_or_default('bias_initializer', 'zeros')) + self._kernel_regularizer = regularizers.get( + params.get_or_default('kernel_regularizer', None)) + self._bias_regularizer = regularizers.get( + params.get_or_default('bias_regularizer', None)) + self._activity_regularizer = regularizers.get( + params.get_or_default('activity_regularizer', None)) + self._kernel_constraint = constraints.get( + params.get_or_default('kernel_constraint', None)) + self._bias_constraint = constraints.get( + params.get_or_default('bias_constraint', None)) self._attention_axes = params.get_or_default('attention_axes', None) self._use_causal_mask = params.get_or_default('use_causal_mask', False) - self._return_attention_scores = params.get_or_default('return_attention_scores', False) + self._return_attention_scores = params.get_or_default( + 'return_attention_scores', False) @property def num_heads(self): @@ -143,20 +155,34 @@ def attention_axes(self): def get_config(self): base_config = super(MultiHeadAttention, self).get_config() config = { - 'num_heads': self._num_heads, - 'key_dim': self._key_dim, - 'value_dim': self._value_dim, - 'dropout': self._dropout, - 'use_bias': self._use_bias, - 'output_shape': self._output_shape, - 'attention_axes': self._attention_axes, - 'kernel_initializer': initializers.serialize(self._kernel_initializer), - 'bias_initializer': initializers.serialize(self._bias_initializer), - 'kernel_regularizer': regularizers.serialize(self._kernel_regularizer), - 'bias_regularizer': regularizers.serialize(self._bias_regularizer), - 'activity_regularizer': regularizers.serialize(self._activity_regularizer), - 'kernel_constraint': constraints.serialize(self._kernel_constraint), - 'bias_constraint': constraints.serialize(self._bias_constraint), + 'num_heads': + self._num_heads, + 'key_dim': + self._key_dim, + 'value_dim': + self._value_dim, + 'dropout': + self._dropout, + 'use_bias': + self._use_bias, + 'output_shape': + self._output_shape, + 'attention_axes': + self._attention_axes, + 'kernel_initializer': + initializers.serialize(self._kernel_initializer), + 'bias_initializer': + initializers.serialize(self._bias_initializer), + 'kernel_regularizer': + regularizers.serialize(self._kernel_regularizer), + 'bias_regularizer': + regularizers.serialize(self._bias_regularizer), + 'activity_regularizer': + regularizers.serialize(self._activity_regularizer), + 'kernel_constraint': + constraints.serialize(self._kernel_constraint), + 'bias_constraint': + constraints.serialize(self._bias_constraint), } config.update(base_config) return config @@ -175,31 +201,37 @@ def build(self, input_shape): query_rank = len(query_shape) value_rank = len(value_shape) key_rank = len(key_shape) - einsum_equation, bias_axes, output_rank = _build_proj_equation(query_rank - 1, bound_dims=1, output_dims=2) + einsum_equation, bias_axes, output_rank = _build_proj_equation( + query_rank - 1, bound_dims=1, output_dims=2) self._query_dense = EinsumDense( - einsum_equation, - output_shape=_get_output_shape(output_rank - 1, [self._num_heads, self._key_dim]), - bias_axes=bias_axes if self._use_bias else None, - name='query', - **self._get_common_kwargs_for_sublayer(), + einsum_equation, + output_shape=_get_output_shape(output_rank - 1, + [self._num_heads, self._key_dim]), + bias_axes=bias_axes if self._use_bias else None, + name='query', + **self._get_common_kwargs_for_sublayer(), ) self._query_dense.build(query_shape) - einsum_equation, bias_axes, output_rank = _build_proj_equation(key_rank - 1, bound_dims=1, output_dims=2) + einsum_equation, bias_axes, output_rank = _build_proj_equation( + key_rank - 1, bound_dims=1, output_dims=2) self._key_dense = EinsumDense( - einsum_equation, - output_shape=_get_output_shape(output_rank - 1, [self._num_heads, self._key_dim]), - bias_axes=bias_axes if self._use_bias else None, - name='key', - **self._get_common_kwargs_for_sublayer(), + einsum_equation, + output_shape=_get_output_shape(output_rank - 1, + [self._num_heads, self._key_dim]), + bias_axes=bias_axes if self._use_bias else None, + name='key', + **self._get_common_kwargs_for_sublayer(), ) self._key_dense.build(key_shape) - einsum_equation, bias_axes, output_rank = _build_proj_equation(value_rank - 1, bound_dims=1, output_dims=2) + einsum_equation, bias_axes, output_rank = _build_proj_equation( + value_rank - 1, bound_dims=1, output_dims=2) self._value_dense = EinsumDense( - einsum_equation, - output_shape=_get_output_shape(output_rank - 1, [self._num_heads, self._value_dim]), - bias_axes=bias_axes if self._use_bias else None, - name='value', - **self._get_common_kwargs_for_sublayer(), + einsum_equation, + output_shape=_get_output_shape(output_rank - 1, + [self._num_heads, self._value_dim]), + bias_axes=bias_axes if self._use_bias else None, + name='value', + **self._get_common_kwargs_for_sublayer(), ) self._value_dense.build(value_shape) # Builds the attention computations for multi-head dot product @@ -207,11 +239,12 @@ def build(self, input_shape): # attention layer once it supports multi-head einsum computations. self._build_attention(output_rank) self._output_dense = self._make_output_dense( - query_shape, - self._get_common_kwargs_for_sublayer(), - 'attention_output', + query_shape, + self._get_common_kwargs_for_sublayer(), + 'attention_output', ) - output_dense_input_shape = list(self._query_dense.compute_output_shape(query_shape)) + output_dense_input_shape = list( + self._query_dense.compute_output_shape(query_shape)) output_dense_input_shape[-1] = self._value_dim self._output_dense.build(tuple(output_dense_input_shape)) self.built = True @@ -235,18 +268,20 @@ def output_dense(self): def _get_common_kwargs_for_sublayer(self): common_kwargs = dict( - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer, - activity_regularizer=self._activity_regularizer, - kernel_constraint=self._kernel_constraint, - bias_constraint=self._bias_constraint, - dtype=tf.float32, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint, + dtype=tf.float32, ) # Create new clone of kernel/bias initializer, so that we don't reuse # the initializer instance, which could lead to same init value since # initializer is stateless. - kernel_initializer = self._kernel_initializer.__class__.from_config(self._kernel_initializer.get_config()) - bias_initializer = self._bias_initializer.__class__.from_config(self._bias_initializer.get_config()) + kernel_initializer = self._kernel_initializer.__class__.from_config( + self._kernel_initializer.get_config()) + bias_initializer = self._bias_initializer.__class__.from_config( + self._bias_initializer.get_config()) common_kwargs['kernel_initializer'] = kernel_initializer common_kwargs['bias_initializer'] = bias_initializer return common_kwargs @@ -271,14 +306,13 @@ def _make_output_dense(self, query_shape, common_kwargs, name=None): else: output_shape = [query_shape[-1]] einsum_equation, bias_axes, output_rank = _build_proj_equation( - query_rank - 1, bound_dims=2, output_dims=len(output_shape) - ) + query_rank - 1, bound_dims=2, output_dims=len(output_shape)) return EinsumDense( - einsum_equation, - output_shape=_get_output_shape(output_rank - 1, output_shape), - bias_axes=bias_axes if self._use_bias else None, - name=name, - **common_kwargs, + einsum_equation, + output_shape=_get_output_shape(output_rank - 1, output_shape), + bias_axes=bias_axes if self._use_bias else None, + name=name, + **common_kwargs, ) def _build_attention(self, rank): @@ -296,12 +330,16 @@ def _build_attention(self, rank): else: self._attention_axes = tuple(self._attention_axes) ( - self._dot_product_equation, - self._combine_equation, - attn_scores_rank, - ) = _build_attention_equation(rank, attn_axes=self._attention_axes) - norm_axes = tuple(range(attn_scores_rank - len(self._attention_axes), attn_scores_rank)) - self._softmax = Softmax(axis=norm_axes) if tf.__version__ >= '2.0' else MaskedSoftmax(axis=norm_axes) + self._dot_product_equation, + self._combine_equation, + attn_scores_rank, + ) = _build_attention_equation( + rank, attn_axes=self._attention_axes) + norm_axes = tuple( + range(attn_scores_rank - len(self._attention_axes), attn_scores_rank)) + self._softmax = Softmax( + axis=norm_axes) if tf.__version__ >= '2.0' else MaskedSoftmax( + axis=norm_axes) self._dropout_layer = Dropout(rate=self._dropout) self._inverse_sqrt_key_dim = 1.0 / math.sqrt(float(self._key_dim)) @@ -314,10 +352,16 @@ def _masked_softmax(self, attention_scores, attention_mask=None): # key_attention_dims>) mask_expansion_axis = -len(self._attention_axes) * 2 - 1 for _ in range(len(attention_scores.shape) - len(attention_mask.shape)): - attention_mask = tf.expand_dims(attention_mask, axis=mask_expansion_axis) + attention_mask = tf.expand_dims( + attention_mask, axis=mask_expansion_axis) return self._softmax(attention_scores, mask=attention_mask) - def _compute_attention(self, query, key, value, attention_mask=None, training=None): + def _compute_attention(self, + query, + key, + value, + attention_mask=None, + training=None): """Applies Dot-product attention with query, key, value tensors. This function defines the computation inside `call` with projected @@ -353,16 +397,19 @@ def _compute_attention(self, query, key, value, attention_mask=None, training=No # This is actually dropping out entire tokens to attend to, which might # seem a bit unusual, but is taken from the original Transformer paper. if self.dropout: - final_attn_scores = self._dropout_layer(attention_scores, training=training) + final_attn_scores = self._dropout_layer( + attention_scores, training=training) else: final_attn_scores = attention_scores # `context_layer` = [B, T, N, H] - attention_output = tf.einsum(self._combine_equation, final_attn_scores, value) + attention_output = tf.einsum(self._combine_equation, final_attn_scores, + value) return attention_output, attention_scores def call(self, inputs, mask=None, training=None, **kwargs): - assert isinstance(inputs, (tuple, list)), 'inputs of MultiHeadAttention must be a list' + assert isinstance( + inputs, (tuple, list)), 'inputs of MultiHeadAttention must be a list' query, value, key = (list(inputs) + [None] * 2)[:3] if key is None: key = value @@ -376,13 +423,13 @@ def call(self, inputs, mask=None, training=None, **kwargs): if attention_mask is None and value_mask is None: value_mask = query_mask attention_mask = self._compute_attention_mask( - query, - value, - query_mask=query_mask, - value_mask=value_mask, - key_mask=key_mask, - attention_mask=attention_mask, - use_causal_mask=self._use_causal_mask, + query, + value, + query_mask=query_mask, + value_mask=value_mask, + key_mask=key_mask, + attention_mask=attention_mask, + use_causal_mask=self._use_causal_mask, ) # N = `num_attention_heads` @@ -395,21 +442,22 @@ def call(self, inputs, mask=None, training=None, **kwargs): # `value` = [B, S, N, H] value = self._value_dense(value) - attention_output, attention_scores = self._compute_attention(query, key, value, attention_mask, training) + attention_output, attention_scores = self._compute_attention( + query, key, value, attention_mask, training) attention_output = self._output_dense(attention_output) if self._return_attention_scores: return attention_output, attention_scores return attention_output def _compute_attention_mask( - self, - query, - value, - query_mask=None, - value_mask=None, - key_mask=None, - attention_mask=None, - use_causal_mask=False, + self, + query, + value, + query_mask=None, + value_mask=None, + key_mask=None, + attention_mask=None, + use_causal_mask=False, ): """Computes the attention mask, using the Keras masks of the inputs. @@ -461,7 +509,8 @@ def _compute_attention_mask( auto_mask = mask if auto_mask is None else auto_mask & mask if auto_mask is not None: # merge attention_mask & automatic mask, to shape [B, T, S] - attention_mask = auto_mask if attention_mask is None else tf.cast(attention_mask, tf.bool) & auto_mask + attention_mask = auto_mask if attention_mask is None else tf.cast( + attention_mask, tf.bool) & auto_mask return attention_mask def _compute_causal_mask(self, query, value=None): @@ -506,22 +555,22 @@ def compute_output_shape(self, input_shape): if query_shape[-1] != value_shape[-1]: raise ValueError( - 'The last dimension of `query_shape` and `value_shape` ' - 'must be equal, but are {query_last_dim}, {value_last_dim}. ' - 'Received: query_shape={query_shape}, value_shape={value_shape}'.format( - query_shape=query_shape, - value_shape=value_shape, - query_last_dim=query_shape[-1], - value_last_dim=value_shape[-1], - ) - ) + 'The last dimension of `query_shape` and `value_shape` ' + 'must be equal, but are {query_last_dim}, {value_last_dim}. ' + 'Received: query_shape={query_shape}, value_shape={value_shape}' + .format( + query_shape=query_shape, + value_shape=value_shape, + query_last_dim=query_shape[-1], + value_last_dim=value_shape[-1], + )) if value_shape[1:-1] != key_shape[1:-1]: raise ValueError( - 'All dimensions of `value` and `key`, except the last one, ' - 'must be equal. Received: value_shape={value_shape} and ' - 'key_shape={key_shape}'.format(key_shape=key_shape, value_shape=value_shape) - ) + 'All dimensions of `value` and `key`, except the last one, ' + 'must be equal. Received: value_shape={value_shape} and ' + 'key_shape={key_shape}'.format( + key_shape=key_shape, value_shape=value_shape)) if self._output_shape: if hasattr(self._output_dense, '__len__'): @@ -579,21 +628,19 @@ def _build_attention_equation(rank, attn_axes): source_notation += _index_to_einsum_variable(letter_offset) letter_offset += 1 - product_notation = ''.join( - [target_notation[i] for i in batch_dims] - + [target_notation[i] for i in attn_axes] - + [source_notation[i] for i in attn_axes] - ) + product_notation = ''.join([target_notation[i] for i in batch_dims] + + [target_notation[i] for i in attn_axes] + + [source_notation[i] for i in attn_axes]) dot_product_equation = '%s,%s->%s' % ( - source_notation, - target_notation, - product_notation, + source_notation, + target_notation, + product_notation, ) attn_scores_rank = len(product_notation) combine_equation = '%s,%s->%s' % ( - product_notation, - source_notation, - target_notation, + product_notation, + source_notation, + target_notation, ) return dot_product_equation, combine_equation, attn_scores_rank @@ -623,8 +670,7 @@ def _build_proj_equation(free_dims, bound_dims, output_dims): output_str += char bias_axes += char equation = '{input_str},{kernel_str}->{output_str}'.format( - input_str=input_str, kernel_str=kernel_str, output_str=output_str - ) + input_str=input_str, kernel_str=kernel_str, output_str=output_str) return equation, bias_axes, len(output_str) diff --git a/easy_rec/python/layers/keras/multi_task.py b/easy_rec/python/layers/keras/multi_task.py index 6433e7183..6d3c97d22 100644 --- a/easy_rec/python/layers/keras/multi_task.py +++ b/easy_rec/python/layers/keras/multi_task.py @@ -3,7 +3,8 @@ import logging import tensorflow as tf -from tensorflow.python.keras.layers import Dense, Layer +from tensorflow.python.keras.layers import Dense +from tensorflow.python.keras.layers import Layer from easy_rec.python.layers.keras.attention import Attention from easy_rec.python.layers.keras.blocks import MLP @@ -27,17 +28,20 @@ def __init__(self, params, name='MMoE', reuse=None, **kwargs): expert_params = Parameter.make_from_pb(params.expert_mlp) expert_params.l2_regularizer = params.l2_regularizer self._has_experts = True - self._experts = [MLP(expert_params, 'expert_%d' % i, reuse=reuse) for i in range(self._num_expert)] + self._experts = [ + MLP(expert_params, 'expert_%d' % i, reuse=reuse) + for i in range(self._num_expert) + ] else: self._has_experts = False self._gates = [] for task_id in range(self._num_task): dense = Dense( - self._num_expert, - activation='softmax', - name='gate_%d' % task_id, - kernel_regularizer=params.l2_regularizer, + self._num_expert, + activation='softmax', + name='gate_%d' % task_id, + kernel_regularizer=params.l2_regularizer, ) self._gates.append(dense) @@ -46,7 +50,9 @@ def call(self, inputs, training=None, **kwargs): logging.warning('num_expert of MMoE layer `%s` is 0' % self.name) return inputs if self._has_experts: - expert_fea_list = [expert(inputs, training=training) for expert in self._experts] + expert_fea_list = [ + expert(inputs, training=training) for expert in self._experts + ] else: expert_fea_list = inputs experts_fea = tf.stack(expert_fea_list, axis=1) diff --git a/easy_rec/python/layers/keras/numerical_embedding.py b/easy_rec/python/layers/keras/numerical_embedding.py index 42cb28738..dda436efe 100644 --- a/easy_rec/python/layers/keras/numerical_embedding.py +++ b/easy_rec/python/layers/keras/numerical_embedding.py @@ -34,7 +34,8 @@ custom_ops = tf.load_op_library(custom_op_path) logging.info('load custom op from %s succeed' % custom_op_path) except Exception as ex: - logging.warning('load custom op from %s failed: %s' % (custom_op_path, str(ex))) + logging.warning('load custom op from %s failed: %s' % + (custom_op_path, str(ex))) custom_ops = None @@ -67,7 +68,13 @@ class NLinear(Layer): assert m(x).shape == (batch_size, n_features, d_embedding_out) """ - def __init__(self, n_tokens, d_in, d_out, bias=True, name='nd_linear', **kwargs): + def __init__(self, + n_tokens, + d_in, + d_out, + bias=True, + name='nd_linear', + **kwargs): """Init with input shapes. Args: @@ -78,18 +85,25 @@ def __init__(self, n_tokens, d_in, d_out, bias=True, name='nd_linear', **kwargs) name: layer name """ super(NLinear, self).__init__(name=name, **kwargs) - self.weight = self.add_weight('weights', [1, n_tokens, d_in, d_out], dtype=tf.float32) + self.weight = self.add_weight( + 'weights', [1, n_tokens, d_in, d_out], dtype=tf.float32) if bias: initializer = tf.constant_initializer(0.0) - self.bias = self.add_weight('bias', [1, n_tokens, d_out], dtype=tf.float32, initializer=initializer) + self.bias = self.add_weight( + 'bias', [1, n_tokens, d_out], + dtype=tf.float32, + initializer=initializer) else: self.bias = None def call(self, x, **kwargs): if x.shape.ndims != 3: - raise ValueError('The input must have three dimensions (batch_size, n_tokens, d_embedding)') + raise ValueError( + 'The input must have three dimensions (batch_size, n_tokens, d_embedding)' + ) if x.shape[2] != self.weight.shape[2]: - raise ValueError('invalid input embedding dimension %d, expect %d' % (int(x.shape[2]), int(self.weight.shape[2]))) + raise ValueError('invalid input embedding dimension %d, expect %d' % + (int(x.shape[2]), int(self.weight.shape[2]))) x = x[..., None] * self.weight # [B, N, D, D_out] x = tf.reduce_sum(x, axis=-2) # [B, N, D_out] @@ -144,17 +158,17 @@ def build(self, input_shape): partitioner = tf.fixed_size_partitioner(num_shards=num_ps) emb_dim = self.embedding_dim // 2 self.coef = self.add_weight( - 'coefficients', - shape=[1, self.num_features, emb_dim], - partitioner=partitioner, - initializer=self.initializer, + 'coefficients', + shape=[1, self.num_features, emb_dim], + partitioner=partitioner, + initializer=self.initializer, ) if self.add_linear_layer: self.linear = NLinear( - self.num_features, - self.embedding_dim, - self.embedding_dim, - name='nd_linear', + self.num_features, + self.embedding_dim, + self.embedding_dim, + name='nd_linear', ) super(PeriodicEmbedding, self).build(input_shape) @@ -204,19 +218,19 @@ def build(self, input_shape): if num_ps > 0: partitioner = tf.fixed_size_partitioner(num_shards=num_ps) self.meta_emb = self.add_weight( - 'meta_embedding', - shape=[self.num_features, self.num_bins, self.emb_dim], - partitioner=partitioner, + 'meta_embedding', + shape=[self.num_features, self.num_bins, self.emb_dim], + partitioner=partitioner, ) self.proj_w = self.add_weight( - 'project_w', - shape=[1, self.num_features, self.num_bins], - partitioner=partitioner, + 'project_w', + shape=[1, self.num_features, self.num_bins], + partitioner=partitioner, ) self.proj_mat = self.add_weight( - 'project_mat', - shape=[self.num_features, self.num_bins, self.num_bins], - partitioner=partitioner, + 'project_mat', + shape=[self.num_features, self.num_bins, self.num_bins], + partitioner=partitioner, ) super(AutoDisEmbedding, self).build(input_shape) @@ -267,16 +281,16 @@ def __init__(self, params, name='nary_dis_embedding', reuse=None, **kwargs): self.output_3d_tensor = params.get_or_default('output_3d_tensor', False) self.output_tensor_list = params.get_or_default('output_tensor_list', False) logging.info( - '{} carries: {}, lengths: {}, vocab_size: {}, intra_ary: {}, replicas: {}, multiplier: {}'.format( - self.name, - ','.join(map(str, self.carries)), - ','.join(map(str, self.lengths)), - self.vocab_size, - self.intra_ary_pooling, - self.num_replicas, - self.multiplier, - ) - ) + '{} carries: {}, lengths: {}, vocab_size: {}, intra_ary: {}, replicas: {}, multiplier: {}' + .format( + self.name, + ','.join(map(str, self.carries)), + ','.join(map(str, self.lengths)), + self.vocab_size, + self.intra_ary_pooling, + self.num_replicas, + self.multiplier, + )) @staticmethod def max_length(carry): @@ -284,7 +298,8 @@ def max_length(carry): return (math.floor(bits) + 1) * carry def build(self, input_shape): - assert isinstance(input_shape, tf.TensorShape), 'NaryDisEmbedding only takes 1 input' + assert isinstance(input_shape, + tf.TensorShape), 'NaryDisEmbedding only takes 1 input' self.num_features = int(input_shape[-1]) logging.info('%s has %d input features', self.name, self.num_features) vocab_size = self.num_features * self.vocab_size @@ -293,7 +308,8 @@ def build(self, input_shape): partitioner = None if num_ps > 0: partitioner = tf.fixed_size_partitioner(num_shards=num_ps) - self.embedding_table = self.add_weight('embed_table', shape=[vocab_size, emb_dim], partitioner=partitioner) + self.embedding_table = self.add_weight( + 'embed_table', shape=[vocab_size, emb_dim], partitioner=partitioner) super(NaryDisEmbedding, self).build(input_shape) def call(self, inputs, **kwargs): @@ -328,7 +344,8 @@ def call(self, inputs, **kwargs): segment_ids = repeat(tf.range(total_length), repeats=splits) embedding = tf.math.segment_mean(embedding, segment_ids) else: - raise ValueError('Unsupported intra ary pooling method %s' % self.intra_ary_pooling) + raise ValueError('Unsupported intra ary pooling method %s' % + self.intra_ary_pooling) # B: batch size # N: num features # C: num carries diff --git a/easy_rec/python/layers/keras/ppnet.py b/easy_rec/python/layers/keras/ppnet.py index 0d9f73227..4f43198e5 100644 --- a/easy_rec/python/layers/keras/ppnet.py +++ b/easy_rec/python/layers/keras/ppnet.py @@ -14,7 +14,13 @@ class GateNN(tf.keras.layers.Layer): - def __init__(self, params, output_units=None, name='gate_nn', reuse=None, **kwargs): + + def __init__(self, + params, + output_units=None, + name='gate_nn', + reuse=None, + **kwargs): super(GateNN, self).__init__(name=name, **kwargs) output_dim = output_units if output_units is not None else params.output_dim hidden_dim = params.get_or_default('hidden_dim', output_dim) @@ -24,7 +30,10 @@ def __init__(self, params, output_units=None, name='gate_nn', reuse=None, **kwar dropout_rate = params.get_or_default('dropout_rate', 0.0) self._sub_layers = [] - dense = tf.keras.layers.Dense(units=hidden_dim, use_bias=not do_batch_norm, kernel_initializer=initializer) + dense = tf.keras.layers.Dense( + units=hidden_dim, + use_bias=not do_batch_norm, + kernel_initializer=initializer) self._sub_layers.append(dense) if do_batch_norm: @@ -41,11 +50,11 @@ def __init__(self, params, output_units=None, name='gate_nn', reuse=None, **kwar raise ValueError('invalid dropout_ratio: %.3f' % dropout_rate) dense = tf.keras.layers.Dense( - units=output_dim, - activation='sigmoid', - use_bias=not do_batch_norm, - kernel_initializer=initializer, - name='weight', + units=output_dim, + activation='sigmoid', + use_bias=not do_batch_norm, + kernel_initializer=initializer, + name='weight', ) self._sub_layers.append(dense) self._sub_layers.append(lambda x: x * 2) @@ -93,21 +102,20 @@ def __init__(self, params, name='ppnet', reuse=None, **kwargs): use_bn_after_act = params.get_or_default('use_bn_after_activation', False) units = list(params.hidden_units) logging.info( - 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' - ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' - % ( - name, - units, - dropout_rate, - activation, - use_bn, - use_final_bn, - final_activation, - use_bias, - initializer, - use_bn_after_act, - ) - ) + 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' + ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' % + ( + name, + units, + dropout_rate, + activation, + use_bn, + use_final_bn, + final_activation, + use_bias, + initializer, + use_bn_after_act, + )) assert len(units) > 0, 'MLP(%s) takes at least one hidden units' % name self.reuse = reuse @@ -120,67 +128,70 @@ def __init__(self, params, name='ppnet', reuse=None, **kwargs): name = 'layer_%d' % i drop_rate = dropout_rate[i] if i < num_dropout else 0.0 self.add_rich_layer( - num_units, - use_bn, - drop_rate, - activation, - initializer, - use_bias, - use_bn_after_act, - name, - params.l2_regularizer, + num_units, + use_bn, + drop_rate, + activation, + initializer, + use_bias, + use_bn_after_act, + name, + params.l2_regularizer, ) - self._sub_layers.append(GateNN(gate_params, num_units, 'gate_%d' % (i + 1))) + self._sub_layers.append( + GateNN(gate_params, num_units, 'gate_%d' % (i + 1))) n = len(units) - 1 drop_rate = dropout_rate[n] if num_dropout > n else 0.0 name = 'layer_%d' % n self.add_rich_layer( - units[-1], - use_final_bn, - drop_rate, - final_activation, - initializer, - use_final_bias, - use_bn_after_act, - name, - params.l2_regularizer, + units[-1], + use_final_bn, + drop_rate, + final_activation, + initializer, + use_final_bias, + use_bn_after_act, + name, + params.l2_regularizer, ) if mode == 'lazy': - self._sub_layers.append(GateNN(gate_params, units[-1], 'gate_%d' % (n + 1))) + self._sub_layers.append( + GateNN(gate_params, units[-1], 'gate_%d' % (n + 1))) def add_rich_layer( - self, - num_units, - use_bn, - dropout_rate, - activation, - initializer, - use_bias, - use_bn_after_activation, - name, - l2_reg=None, + self, + num_units, + use_bn, + dropout_rate, + activation, + initializer, + use_bias, + use_bn_after_activation, + name, + l2_reg=None, ): act_layer = activation_layer(activation, name='%s/act' % name) if use_bn and not use_bn_after_activation: dense = tf.keras.layers.Dense( - units=num_units, - use_bias=use_bias, - kernel_initializer=initializer, - kernel_regularizer=l2_reg, - name='%s/dense' % name, + units=num_units, + use_bias=use_bias, + kernel_initializer=initializer, + kernel_regularizer=l2_reg, + name='%s/dense' % name, ) self._sub_layers.append(dense) - bn = tf.keras.layers.BatchNormalization(name='%s/bn' % name, trainable=True) + bn = tf.keras.layers.BatchNormalization( + name='%s/bn' % name, trainable=True) self._sub_layers.append(bn) self._sub_layers.append(act_layer) else: dense = tf.keras.layers.Dense( - num_units, - use_bias=use_bias, - kernel_initializer=initializer, - kernel_regularizer=l2_reg, - name='%s/dense' % name, + num_units, + use_bias=use_bias, + kernel_initializer=initializer, + kernel_regularizer=l2_reg, + name='%s/dense' % name, ) self._sub_layers.append(dense) self._sub_layers.append(act_layer) diff --git a/easy_rec/python/layers/keras/transformer.py b/easy_rec/python/layers/keras/transformer.py index 32f0027bc..23e8196f3 100644 --- a/easy_rec/python/layers/keras/transformer.py +++ b/easy_rec/python/layers/keras/transformer.py @@ -4,7 +4,10 @@ import numpy as np import tensorflow as tf -from tensorflow.python.keras.layers import Dense, Dropout, Embedding, Layer +from tensorflow.python.keras.layers import Dense +from tensorflow.python.keras.layers import Dropout +from tensorflow.python.keras.layers import Embedding +from tensorflow.python.keras.layers import Layer from easy_rec.python.layers.keras import MultiHeadAttention from easy_rec.python.layers.keras.layer_norm import LayerNormalization @@ -64,11 +67,13 @@ def positional_encoding(length, depth): depths = np.arange(depth)[np.newaxis, :] / depth # (1, depth) angle_rates = 1 / (10000**depths) # (1, depth) angle_rads = positions * angle_rates # (pos, depth) - pos_encoding = np.concatenate([np.sin(angle_rads), np.cos(angle_rads)], axis=-1) + pos_encoding = np.concatenate( + [np.sin(angle_rads), np.cos(angle_rads)], axis=-1) return tf.cast(pos_encoding, dtype=tf.float32) class PositionalEmbedding(Layer): + def __init__(self, vocab_size, d_model, max_position, name='pos_embedding'): super(PositionalEmbedding, self).__init__(name=name) self.d_model = d_model @@ -104,7 +109,9 @@ def __init__(self, params, name='transformer_encoder', reuse=None, **kwargs): self.output_all = params.get_or_default('output_all_token_embeddings', True) self.pos_encoding = PositionalEmbedding(vocab_size, d_model, max_position) self.dropout = Dropout(dropout_rate) - self.enc_layers = [TransformerBlock(params, 'layer_%d' % i) for i in range(num_layers)] + self.enc_layers = [ + TransformerBlock(params, 'layer_%d' % i) for i in range(num_layers) + ] self._vocab_size = vocab_size self._max_position = max_position @@ -128,6 +135,7 @@ def call(self, inputs, training=None, **kwargs): class TextEncoder(Layer): + def __init__(self, params, name='text_encoder', reuse=None, **kwargs): super(TextEncoder, self).__init__(name=name, **kwargs) self.separator = params.get_or_default('separator', ' ') @@ -140,9 +148,9 @@ def __init__(self, params, name='text_encoder', reuse=None, **kwargs): self.default_token_id = params.get_or_default('default_token_id', 0) if vocab_file is not None: self.vocab = tf.feature_column.categorical_column_with_vocabulary_file( - 'tokens', - vocabulary_file=vocab_file, - default_value=self.default_token_id, + 'tokens', + vocabulary_file=vocab_file, + default_value=self.default_token_id, ) logging.info('vocab file of TextEncoder(%s) is %s', name, vocab_file) trans_params.vocab_size = self.vocab.vocabulary_size @@ -164,23 +172,25 @@ def call(self, inputs, training=None, **kwargs): if self.vocab is not None: features = {'tokens': tokens} token_ids = self.vocab._transform_feature(features) - token_ids = tf.sparse.to_dense(token_ids, default_value=self.default_token_id, name='token_ids') + token_ids = tf.sparse.to_dense( + token_ids, default_value=self.default_token_id, name='token_ids') length = tf.shape(token_ids)[-1] token_ids = tf.cond( - tf.less_equal(length, self.encoder.max_position), - lambda: token_ids, - lambda: tf.slice(token_ids, [0, 0], [-1, self.encoder.max_position]), + tf.less_equal(length, self.encoder.max_position), + lambda: token_ids, + lambda: tf.slice(token_ids, [0, 0], [-1, self.encoder.max_position]), ) mask = tf.not_equal(token_ids, self.default_token_id, name='mask') else: tokens = tf.sparse.to_dense(tokens, default_value='') length = tf.shape(tokens)[-1] tokens = tf.cond( - tf.less_equal(length, self.encoder.max_position), - lambda: tokens, - lambda: tf.slice(tokens, [0, 0], [-1, self.encoder.max_position]), + tf.less_equal(length, self.encoder.max_position), + lambda: tokens, + lambda: tf.slice(tokens, [0, 0], [-1, self.encoder.max_position]), ) - token_ids = tf.string_to_hash_bucket_fast(tokens, self.encoder.vocab_size, name='token_ids') + token_ids = tf.string_to_hash_bucket_fast( + tokens, self.encoder.vocab_size, name='token_ids') mask = tf.not_equal(tokens, '', name='mask') encoding = self.encoder([token_ids, mask], training=training) diff --git a/easy_rec/python/layers/layer_norm.py b/easy_rec/python/layers/layer_norm.py index 441648681..d54bd75b9 100644 --- a/easy_rec/python/layers/layer_norm.py +++ b/easy_rec/python/layers/layer_norm.py @@ -17,16 +17,16 @@ def __init__(self, hidden_size, params={}): def build(self, _): self.scale = tf.get_variable( - 'layer_norm_scale', - [self.hidden_size], - initializer=tf.keras.initializers.Ones(), - dtype=tf.float32, + 'layer_norm_scale', + [self.hidden_size], + initializer=tf.keras.initializers.Ones(), + dtype=tf.float32, ) self.bias = tf.get_variable( - 'layer_norm_bias', - [self.hidden_size], - initializer=tf.keras.initializers.Zeros(), - dtype=tf.float32, + 'layer_norm_bias', + [self.hidden_size], + initializer=tf.keras.initializers.Zeros(), + dtype=tf.float32, ) self.built = True diff --git a/easy_rec/python/layers/mmoe.py b/easy_rec/python/layers/mmoe.py index fcaaa9342..df56d6ed0 100644 --- a/easy_rec/python/layers/mmoe.py +++ b/easy_rec/python/layers/mmoe.py @@ -11,14 +11,15 @@ class MMOE: + def __init__( - self, - expert_dnn_config, - l2_reg, - num_task, - num_expert=None, - name='mmoe', - is_training=False, + self, + expert_dnn_config, + l2_reg, + num_task, + num_expert=None, + name='mmoe', + is_training=False, ): """Initializes a `DNN` Layer. @@ -37,7 +38,7 @@ def __init__( self._num_expert = len(expert_dnn_config) else: assert ( - num_expert is not None and num_expert > 0 + num_expert is not None and num_expert > 0 ), 'param `num_expert` must be large than zero, when expert_dnn_config is not a list' self._expert_dnn_configs = [expert_dnn_config] * num_expert self._num_expert = num_expert @@ -54,10 +55,10 @@ def num_expert(self): def gate(self, unit, deep_fea, name): fea = tf.layers.dense( - inputs=deep_fea, - units=unit, - kernel_regularizer=self._l2_reg, - name='%s/dnn' % name, + inputs=deep_fea, + units=unit, + kernel_regularizer=self._l2_reg, + name='%s/dnn' % name, ) fea = tf.nn.softmax(fea, axis=1) return fea @@ -67,10 +68,10 @@ def __call__(self, deep_fea): for expert_id in range(self._num_expert): expert_dnn_config = self._expert_dnn_configs[expert_id] expert_dnn = dnn.DNN( - expert_dnn_config, - self._l2_reg, - name='%s/expert_%d' % (self._name, expert_id), - is_training=self._is_training, + expert_dnn_config, + self._l2_reg, + name='%s/expert_%d' % (self._name, expert_id), + is_training=self._is_training, ) expert_fea = expert_dnn(deep_fea) expert_fea_list.append(expert_fea) @@ -78,7 +79,8 @@ def __call__(self, deep_fea): task_input_list = [] for task_id in range(self._num_task): - gate = self.gate(self._num_expert, deep_fea, name='%s/gate_%d' % (self._name, task_id)) + gate = self.gate( + self._num_expert, deep_fea, name='%s/gate_%d' % (self._name, task_id)) gate = tf.expand_dims(gate, -1) task_input = tf.multiply(experts_fea, gate) task_input = tf.reduce_sum(task_input, axis=1) diff --git a/easy_rec/python/layers/multihead_attention.py b/easy_rec/python/layers/multihead_attention.py index 09b8fea4f..23443af5b 100644 --- a/easy_rec/python/layers/multihead_attention.py +++ b/easy_rec/python/layers/multihead_attention.py @@ -7,6 +7,7 @@ class MultiHeadAttention: + def __init__(self, head_num, head_size, l2_reg, use_res=False, name=''): """Initializes a `MultiHeadAttention` Layer. @@ -36,11 +37,14 @@ def _split_multihead_qkv(self, q, k, v): k: Key matrix of shape [bs, head_num, feature_num, head_size]. v: Value matrix of shape [bs, head_num, feature_num, head_size]. """ - reshaped_q = tf.reshape(q, shape=[-1, q.shape[1], self._head_num, self._head_size]) + reshaped_q = tf.reshape( + q, shape=[-1, q.shape[1], self._head_num, self._head_size]) q = tf.transpose(reshaped_q, perm=[0, 2, 1, 3]) - reshaped_k = tf.reshape(k, shape=[-1, k.shape[1], self._head_num, self._head_size]) + reshaped_k = tf.reshape( + k, shape=[-1, k.shape[1], self._head_num, self._head_size]) k = tf.transpose(reshaped_k, perm=[0, 2, 1, 3]) - reshaped_v = tf.reshape(v, shape=[-1, v.shape[1], self._head_num, self._head_size]) + reshaped_v = tf.reshape( + v, shape=[-1, v.shape[1], self._head_num, self._head_size]) v = tf.transpose(reshaped_v, perm=[0, 2, 1, 3]) return q, k, v @@ -57,7 +61,9 @@ def _scaled_dot_product_attention(self, q, k, v): k: Key matrix of shape [bs, head_num, feature_num, head_size]. v: Value matrix of shape [bs, head_num, feature_num, head_size]. """ - product = tf.linalg.matmul(a=q, b=k, transpose_b=True) / (self._head_size**-0.5) + product = tf.linalg.matmul( + a=q, b=k, transpose_b=True) / ( + self._head_size**-0.5) weights = tf.nn.softmax(product) out = tf.linalg.matmul(weights, v) return out @@ -76,25 +82,25 @@ def _compute_qkv(self, q, k, v): v: Value matrix of shape [bs, feature_num, head_size * n_head]. """ q = tf.layers.dense( - q, - self._head_num * self._head_size, - use_bias=False, - kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'query'), + q, + self._head_num * self._head_size, + use_bias=False, + kernel_regularizer=self._l2_reg, + name='%s/%s/dnn' % (self._name, 'query'), ) k = tf.layers.dense( - k, - self._head_num * self._head_size, - use_bias=False, - kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'key'), + k, + self._head_num * self._head_size, + use_bias=False, + kernel_regularizer=self._l2_reg, + name='%s/%s/dnn' % (self._name, 'key'), ) v = tf.layers.dense( - v, - self._head_num * self._head_size, - use_bias=False, - kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'value'), + v, + self._head_num * self._head_size, + use_bias=False, + kernel_regularizer=self._l2_reg, + name='%s/%s/dnn' % (self._name, 'value'), ) return q, k, v @@ -122,7 +128,7 @@ def _multi_head_attention(self, attention_input): """ if isinstance(attention_input, list): assert ( - len(attention_input) == 3 or len(attention_input) == 1 + len(attention_input) == 3 or len(attention_input) == 1 ), 'If the input of multi_head_attention is a list, the length must be 1 or 3.' if len(attention_input) == 3: @@ -145,11 +151,11 @@ def _multi_head_attention(self, attention_input): if self._use_res: W_0_x = tf.layers.dense( - ori_v, - out.shape[2], - use_bias=False, - kernel_regularizer=self._l2_reg, - name='%s/dnn' % (self._name), + ori_v, + out.shape[2], + use_bias=False, + kernel_regularizer=self._l2_reg, + name='%s/dnn' % (self._name), ) res_out = tf.nn.relu(out + W_0_x) return res_out diff --git a/easy_rec/python/layers/multihead_cross_attention.py b/easy_rec/python/layers/multihead_cross_attention.py index 6aeff64cf..77697960c 100644 --- a/easy_rec/python/layers/multihead_cross_attention.py +++ b/easy_rec/python/layers/multihead_cross_attention.py @@ -1,6 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import math @@ -38,21 +40,21 @@ def dropout(input_tensor, dropout_prob): def attention_layer( - from_tensor, - to_tensor, - size_per_head, - num_attention_heads=1, - attention_mask=None, - query_act=None, - key_act=None, - value_act=None, - attention_probs_dropout_prob=0.0, - initializer_range=0.02, - do_return_2d_tensor=False, - batch_size=None, - from_seq_length=None, - to_seq_length=None, - reuse=None, + from_tensor, + to_tensor, + size_per_head, + num_attention_heads=1, + attention_mask=None, + query_act=None, + key_act=None, + value_act=None, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + do_return_2d_tensor=False, + batch_size=None, + from_seq_length=None, + to_seq_length=None, + reuse=None, ): """Performs multi-headed attention from `from_tensor` to `to_tensor`. @@ -108,8 +110,10 @@ def attention_layer( ValueError: Any of the arguments or tensor shapes are invalid. """ - def transpose_for_scores(input_tensor, batch_size, num_attention_heads, seq_length, width): - output_tensor = tf.reshape(input_tensor, [batch_size, seq_length, num_attention_heads, width]) + def transpose_for_scores(input_tensor, batch_size, num_attention_heads, + seq_length, width): + output_tensor = tf.reshape( + input_tensor, [batch_size, seq_length, num_attention_heads, width]) output_tensor = tf.transpose(output_tensor, [0, 2, 1, 3]) return output_tensor @@ -118,7 +122,8 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, seq_leng to_shape = get_shape_list(to_tensor, expected_rank=[2, 3]) if len(from_shape) != len(to_shape): - raise ValueError('The rank of `from_tensor` must match the rank of `to_tensor`.') + raise ValueError( + 'The rank of `from_tensor` must match the rank of `to_tensor`.') if len(from_shape) == 3: batch_size = from_shape[0] @@ -127,10 +132,9 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, seq_leng elif len(from_shape) == 2: if batch_size is None or from_seq_length is None or to_seq_length is None: raise ValueError( - 'When passing in rank 2 tensors to attention_layer, the values ' - 'for `batch_size`, `from_seq_length`, and `to_seq_length` ' - 'must all be specified.' - ) + 'When passing in rank 2 tensors to attention_layer, the values ' + 'for `batch_size`, `from_seq_length`, and `to_seq_length` ' + 'must all be specified.') # Scalar dimensions referenced here: # B = batch size (number of sequences) @@ -144,45 +148,49 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, seq_leng # `query_layer` = [B*F, N*H] query_layer = tf.layers.dense( - from_tensor_2d, - num_attention_heads * size_per_head, - activation=query_act, - name='query', - kernel_initializer=create_initializer(initializer_range), - reuse=reuse, + from_tensor_2d, + num_attention_heads * size_per_head, + activation=query_act, + name='query', + kernel_initializer=create_initializer(initializer_range), + reuse=reuse, ) # `key_layer` = [B*T, N*H] key_layer = tf.layers.dense( - to_tensor_2d, - num_attention_heads * size_per_head, - activation=key_act, - name='key', - kernel_initializer=create_initializer(initializer_range), - reuse=reuse, + to_tensor_2d, + num_attention_heads * size_per_head, + activation=key_act, + name='key', + kernel_initializer=create_initializer(initializer_range), + reuse=reuse, ) # `value_layer` = [B*T, N*H] value_layer = tf.layers.dense( - to_tensor_2d, - num_attention_heads * size_per_head, - activation=value_act, - name='value', - kernel_initializer=create_initializer(initializer_range), - reuse=reuse, + to_tensor_2d, + num_attention_heads * size_per_head, + activation=value_act, + name='value', + kernel_initializer=create_initializer(initializer_range), + reuse=reuse, ) # `query_layer` = [B, N, F, H] - query_layer = transpose_for_scores(query_layer, batch_size, num_attention_heads, from_seq_length, size_per_head) + query_layer = transpose_for_scores(query_layer, batch_size, + num_attention_heads, from_seq_length, + size_per_head) # `key_layer` = [B, N, T, H] - key_layer = transpose_for_scores(key_layer, batch_size, num_attention_heads, to_seq_length, size_per_head) + key_layer = transpose_for_scores(key_layer, batch_size, num_attention_heads, + to_seq_length, size_per_head) # Take the dot product between "query" and "key" to get the raw # attention scores. # `attention_scores` = [B, N, F, T] attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) - attention_scores = tf.multiply(attention_scores, 1.0 / math.sqrt(float(size_per_head))) + attention_scores = tf.multiply(attention_scores, + 1.0 / math.sqrt(float(size_per_head))) if attention_mask is not None: # `attention_mask` = [B, 1, F, T] @@ -206,7 +214,9 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, seq_leng attention_probs = dropout(attention_probs, attention_probs_dropout_prob) # `value_layer` = [B, T, N, H] - value_layer = tf.reshape(value_layer, [batch_size, to_seq_length, num_attention_heads, size_per_head]) + value_layer = tf.reshape( + value_layer, + [batch_size, to_seq_length, num_attention_heads, size_per_head]) # `value_layer` = [B, N, T, H] value_layer = tf.transpose(value_layer, [0, 2, 1, 3]) @@ -220,32 +230,32 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, seq_leng if do_return_2d_tensor: # `context_layer` = [B*F, N*H] context_layer = tf.reshape( - context_layer, - [batch_size * from_seq_length, num_attention_heads * size_per_head], + context_layer, + [batch_size * from_seq_length, num_attention_heads * size_per_head], ) else: # `context_layer` = [B, F, N*H] context_layer = tf.reshape( - context_layer, - [batch_size, from_seq_length, num_attention_heads * size_per_head], + context_layer, + [batch_size, from_seq_length, num_attention_heads * size_per_head], ) return context_layer def transformer_encoder( - input_tensor, - attention_mask=None, - hidden_size=768, - num_hidden_layers=12, - num_attention_heads=12, - intermediate_size=3072, - intermediate_act_fn=gelu, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - initializer_range=0.02, - reuse=None, - name='transformer', + input_tensor, + attention_mask=None, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + intermediate_act_fn=gelu, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + reuse=None, + name='transformer', ): """Multi-headed, multi-layer Transformer from "Attention is All You Need". @@ -281,9 +291,8 @@ def transformer_encoder( """ if hidden_size % num_attention_heads != 0: raise ValueError( - 'The hidden size (%d) is not a multiple of the number of attention ' - 'heads (%d)' % (hidden_size, num_attention_heads) - ) + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % (hidden_size, num_attention_heads)) attention_head_size = int(hidden_size / num_attention_heads) input_shape = get_shape_list(input_tensor, expected_rank=3) @@ -294,7 +303,8 @@ def transformer_encoder( # The Transformer performs sum residuals on all layers so the input needs # to be the same as the hidden size. if input_width != hidden_size: - raise ValueError('The width of the input tensor (%d) != hidden size (%d)' % (input_width, hidden_size)) + raise ValueError('The width of the input tensor (%d) != hidden size (%d)' % + (input_width, hidden_size)) # We keep the representation as a 2D tensor to avoid re-shaping it back and # forth from a 3D tensor to a 2D tensor. Re-shapes are normally free on @@ -310,27 +320,27 @@ def transformer_encoder( with tf.variable_scope('self'): # [batch_size * from_seq_length, num_attention_heads * size_per_head] attention_output = attention_layer( - from_tensor=layer_input, - to_tensor=layer_input, - size_per_head=attention_head_size, - num_attention_heads=num_attention_heads, - attention_mask=attention_mask, - attention_probs_dropout_prob=attention_probs_dropout_prob, - initializer_range=initializer_range, - do_return_2d_tensor=True, - batch_size=batch_size, - from_seq_length=seq_length, - to_seq_length=seq_length, - reuse=reuse, + from_tensor=layer_input, + to_tensor=layer_input, + size_per_head=attention_head_size, + num_attention_heads=num_attention_heads, + attention_mask=attention_mask, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + batch_size=batch_size, + from_seq_length=seq_length, + to_seq_length=seq_length, + reuse=reuse, ) # Run a linear projection of `hidden_size` then add a residual # with `layer_input`. with tf.variable_scope('output', reuse=reuse): attention_output = tf.layers.dense( - attention_output, - hidden_size, - kernel_initializer=create_initializer(initializer_range), + attention_output, + hidden_size, + kernel_initializer=create_initializer(initializer_range), ) attention_output = dropout(attention_output, hidden_dropout_prob) attention_output = layer_norm(attention_output + layer_input) @@ -338,18 +348,18 @@ def transformer_encoder( # The activation is only applied to the "intermediate" hidden layer. with tf.variable_scope('intermediate', reuse=reuse): intermediate_output = tf.layers.dense( - attention_output, - intermediate_size, - activation=intermediate_act_fn, - kernel_initializer=create_initializer(initializer_range), + attention_output, + intermediate_size, + activation=intermediate_act_fn, + kernel_initializer=create_initializer(initializer_range), ) # Down-project back to `hidden_size` then add the residual. with tf.variable_scope('output', reuse=reuse): layer_output = tf.layers.dense( - intermediate_output, - hidden_size, - kernel_initializer=create_initializer(initializer_range), + intermediate_output, + hidden_size, + kernel_initializer=create_initializer(initializer_range), ) layer_output = dropout(layer_output, hidden_dropout_prob) layer_output = layer_norm(layer_output + attention_output) @@ -360,18 +370,18 @@ def transformer_encoder( def cross_attention_block( - from_tensor, - to_tensor, - layer_idx, - size_per_head, - cross_attention_mask=None, - self_attention_mask=None, - num_attention_heads=1, - intermediate_size=512, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - initializer_range=0.02, - name='', + from_tensor, + to_tensor, + layer_idx, + size_per_head, + cross_attention_mask=None, + self_attention_mask=None, + num_attention_heads=1, + intermediate_size=512, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + name='', ): """Multi-headed cross attention block. @@ -416,33 +426,33 @@ def cross_attention_block( with tf.variable_scope('cross'): # [batch_size * from_seq_length, num_attention_heads * size_per_head] cross_attention_output = attention_layer( - from_tensor=from_tensor, - to_tensor=to_tensor, - size_per_head=size_per_head, - num_attention_heads=num_attention_heads, - attention_mask=cross_attention_mask, - attention_probs_dropout_prob=attention_probs_dropout_prob, - initializer_range=initializer_range, - do_return_2d_tensor=True, - batch_size=batch_size, - from_seq_length=from_seq_length, - to_seq_length=to_seq_length, + from_tensor=from_tensor, + to_tensor=to_tensor, + size_per_head=size_per_head, + num_attention_heads=num_attention_heads, + attention_mask=cross_attention_mask, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + batch_size=batch_size, + from_seq_length=from_seq_length, + to_seq_length=to_seq_length, ) with tf.variable_scope('self'): # [batch_size * from_seq_length, num_attention_heads * size_per_head] self_attention_output = attention_layer( - from_tensor=cross_attention_output, - to_tensor=cross_attention_output, - size_per_head=size_per_head, - num_attention_heads=num_attention_heads, - attention_mask=self_attention_mask, - attention_probs_dropout_prob=attention_probs_dropout_prob, - initializer_range=initializer_range, - do_return_2d_tensor=True, - batch_size=batch_size, - from_seq_length=from_seq_length, - to_seq_length=from_seq_length, + from_tensor=cross_attention_output, + to_tensor=cross_attention_output, + size_per_head=size_per_head, + num_attention_heads=num_attention_heads, + attention_mask=self_attention_mask, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + batch_size=batch_size, + from_seq_length=from_seq_length, + to_seq_length=from_seq_length, ) with tf.variable_scope('output'): @@ -452,42 +462,44 @@ def cross_attention_block( # The activation is only applied to the "intermediate" hidden layer. with tf.variable_scope('intermediate'): intermediate_output = tf.layers.dense( - attention_output, - intermediate_size, - activation=tf.nn.relu, - kernel_initializer=create_initializer(initializer_range), + attention_output, + intermediate_size, + activation=tf.nn.relu, + kernel_initializer=create_initializer(initializer_range), ) # Down-project back to `hidden_size` then add the residual. with tf.variable_scope('output'): layer_output = tf.layers.dense( - intermediate_output, - num_attention_heads * size_per_head, - kernel_initializer=create_initializer(initializer_range), + intermediate_output, + num_attention_heads * size_per_head, + kernel_initializer=create_initializer(initializer_range), ) layer_output = dropout(layer_output, hidden_dropout_prob) # [batch_size * from_seq_length, num_attention_heads * size_per_head] layer_output = layer_norm(layer_output + attention_output) - final_output = reshape_from_matrix(layer_output, [batch_size, from_seq_length, num_attention_heads * size_per_head]) + final_output = reshape_from_matrix( + layer_output, + [batch_size, from_seq_length, num_attention_heads * size_per_head]) return final_output # [batch_size, from_seq_length, num_attention_heads * size_per_head] def cross_attention_tower( - left_tensor, - right_tensor, - num_hidden_layers=1, - num_attention_heads=12, - left_size_per_head=64, - right_size_per_head=64, - left_intermediate_size=0, - right_intermediate_size=0, - left_input_mask=None, - right_input_mask=None, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - initializer_range=0.02, - name='', + left_tensor, + right_tensor, + num_hidden_layers=1, + num_attention_heads=12, + left_size_per_head=64, + right_size_per_head=64, + left_intermediate_size=0, + right_intermediate_size=0, + left_input_mask=None, + right_input_mask=None, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + name='', ): """Multi-headed, multi layer cross attention block. @@ -529,50 +541,54 @@ def cross_attention_tower( left_attention_mask = None if left_input_mask is not None: - left_attention_mask = create_attention_mask_from_input_mask(left_tensor, left_attention_mask) + left_attention_mask = create_attention_mask_from_input_mask( + left_tensor, left_attention_mask) left_2_right_attention_mask = None if right_input_mask is not None: - left_2_right_attention_mask = create_attention_mask_from_input_mask(left_tensor, right_input_mask) + left_2_right_attention_mask = create_attention_mask_from_input_mask( + left_tensor, right_input_mask) right_attention_mask = None if right_input_mask is not None: - right_attention_mask = create_attention_mask_from_input_mask(right_tensor, right_input_mask) + right_attention_mask = create_attention_mask_from_input_mask( + right_tensor, right_input_mask) right_2_left_attention_mask = None if left_input_mask is not None: - right_2_left_attention_mask = create_attention_mask_from_input_mask(right_tensor, left_input_mask) + right_2_left_attention_mask = create_attention_mask_from_input_mask( + right_tensor, left_input_mask) prev_left_output = left_tensor prev_right_output = right_tensor for layer_idx in range(num_hidden_layers): left_output = cross_attention_block( - prev_left_output, - prev_right_output, - layer_idx, - num_attention_heads=num_attention_heads, - size_per_head=left_size_per_head, - intermediate_size=left_intermediate_size, - hidden_dropout_prob=hidden_dropout_prob, - cross_attention_mask=left_2_right_attention_mask, - self_attention_mask=left_attention_mask, - attention_probs_dropout_prob=attention_probs_dropout_prob, - initializer_range=initializer_range, - name='%sleft_to_right_' % name, + prev_left_output, + prev_right_output, + layer_idx, + num_attention_heads=num_attention_heads, + size_per_head=left_size_per_head, + intermediate_size=left_intermediate_size, + hidden_dropout_prob=hidden_dropout_prob, + cross_attention_mask=left_2_right_attention_mask, + self_attention_mask=left_attention_mask, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + name='%sleft_to_right_' % name, ) right_output = cross_attention_block( - prev_right_output, - prev_left_output, - layer_idx, - num_attention_heads=num_attention_heads, - size_per_head=right_size_per_head, - intermediate_size=right_intermediate_size, - hidden_dropout_prob=hidden_dropout_prob, - cross_attention_mask=right_2_left_attention_mask, - self_attention_mask=right_attention_mask, - attention_probs_dropout_prob=attention_probs_dropout_prob, - initializer_range=initializer_range, - name='%sright_to_left_' % name, + prev_right_output, + prev_left_output, + layer_idx, + num_attention_heads=num_attention_heads, + size_per_head=right_size_per_head, + intermediate_size=right_intermediate_size, + hidden_dropout_prob=hidden_dropout_prob, + cross_attention_mask=right_2_left_attention_mask, + self_attention_mask=right_attention_mask, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + name='%sright_to_left_' % name, ) prev_left_output = left_output prev_right_output = right_output @@ -581,14 +597,16 @@ def cross_attention_tower( def layer_norm(input_tensor, name=None): """Run layer normalization on the last dimension of the tensor.""" - return tf_layer_norm(inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name) + return tf_layer_norm( + inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name) def reshape_to_matrix(input_tensor): """Reshapes a >= rank 2 tensor to a rank 2 tensor (i.e., a matrix).""" ndims = input_tensor.shape.ndims if ndims < 2: - raise ValueError('Input tensor must have at least rank 2. Shape = %s' % (input_tensor.shape)) + raise ValueError('Input tensor must have at least rank 2. Shape = %s' % + (input_tensor.shape)) if ndims == 2: return input_tensor @@ -627,14 +645,16 @@ def create_attention_mask_from_input_mask(from_tensor, to_mask): to_shape = get_shape_list(to_mask, expected_rank=2) to_seq_length = to_shape[1] - to_mask = tf.cast(tf.reshape(to_mask, [batch_size, 1, to_seq_length]), tf.float32) + to_mask = tf.cast( + tf.reshape(to_mask, [batch_size, 1, to_seq_length]), tf.float32) # We don't assume that `from_tensor` is a mask (although it could be). We # don't actually care if we attend *from* padding tokens (only *to* padding) # tokens so we create a tensor of all ones. # # `broadcast_ones` = [batch_size, from_seq_length, 1] - broadcast_ones = tf.ones(shape=tf.stack([batch_size, from_seq_length, 1]), dtype=tf.float32) + broadcast_ones = tf.ones( + shape=tf.stack([batch_size, from_seq_length, 1]), dtype=tf.float32) # Here we broadcast along two dimensions to create the mask. mask = broadcast_ones * to_mask @@ -643,18 +663,18 @@ def create_attention_mask_from_input_mask(from_tensor, to_mask): def embedding_postprocessor( - input_tensor, - use_token_type=False, - token_type_ids=None, - token_type_vocab_size=16, - token_type_embedding_name='token_type_embeddings', - reuse_token_type=None, - use_position_embeddings=True, - position_embedding_name='position_embeddings', - reuse_position_embedding=None, - initializer_range=0.02, - max_position_embeddings=512, - dropout_prob=0.1, + input_tensor, + use_token_type=False, + token_type_ids=None, + token_type_vocab_size=16, + token_type_embedding_name='token_type_embeddings', + reuse_token_type=None, + use_position_embeddings=True, + position_embedding_name='position_embeddings', + reuse_position_embedding=None, + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.1, ): """Performs various post-processing on a word embedding tensor. @@ -694,29 +714,32 @@ def embedding_postprocessor( if use_token_type: if token_type_ids is None: - raise ValueError('`token_type_ids` must be specified if' '`use_token_type` is True.') + raise ValueError('`token_type_ids` must be specified if' + '`use_token_type` is True.') with tf.variable_scope('token_type', reuse=reuse_token_type): token_type_table = tf.get_variable( - name=token_type_embedding_name, - shape=[token_type_vocab_size, width], - initializer=create_initializer(initializer_range), + name=token_type_embedding_name, + shape=[token_type_vocab_size, width], + initializer=create_initializer(initializer_range), ) # This vocab will be small so we always do one-hot here, since it is always # faster for a small vocabulary. flat_token_type_ids = tf.reshape(token_type_ids, [-1]) one_hot_ids = tf.one_hot(flat_token_type_ids, depth=token_type_vocab_size) token_type_embeddings = tf.matmul(one_hot_ids, token_type_table) - token_type_embeddings = tf.reshape(token_type_embeddings, [batch_size, seq_length, width]) + token_type_embeddings = tf.reshape(token_type_embeddings, + [batch_size, seq_length, width]) output += token_type_embeddings if use_position_embeddings: assert_op = tf.assert_less_equal(seq_length, max_position_embeddings) with tf.control_dependencies([assert_op]): - with tf.variable_scope('position_embedding', reuse=reuse_position_embedding): + with tf.variable_scope( + 'position_embedding', reuse=reuse_position_embedding): full_position_embeddings = tf.get_variable( - name=position_embedding_name, - shape=[max_position_embeddings, width], - initializer=create_initializer(initializer_range), + name=position_embedding_name, + shape=[max_position_embeddings, width], + initializer=create_initializer(initializer_range), ) # Since the position embedding table is a learned variable, we create it # using a (long) sequence length `max_position_embeddings`. The actual @@ -727,7 +750,8 @@ def embedding_postprocessor( # for position [0, 1, 2, ..., max_position_embeddings-1], and the current # sequence has positions [0, 1, 2, ... seq_length-1], so we can just # perform a slice. - position_embeddings = tf.slice(full_position_embeddings, [0, 0], [seq_length, -1]) + position_embeddings = tf.slice(full_position_embeddings, [0, 0], + [seq_length, -1]) num_dims = len(output.shape.as_list()) # Only the last two dimensions are relevant (`seq_length` and `width`), so @@ -737,7 +761,8 @@ def embedding_postprocessor( for _ in range(num_dims - 2): position_broadcast_shape.append(1) position_broadcast_shape.extend([seq_length, width]) - position_embeddings = tf.reshape(position_embeddings, position_broadcast_shape) + position_embeddings = tf.reshape(position_embeddings, + position_broadcast_shape) output += position_embeddings output = layer_norm_and_dropout(output, dropout_prob) diff --git a/easy_rec/python/layers/senet.py b/easy_rec/python/layers/senet.py index 91b27d4ec..ce85d7ccc 100644 --- a/easy_rec/python/layers/senet.py +++ b/easy_rec/python/layers/senet.py @@ -21,7 +21,12 @@ class SENet: name: str, name of the layer. """ - def __init__(self, num_fields, num_squeeze_group, reduction_ratio, l2_reg, name='SENet'): + def __init__(self, + num_fields, + num_squeeze_group, + reduction_ratio, + l2_reg, + name='SENet'): self.num_fields = num_fields self.num_squeeze_group = num_squeeze_group self.reduction_ratio = reduction_ratio @@ -38,7 +43,9 @@ def __call__(self, inputs): for input in inputs: emb_size += int(input.shape[-1]) - group_embs = [tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs] + group_embs = [ + tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs + ] squeezed = [] for emb in group_embs: @@ -47,18 +54,18 @@ def __call__(self, inputs): z = tf.concat(squeezed, axis=1) # [bs, field_size * num_groups * 2] reduced = tf.layers.dense( - inputs=z, - units=reduction_size, - kernel_regularizer=self._l2_reg, - activation='relu', - name='%s/reduce' % self._name, + inputs=z, + units=reduction_size, + kernel_regularizer=self._l2_reg, + activation='relu', + name='%s/reduce' % self._name, ) excited_weights = tf.layers.dense( - inputs=reduced, - units=emb_size, - kernel_initializer='glorot_normal', - name='%s/excite' % self._name, + inputs=reduced, + units=emb_size, + kernel_initializer='glorot_normal', + name='%s/excite' % self._name, ) # Re-weight diff --git a/easy_rec/python/layers/seq_input_layer.py b/easy_rec/python/layers/seq_input_layer.py index 35adc27fe..ce3744009 100644 --- a/easy_rec/python/layers/seq_input_layer.py +++ b/easy_rec/python/layers/seq_input_layer.py @@ -17,25 +17,29 @@ class SeqInputLayer(object): + def __init__( - self, - feature_configs, - feature_groups_config, - embedding_regularizer=None, - ev_params=None, + self, + feature_configs, + feature_groups_config, + embedding_regularizer=None, + ev_params=None, ): - self._feature_groups_config = {x.group_name: x for x in feature_groups_config} + self._feature_groups_config = { + x.group_name: x for x in feature_groups_config + } wide_and_deep_dict = self.get_wide_deep_dict() - self._fc_parser = FeatureColumnParser(feature_configs, wide_and_deep_dict, ev_params=ev_params) + self._fc_parser = FeatureColumnParser( + feature_configs, wide_and_deep_dict, ev_params=ev_params) self._embedding_regularizer = embedding_regularizer def __call__( - self, - features, - group_name, - feature_name_to_output_tensors={}, - allow_key_search=True, - scope_name=None, + self, + features, + group_name, + feature_name_to_output_tensors={}, + allow_key_search=True, + scope_name=None, ): feature_column_dict = self._fc_parser.deep_columns feature_column_dict.update(self._fc_parser.sequence_columns) @@ -55,67 +59,75 @@ def _seq_embed_summary_name(input_name): if scope_name is None: scope_name = group_name # name_scope is specified to avoid adding _1 _2 after scope_name - with variable_scope.variable_scope(scope_name, reuse=variable_scope.AUTO_REUSE), ops.name_scope(scope_name + '/'): + with variable_scope.variable_scope( + scope_name, + reuse=variable_scope.AUTO_REUSE), ops.name_scope(scope_name + '/'): key_tensors = [] hist_tensors = [] check_op_list = [] for x in feature_dict.seq_att_map: for key in x.key: if key not in feature_name_to_output_tensors or ( - feature_name_to_output_tensors[key] is None and allow_key_search - ): + feature_name_to_output_tensors[key] is None and allow_key_search): qfc = feature_column_dict[key] with variable_scope.variable_scope(qfc._var_scope_name): - tmp_key_tensor = feature_column_dict[key]._get_dense_tensor(builder) + tmp_key_tensor = feature_column_dict[key]._get_dense_tensor( + builder) regularizers.apply_regularization( - self._embedding_regularizer, - weights_list=[tmp_key_tensor], + self._embedding_regularizer, + weights_list=[tmp_key_tensor], ) key_tensors.append(tmp_key_tensor) elif feature_name_to_output_tensors[key] is None: assert feature_name_to_output_tensors[key] is not None, ( - 'When allow_key_search is False, key: %s should defined in same feature group.' % key - ) + 'When allow_key_search is False, key: %s should defined in same feature group.' + % key) else: key_tensors.append(feature_name_to_output_tensors[key]) if tf_summary: for key_tensor in key_tensors: - tf.summary.histogram(_seq_embed_summary_name(key_tensor.name), key_tensor) + tf.summary.histogram( + _seq_embed_summary_name(key_tensor.name), key_tensor) cur_hist_seqs = [] for hist_seq in x.hist_seq: seq_fc = feature_column_dict[hist_seq] with variable_scope.variable_scope(seq_fc._var_scope_name): - cur_hist_seqs.append(feature_column_dict[hist_seq]._get_sequence_dense_tensor(builder)) + cur_hist_seqs.append( + feature_column_dict[hist_seq]._get_sequence_dense_tensor( + builder)) hist_tensors.extend(cur_hist_seqs) aux_hist_emb_list = [] for aux_hist_seq in x.aux_hist_seq: seq_fc = feature_column_dict[aux_hist_seq] with variable_scope.variable_scope(seq_fc._var_scope_name): - aux_hist_embedding, _ = feature_column_dict[aux_hist_seq]._get_sequence_dense_tensor(builder) + aux_hist_embedding, _ = feature_column_dict[ + aux_hist_seq]._get_sequence_dense_tensor(builder) aux_hist_emb_list.append(aux_hist_embedding) if tf_summary: for hist_embed, hist_seq_len in hist_tensors: - tf.summary.histogram(_seq_embed_summary_name(hist_embed.name), hist_embed) - tf.summary.histogram(_seq_embed_summary_name(hist_seq_len.name), hist_seq_len) + tf.summary.histogram( + _seq_embed_summary_name(hist_embed.name), hist_embed) + tf.summary.histogram( + _seq_embed_summary_name(hist_seq_len.name), hist_seq_len) for idx in range(1, len(cur_hist_seqs)): check_op = tf.assert_equal( - cur_hist_seqs[0][1], - cur_hist_seqs[idx][1], - message='SequenceFeature Error: The size of %s not equal to the size of %s.' - % (x.hist_seq[idx], x.hist_seq[0]), + cur_hist_seqs[0][1], + cur_hist_seqs[idx][1], + message='SequenceFeature Error: The size of %s not equal to the size of %s.' + % (x.hist_seq[idx], x.hist_seq[0]), ) check_op_list.append(check_op) with tf.control_dependencies(check_op_list): features = { - 'key': tf.concat(key_tensors, axis=-1), - 'hist_seq_emb': tf.concat([x[0] for x in hist_tensors], axis=-1), - 'hist_seq_len': hist_tensors[0][1], - 'aux_hist_seq_emb_list': aux_hist_emb_list, + 'key': tf.concat(key_tensors, axis=-1), + 'hist_seq_emb': tf.concat([x[0] for x in hist_tensors], axis=-1), + 'hist_seq_len': hist_tensors[0][1], + 'aux_hist_seq_emb_list': aux_hist_emb_list, } return features diff --git a/easy_rec/python/layers/sequence_feature_layer.py b/easy_rec/python/layers/sequence_feature_layer.py index 58495c28b..196f89483 100644 --- a/easy_rec/python/layers/sequence_feature_layer.py +++ b/easy_rec/python/layers/sequence_feature_layer.py @@ -5,7 +5,8 @@ from tensorflow.python.framework import ops from easy_rec.python.compat import regularizers -from easy_rec.python.layers import dnn, seq_input_layer +from easy_rec.python.layers import dnn +from easy_rec.python.layers import seq_input_layer from easy_rec.python.utils import conditional if tf.__version__ >= '2.0': @@ -13,15 +14,16 @@ class SequenceFeatureLayer(object): + def __init__( - self, - feature_configs, - feature_groups_config, - ev_params=None, - embedding_regularizer=None, - kernel_regularizer=None, - is_training=False, - is_predicting=False, + self, + feature_configs, + feature_groups_config, + ev_params=None, + embedding_regularizer=None, + kernel_regularizer=None, + is_training=False, + is_predicting=False, ): self._seq_feature_groups_config = [] for x in feature_groups_config: @@ -30,10 +32,10 @@ def __init__( self._seq_input_layer = None if len(self._seq_feature_groups_config) > 0: self._seq_input_layer = seq_input_layer.SeqInputLayer( - feature_configs, - self._seq_feature_groups_config, - embedding_regularizer=embedding_regularizer, - ev_params=ev_params, + feature_configs, + self._seq_feature_groups_config, + embedding_regularizer=embedding_regularizer, + ev_params=ev_params, ) self._embedding_regularizer = embedding_regularizer self._kernel_regularizer = kernel_regularizer @@ -41,19 +43,19 @@ def __init__( self._is_predicting = is_predicting def negative_sampler_target_attention( - self, - dnn_config, - deep_fea, - concat_features, - name, - need_key_feature=True, - allow_key_transform=False, + self, + dnn_config, + deep_fea, + concat_features, + name, + need_key_feature=True, + allow_key_transform=False, ): cur_id, hist_id_col, seq_len, aux_hist_emb_list = ( - deep_fea['key'], - deep_fea['hist_seq_emb'], - deep_fea['hist_seq_len'], - deep_fea['aux_hist_seq_emb_list'], + deep_fea['key'], + deep_fea['hist_seq_emb'], + deep_fea['hist_seq_len'], + deep_fea['aux_hist_seq_emb_list'], ) seq_max_len = tf.shape(hist_id_col)[1] @@ -64,37 +66,44 @@ def negative_sampler_target_attention( pos_feature = cur_id[:batch_size] neg_feature = cur_id[batch_size:] cur_id = tf.concat( - [ - pos_feature[:, tf.newaxis, :], - tf.tile(neg_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1]), - ], - axis=1, + [ + pos_feature[:, tf.newaxis, :], + tf.tile( + neg_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1]), + ], + axis=1, ) # noqa: E126 neg_num_add_1 = tf.shape(cur_id)[1] - hist_id_col_tmp = tf.tile(hist_id_col[:, :, :], multiples=[1, neg_num_add_1, 1]) - hist_id_col = tf.reshape(hist_id_col_tmp, [batch_size * neg_num_add_1, seq_max_len, seq_emb_dim]) + hist_id_col_tmp = tf.tile( + hist_id_col[:, :, :], multiples=[1, neg_num_add_1, 1]) + hist_id_col = tf.reshape( + hist_id_col_tmp, [batch_size * neg_num_add_1, seq_max_len, seq_emb_dim]) - concat_features = tf.tile(concat_features[:, tf.newaxis, :], multiples=[1, neg_num_add_1, 1]) + concat_features = tf.tile( + concat_features[:, tf.newaxis, :], multiples=[1, neg_num_add_1, 1]) seq_len = tf.tile(seq_len, multiples=[neg_num_add_1]) if allow_key_transform and (cur_id_dim != seq_emb_dim): - cur_id = tf.layers.dense(cur_id, seq_emb_dim, name='sequence_key_transform_layer') + cur_id = tf.layers.dense( + cur_id, seq_emb_dim, name='sequence_key_transform_layer') cur_ids = tf.tile(cur_id, [1, 1, seq_max_len]) - cur_ids = tf.reshape(cur_ids, tf.shape(hist_id_col)) # (B * neg_num_add_1, seq_max_len, seq_emb_dim) + cur_ids = tf.reshape( + cur_ids, + tf.shape(hist_id_col)) # (B * neg_num_add_1, seq_max_len, seq_emb_dim) din_net = tf.concat( - [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1, + [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], + axis=-1, ) # (B * neg_num_add_1, seq_max_len, seq_emb_dim*4) din_layer = dnn.DNN( - dnn_config, - self._kernel_regularizer, - name, - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True, + dnn_config, + self._kernel_regularizer, + name, + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True, ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) @@ -102,14 +111,16 @@ def negative_sampler_target_attention( seq_len = tf.expand_dims(seq_len, 1) mask = tf.sequence_mask(seq_len) padding = tf.ones_like(scores) * (-(2**32) + 1) - scores = tf.where(mask, scores, padding) # [B*neg_num_add_1, 1, seq_max_len] + scores = tf.where(mask, scores, + padding) # [B*neg_num_add_1, 1, seq_max_len] # Scale scores = tf.nn.softmax(scores) # (B * neg_num_add_1, 1, seq_max_len) - hist_din_emb = tf.matmul(scores, hist_id_col) # [B * neg_num_add_1, 1, seq_emb_dim] - hist_din_emb = tf.reshape( - hist_din_emb, [batch_size, neg_num_add_1, seq_emb_dim] - ) # [B * neg_num_add_1, seq_emb_dim] + hist_din_emb = tf.matmul(scores, + hist_id_col) # [B * neg_num_add_1, 1, seq_emb_dim] + hist_din_emb = tf.reshape(hist_din_emb, + [batch_size, neg_num_add_1, seq_emb_dim + ]) # [B * neg_num_add_1, seq_emb_dim] if len(aux_hist_emb_list) > 0: all_hist_dim_emb = [hist_din_emb] for hist_col in aux_hist_emb_list: @@ -123,19 +134,19 @@ def negative_sampler_target_attention( return din_output, concat_features def target_attention( - self, - dnn_config, - deep_fea, - name, - need_key_feature=True, - allow_key_transform=False, - transform_dnn=False, + self, + dnn_config, + deep_fea, + name, + need_key_feature=True, + allow_key_transform=False, + transform_dnn=False, ): cur_id, hist_id_col, seq_len, aux_hist_emb_list = ( - deep_fea['key'], - deep_fea['hist_seq_emb'], - deep_fea['hist_seq_len'], - deep_fea['aux_hist_seq_emb_list'], + deep_fea['key'], + deep_fea['hist_seq_emb'], + deep_fea['hist_seq_len'], + deep_fea['aux_hist_seq_emb_list'], ) seq_max_len = tf.shape(hist_id_col)[1] @@ -149,25 +160,27 @@ def target_attention( cur_key_layer_name = 'sequence_key_transform_layer_' + name cur_id = tf.layers.dense(cur_id, seq_emb_dim, name=cur_key_layer_name) cur_fea_layer_name = 'sequence_fea_transform_layer_' + name - hist_id_col = tf.layers.dense(hist_id_col, seq_emb_dim, name=cur_fea_layer_name) + hist_id_col = tf.layers.dense( + hist_id_col, seq_emb_dim, name=cur_fea_layer_name) else: - cur_id = cur_id[: tf.shape(hist_id_col)[0], ...] # for negative sampler + cur_id = cur_id[:tf.shape(hist_id_col)[0], ...] # for negative sampler cur_ids = tf.tile(cur_id, [1, seq_max_len]) - cur_ids = tf.reshape(cur_ids, tf.shape(hist_id_col)) # (B, seq_max_len, seq_emb_dim) + cur_ids = tf.reshape(cur_ids, + tf.shape(hist_id_col)) # (B, seq_max_len, seq_emb_dim) din_net = tf.concat( - [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1, + [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], + axis=-1, ) # (B, seq_max_len, seq_emb_dim*4) din_layer = dnn.DNN( - dnn_config, - self._kernel_regularizer, - name, - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True, + dnn_config, + self._kernel_regularizer, + name, + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True, ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) @@ -180,7 +193,8 @@ def target_attention( # Scale scores = tf.nn.softmax(scores) # (B, 1, seq_max_len) hist_din_emb = tf.matmul(scores, hist_id_col) # [B, 1, seq_emb_dim] - hist_din_emb = tf.reshape(hist_din_emb, [-1, seq_emb_dim]) # [B, seq_emb_dim] + hist_din_emb = tf.reshape(hist_din_emb, + [-1, seq_emb_dim]) # [B, seq_emb_dim] if len(aux_hist_emb_list) > 0: all_hist_dim_emb = [hist_din_emb] for hist_col in aux_hist_emb_list: @@ -195,13 +209,13 @@ def target_attention( return din_output def __call__( - self, - features, - concat_features, - all_seq_att_map_config, - feature_name_to_output_tensors=None, - negative_sampler=False, - scope_name=None, + self, + features, + concat_features, + all_seq_att_map_config, + feature_name_to_output_tensors=None, + negative_sampler=False, + scope_name=None, ): logging.info('use sequence feature layer.') all_seq_fea = [] @@ -215,23 +229,27 @@ def __call__( place_on_cpu = os.getenv('place_embedding_on_cpu') place_on_cpu = eval(place_on_cpu) if place_on_cpu else False - with conditional(self._is_predicting and place_on_cpu, ops.device('/CPU:0')): + with conditional(self._is_predicting and place_on_cpu, + ops.device('/CPU:0')): seq_features = self._seq_input_layer( - features, - group_name, - feature_name_to_output_tensors, - allow_key_search, - scope_name, + features, + group_name, + feature_name_to_output_tensors, + allow_key_search, + scope_name, ) # apply regularization for sequence feature key in seq_input_layer. - regularizers.apply_regularization(self._embedding_regularizer, weights_list=[seq_features['hist_seq_emb']]) + regularizers.apply_regularization( + self._embedding_regularizer, + weights_list=[seq_features['hist_seq_emb']]) seq_dnn_config = None if seq_att_map_config.HasField('seq_dnn'): seq_dnn_config = seq_att_map_config.seq_dnn else: - logging.info('seq_dnn not set in seq_att_groups, will use default settings') + logging.info( + 'seq_dnn not set in seq_att_groups, will use default settings') # If not set seq_dnn, will use default settings from easy_rec.python.protos.dnn_pb2 import DNN @@ -240,21 +258,21 @@ def __call__( cur_target_attention_name = 'seq_dnn' + group_name if negative_sampler: seq_fea, concat_features = self.negative_sampler_target_attention( - seq_dnn_config, - seq_features, - concat_features, - name=cur_target_attention_name, - need_key_feature=need_key_feature, - allow_key_transform=allow_key_transform, + seq_dnn_config, + seq_features, + concat_features, + name=cur_target_attention_name, + need_key_feature=need_key_feature, + allow_key_transform=allow_key_transform, ) else: seq_fea = self.target_attention( - seq_dnn_config, - seq_features, - name=cur_target_attention_name, - need_key_feature=need_key_feature, - allow_key_transform=allow_key_transform, - transform_dnn=transform_dnn, + seq_dnn_config, + seq_features, + name=cur_target_attention_name, + need_key_feature=need_key_feature, + allow_key_transform=allow_key_transform, + transform_dnn=transform_dnn, ) all_seq_fea.append(seq_fea) return concat_features, all_seq_fea diff --git a/easy_rec/python/layers/uniter.py b/easy_rec/python/layers/uniter.py index 4d98bc62b..35fbd2e1b 100644 --- a/easy_rec/python/layers/uniter.py +++ b/easy_rec/python/layers/uniter.py @@ -2,7 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import dnn, multihead_cross_attention +from easy_rec.python.layers import dnn +from easy_rec.python.layers import multihead_cross_attention from easy_rec.python.utils.activation import get_activation from easy_rec.python.utils.shape_utils import get_shape_list @@ -17,7 +18,8 @@ class Uniter(object): https://arxiv.org/abs/1909.11740 """ - def __init__(self, model_config, feature_configs, features, uniter_config, input_layer): + def __init__(self, model_config, feature_configs, features, uniter_config, + input_layer): self._model_config = uniter_config tower_num = 0 self._img_features = None @@ -30,7 +32,8 @@ def __init__(self, model_config, feature_configs, features, uniter_config, input tower_num += 1 self._txt_seq_features = None if input_layer.has_group('text'): - self._txt_seq_features, _, _ = input_layer(features, 'text', is_combine=False) + self._txt_seq_features, _, _ = input_layer( + features, 'text', is_combine=False) tower_num += 1 self._use_token_type = True if tower_num > 1 else False self._other_features = None @@ -48,16 +51,20 @@ def __init__(self, model_config, feature_configs, features, uniter_config, input self._general_feature_num = len(fea_group.feature_names) general_feature_names = set(fea_group.feature_names) assert self._general_feature_num == len( - general_feature_names + general_feature_names ), 'there are duplicate features in `general` feature group' elif fea_group.group_name == 'image': self._img_feature_num = len(fea_group.feature_names) img_feature_names = set(fea_group.feature_names) - assert self._img_feature_num == len(img_feature_names), 'there are duplicate features in `image` feature group' + assert self._img_feature_num == len( + img_feature_names + ), 'there are duplicate features in `image` feature group' elif fea_group.group_name == 'text': self._txt_feature_num = len(fea_group.feature_names) txt_feature_names = set(fea_group.feature_names) - assert self._txt_feature_num == len(txt_feature_names), 'there are duplicate features in `text` feature group' + assert self._txt_feature_num == len( + txt_feature_names + ), 'there are duplicate features in `text` feature group' if self._txt_feature_num > 1 or self._img_feature_num > 1: self._use_token_type = True @@ -83,37 +90,40 @@ def __init__(self, model_config, feature_configs, features, uniter_config, input txt_fea_emb_dim_list.append(feature_config.embedding_dim) if feature_config.HasField('max_seq_len'): assert feature_config.max_seq_len > 0, ( - 'feature config `max_seq_len` must be greater than 0 for feature: ' + fea_name - ) + 'feature config `max_seq_len` must be greater than 0 for feature: ' + + fea_name) if feature_config.max_seq_len > max_seq_len: max_seq_len = feature_config.max_seq_len unique_dim_num = len(set(txt_fea_emb_dim_list)) assert ( - unique_dim_num <= 1 and len(txt_fea_emb_dim_list) == self._txt_feature_num + unique_dim_num <= 1 and + len(txt_fea_emb_dim_list) == self._txt_feature_num ), 'Uniter requires that all `text` feature dimensions must be consistent.' unique_dim_num = len(set(img_fea_emb_dim_list)) assert ( - unique_dim_num <= 1 and len(img_fea_emb_dim_list) == self._img_feature_num + unique_dim_num <= 1 and + len(img_fea_emb_dim_list) == self._img_feature_num ), 'Uniter requires that all `image` feature dimensions must be consistent.' unique_dim_num = len(set(general_emb_dim_list)) assert ( - unique_dim_num <= 1 and len(general_emb_dim_list) == self._general_feature_num + unique_dim_num <= 1 and + len(general_emb_dim_list) == self._general_feature_num ), 'Uniter requires that all `general` feature dimensions must be consistent.' if self._txt_feature_num > 0 and uniter_config.use_position_embeddings: assert ( - uniter_config.max_position_embeddings > 0 - ), 'model config `max_position_embeddings` must be greater than 0. ' + uniter_config.max_position_embeddings > + 0), 'model config `max_position_embeddings` must be greater than 0. ' assert uniter_config.max_position_embeddings >= max_seq_len, ( - 'model config `max_position_embeddings` must be greater than' - ' or equal to the maximum of all feature config ' - '`max_seq_len`, which is %d' - ) % max_seq_len + 'model config `max_position_embeddings` must be greater than' + ' or equal to the maximum of all feature config ' + '`max_seq_len`, which is %d') % max_seq_len self._img_emb_size = img_fea_emb_dim_list[0] if img_fea_emb_dim_list else 0 self._txt_emb_size = txt_fea_emb_dim_list[0] if txt_fea_emb_dim_list else 0 - self._general_emb_size = general_emb_dim_list[0] if general_emb_dim_list else 0 + self._general_emb_size = general_emb_dim_list[ + 0] if general_emb_dim_list else 0 if self._img_features is not None: assert self._img_emb_size > 0, '`image` feature dimensions must be greater than 0, set by `raw_input_dim`' @@ -125,27 +135,31 @@ def text_embeddings(self, token_type_id): general_features = self._general_features if self._general_emb_size != hidden_size: # Run a linear projection of `hidden_size` - general_features = tf.reshape(general_features, shape=[-1, self._general_emb_size]) - general_features = tf.layers.dense(general_features, hidden_size, name='txt_projection') - general_features = tf.reshape(general_features, shape=[-1, self._general_feature_num, hidden_size]) + general_features = tf.reshape( + general_features, shape=[-1, self._general_emb_size]) + general_features = tf.layers.dense( + general_features, hidden_size, name='txt_projection') + general_features = tf.reshape( + general_features, shape=[-1, self._general_feature_num, hidden_size]) batch_size = tf.shape(general_features)[0] general_features = multihead_cross_attention.embedding_postprocessor( - general_features, - use_token_type=self._use_token_type, - token_type_ids=tf.ones( - shape=tf.stack([batch_size, self._general_feature_num]), - dtype=tf.int32, - ) - * token_type_id, - token_type_vocab_size=self._token_type_vocab_size, - reuse_token_type=tf.AUTO_REUSE, - use_position_embeddings=False, - dropout_prob=self._model_config.hidden_dropout_prob, + general_features, + use_token_type=self._use_token_type, + token_type_ids=tf.ones( + shape=tf.stack([batch_size, self._general_feature_num]), + dtype=tf.int32, + ) * token_type_id, + token_type_vocab_size=self._token_type_vocab_size, + reuse_token_type=tf.AUTO_REUSE, + use_position_embeddings=False, + dropout_prob=self._model_config.hidden_dropout_prob, ) all_txt_features.append(general_features) - mask = tf.ones(shape=tf.stack([batch_size, self._general_feature_num]), dtype=tf.int32) + mask = tf.ones( + shape=tf.stack([batch_size, self._general_feature_num]), + dtype=tf.int32) input_masks.append(mask) if self._txt_seq_features is not None: @@ -160,25 +174,28 @@ def dynamic_mask(x, max_len): batch_size, max_seq_len, emb_size = get_shape_list(seq_fea, 3) if emb_size != hidden_size: seq_fea = tf.reshape(seq_fea, shape=[-1, emb_size]) - seq_fea = tf.layers.dense(seq_fea, hidden_size, name='txt_seq_projection_%d' % i) + seq_fea = tf.layers.dense( + seq_fea, hidden_size, name='txt_seq_projection_%d' % i) seq_fea = tf.reshape(seq_fea, shape=[-1, max_seq_len, hidden_size]) seq_fea = multihead_cross_attention.embedding_postprocessor( - seq_fea, - use_token_type=self._use_token_type, - token_type_ids=tf.ones(shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * (i + token_type_id), - token_type_vocab_size=self._token_type_vocab_size, - reuse_token_type=tf.AUTO_REUSE, - use_position_embeddings=self._model_config.use_position_embeddings, - max_position_embeddings=self._model_config.max_position_embeddings, - position_embedding_name='txt_position_embeddings_%d' % i, - dropout_prob=self._model_config.hidden_dropout_prob, + seq_fea, + use_token_type=self._use_token_type, + token_type_ids=tf.ones( + shape=tf.stack([batch_size, max_seq_len]), dtype=tf.int32) * + (i + token_type_id), + token_type_vocab_size=self._token_type_vocab_size, + reuse_token_type=tf.AUTO_REUSE, + use_position_embeddings=self._model_config.use_position_embeddings, + max_position_embeddings=self._model_config.max_position_embeddings, + position_embedding_name='txt_position_embeddings_%d' % i, + dropout_prob=self._model_config.hidden_dropout_prob, ) all_txt_features.append(seq_fea) input_mask = tf.map_fn( - fn=lambda t: dynamic_mask(t, max_seq_len), - elems=tf.to_int32(seq_len), + fn=lambda t: dynamic_mask(t, max_seq_len), + elems=tf.to_int32(seq_len), ) input_masks.append(input_mask) @@ -191,21 +208,26 @@ def image_embeddings(self): image_features = self._img_features if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` - image_features = tf.reshape(image_features, shape=[-1, self._img_emb_size]) - image_features = tf.layers.dense(image_features, hidden_size, name='img_projection') - image_features = tf.reshape(image_features, shape=[-1, self._img_feature_num, hidden_size]) + image_features = tf.reshape( + image_features, shape=[-1, self._img_emb_size]) + image_features = tf.layers.dense( + image_features, hidden_size, name='img_projection') + image_features = tf.reshape( + image_features, shape=[-1, self._img_feature_num, hidden_size]) batch_size = tf.shape(image_features)[0] img_fea = multihead_cross_attention.embedding_postprocessor( - image_features, - use_token_type=self._use_token_type, - token_type_ids=tf.zeros(shape=tf.stack([batch_size, self._img_feature_num]), dtype=tf.int32), - token_type_vocab_size=self._token_type_vocab_size, - reuse_token_type=tf.AUTO_REUSE, - use_position_embeddings=self._model_config.use_position_embeddings, - max_position_embeddings=self._model_config.max_position_embeddings, - position_embedding_name='img_position_embeddings', - dropout_prob=self._model_config.hidden_dropout_prob, + image_features, + use_token_type=self._use_token_type, + token_type_ids=tf.zeros( + shape=tf.stack([batch_size, self._img_feature_num]), + dtype=tf.int32), + token_type_vocab_size=self._token_type_vocab_size, + reuse_token_type=tf.AUTO_REUSE, + use_position_embeddings=self._model_config.use_position_embeddings, + max_position_embeddings=self._model_config.max_position_embeddings, + position_embedding_name='img_position_embeddings', + dropout_prob=self._model_config.hidden_dropout_prob, ) return img_fea @@ -240,7 +262,8 @@ def __call__(self, is_training, *args, **kwargs): if img_fea is not None: all_features.append(img_fea) - mask = tf.ones(shape=tf.stack([batch_size, self._img_feature_num]), dtype=tf.int32) + mask = tf.ones( + shape=tf.stack([batch_size, self._img_feature_num]), dtype=tf.int32) masks.append(mask) if txt_features: @@ -250,21 +273,21 @@ def __call__(self, is_training, *args, **kwargs): all_fea = tf.concat(all_features, axis=1) input_mask = tf.concat(masks, axis=1) attention_mask = multihead_cross_attention.create_attention_mask_from_input_mask( - from_tensor=all_fea, to_mask=input_mask - ) + from_tensor=all_fea, to_mask=input_mask) hidden_act = get_activation(self._model_config.hidden_act) attention_fea = multihead_cross_attention.transformer_encoder( - all_fea, - hidden_size=hidden_size, - num_hidden_layers=self._model_config.num_hidden_layers, - num_attention_heads=self._model_config.num_attention_heads, - attention_mask=attention_mask, - intermediate_size=self._model_config.intermediate_size, - intermediate_act_fn=hidden_act, - hidden_dropout_prob=self._model_config.hidden_dropout_prob, - attention_probs_dropout_prob=self._model_config.attention_probs_dropout_prob, - initializer_range=self._model_config.initializer_range, - name='uniter', + all_fea, + hidden_size=hidden_size, + num_hidden_layers=self._model_config.num_hidden_layers, + num_attention_heads=self._model_config.num_attention_heads, + attention_mask=attention_mask, + intermediate_size=self._model_config.intermediate_size, + intermediate_act_fn=hidden_act, + hidden_dropout_prob=self._model_config.hidden_dropout_prob, + attention_probs_dropout_prob=self._model_config + .attention_probs_dropout_prob, + initializer_range=self._model_config.initializer_range, + name='uniter', ) # shape: [batch_size, seq_length, hidden_size] print('attention_fea:', attention_fea.shape) mm_fea = attention_fea[:, 0, :] # [CLS] feature @@ -274,10 +297,10 @@ def __call__(self, is_training, *args, **kwargs): if self._model_config.HasField('other_feature_dnn'): l2_reg = kwargs['l2_reg'] if 'l2_reg' in kwargs else 0 other_dnn_layer = dnn.DNN( - self._model_config.other_feature_dnn, - l2_reg, - 'other_dnn', - is_training, + self._model_config.other_feature_dnn, + l2_reg, + 'other_dnn', + is_training, ) other_fea = other_dnn_layer(self._other_features) else: diff --git a/easy_rec/python/layers/utils.py b/easy_rec/python/layers/utils.py index 4eedb70bf..e650809fd 100644 --- a/easy_rec/python/layers/utils.py +++ b/easy_rec/python/layers/utils.py @@ -14,13 +14,16 @@ # ============================================================================== """Common util functions used by layers.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import json from google.protobuf import struct_pb2 from google.protobuf.descriptor import FieldDescriptor -from tensorflow.python.framework import ops, sparse_tensor +from tensorflow.python.framework import ops +from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import variables try: @@ -33,9 +36,9 @@ def _tensor_to_map(tensor): return { - 'node_path': tensor.name, - 'shape': tensor.shape.as_list() if tensor.shape else None, - 'dtype': tensor.dtype.name, + 'node_path': tensor.name, + 'shape': tensor.shape.as_list() if tensor.shape else None, + 'dtype': tensor.dtype.name, } @@ -77,11 +80,13 @@ def _process_item(collection_name, name, func): if key in ColumnNameInCollection: idx_found = ColumnNameInCollection[key] if idx_found >= len(col): - raise Exception('Find column name in collection failed: index out of range') + raise Exception( + 'Find column name in collection failed: index out of range') item_found = json.loads(col[idx_found]) if item_found['name'] != name: - raise Exception('Find column name in collection failed: item name not match') + raise Exception( + 'Find column name in collection failed: item name not match') func(item_found) col[idx_found] = json.dumps(item_found) else: @@ -91,6 +96,7 @@ def _process_item(collection_name, name, func): def append_attr_to_collection(collection_name, name, key, value): + def append(item_found): if key not in item_found: item_found[key] = [] @@ -100,6 +106,7 @@ def append(item_found): def update_attr_to_collection(collection_name, attrs): + def update(item_found): item_found.update(attrs) @@ -119,7 +126,11 @@ def unique_name_in_collection(collection_name, name): return unique_name -def gen_embedding_attrs(column=None, variable=None, bucket_size=None, combiner=None, is_embedding_var=None): +def gen_embedding_attrs(column=None, + variable=None, + bucket_size=None, + combiner=None, + is_embedding_var=None): attrs = dict() attrs['name'] = column.name attrs['bucket_size'] = bucket_size @@ -131,11 +142,12 @@ def gen_embedding_attrs(column=None, variable=None, bucket_size=None, combiner=N attrs['is_embedding_var'] = True attrs['embedding_var_keys'] = variable._shared_name + '-keys' attrs['embedding_var_values'] = variable._shared_name + '-values' - elif (isinstance(variable, variables.PartitionedVariable)) and ( - isinstance(variable._get_variable_list()[0], kv_variable_ops.EmbeddingVariable) - ): + elif (isinstance(variable, variables.PartitionedVariable)) and (isinstance( + variable._get_variable_list()[0], kv_variable_ops.EmbeddingVariable)): attrs['embedding_var_keys'] = [v._shared_name + '-keys' for v in variable] - attrs['embedding_var_values'] = [v._shared_name + '-values' for v in variable] + attrs['embedding_var_values'] = [ + v._shared_name + '-values' for v in variable + ] else: attrs['is_embedding_var'] = False else: @@ -145,8 +157,11 @@ def gen_embedding_attrs(column=None, variable=None, bucket_size=None, combiner=N def mark_input_src(name, src_desc): ops.add_to_collection( - ops.GraphKeys.RANK_SERVICE_INPUT_SRC, - json.dumps({'name': name, 'src': src_desc}), + ops.GraphKeys.RANK_SERVICE_INPUT_SRC, + json.dumps({ + 'name': name, + 'src': src_desc + }), ) @@ -160,6 +175,7 @@ def is_proto_message(pb_obj, field): class Parameter(object): + def __init__(self, params, is_struct, l2_reg=None): self.params = params self.is_struct = is_struct diff --git a/easy_rec/python/layers/variational_dropout_layer.py b/easy_rec/python/layers/variational_dropout_layer.py index a965aa5f1..270e3a67a 100644 --- a/easy_rec/python/layers/variational_dropout_layer.py +++ b/easy_rec/python/layers/variational_dropout_layer.py @@ -6,11 +6,9 @@ import tensorflow as tf from easy_rec.python.compat.feature_column.feature_column import ( # NOQA - _SharedEmbeddingColumn, -) + _SharedEmbeddingColumn,) from easy_rec.python.compat.feature_column.feature_column_v2 import ( # NOQA - EmbeddingColumn, -) + EmbeddingColumn,) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -24,7 +22,11 @@ class VariationalDropoutLayer(object): arXiv: 1712.08645 """ - def __init__(self, variational_dropout_config, features_dimension, is_training=False, name=''): + def __init__(self, + variational_dropout_config, + features_dimension, + is_training=False, + name=''): self._config = variational_dropout_config self.features_dimension = features_dimension self.features_total_dimension = sum(self.features_dimension.values()) @@ -39,14 +41,14 @@ def __init__(self, variational_dropout_config, features_dimension, is_training=F logit_p_name = 'logit_p' if name == 'all' else 'logit_p_%s' % name self.logit_p = tf.get_variable( - name=logit_p_name, - shape=self.drop_param_shape, - dtype=tf.float32, - initializer=None, + name=logit_p_name, + shape=self.drop_param_shape, + dtype=tf.float32, + initializer=None, ) tf.add_to_collection( - 'variational_dropout', - json.dumps([name, list(self.features_dimension.items())]), + 'variational_dropout', + json.dumps([name, list(self.features_dimension.items())]), ) def get_lambda(self): @@ -64,7 +66,8 @@ def build_expand_index(self, batch_size): expanded_index = tf.tile(expanded_index, [batch_size, 1]) batch_size_range = tf.range(batch_size) expand_range_axis = tf.expand_dims(batch_size_range, 1) - batch_size_range_expand_dim_len = tf.tile(expand_range_axis, [1, self.features_total_dimension]) + batch_size_range_expand_dim_len = tf.tile( + expand_range_axis, [1, self.features_total_dimension]) index_i = tf.reshape(batch_size_range_expand_dim_len, [-1, 1]) expanded_index = tf.concat([index_i, expanded_index], 1) return expanded_index @@ -107,14 +110,12 @@ def sampled_from_logit_p(self, num_samples): def concrete_dropout_neuron(self, dropout_p, temp=1.0 / 10.0): EPSILON = np.finfo(float).eps - unif_noise = tf.random_uniform(tf.shape(dropout_p), dtype=tf.float32, seed=None, name='unif_noise') + unif_noise = tf.random_uniform( + tf.shape(dropout_p), dtype=tf.float32, seed=None, name='unif_noise') approx = ( - tf.log(dropout_p + EPSILON) - - tf.log(1.0 - dropout_p + EPSILON) - + tf.log(unif_noise + EPSILON) - - tf.log(1.0 - unif_noise + EPSILON) - ) + tf.log(dropout_p + EPSILON) - tf.log(1.0 - dropout_p + EPSILON) + + tf.log(unif_noise + EPSILON) - tf.log(1.0 - unif_noise + EPSILON)) approx_output = tf.sigmoid(approx / temp) return 1 - approx_output @@ -124,9 +125,10 @@ def __call__(self, output_features): noisy_input = self.sample_noisy_input(output_features) dropout_p = tf.sigmoid(self.logit_p) variational_dropout_penalty = 1.0 - dropout_p - variational_dropout_penalty_lambda = self.get_lambda() / tf.cast(batch_size, dtype=tf.float32) + variational_dropout_penalty_lambda = self.get_lambda() / tf.cast( + batch_size, dtype=tf.float32) variational_dropout_loss_sum = variational_dropout_penalty_lambda * tf.reduce_sum( - variational_dropout_penalty, axis=0 - ) - tf.add_to_collection('variational_dropout_loss', variational_dropout_loss_sum) + variational_dropout_penalty, axis=0) + tf.add_to_collection('variational_dropout_loss', + variational_dropout_loss_sum) return noisy_input diff --git a/easy_rec/python/loss/circle_loss.py b/easy_rec/python/loss/circle_loss.py index 6e6dc9398..1b442abe1 100644 --- a/easy_rec/python/loss/circle_loss.py +++ b/easy_rec/python/loss/circle_loss.py @@ -6,7 +6,12 @@ tf = tf.compat.v1 -def circle_loss(embeddings, labels, sessions=None, margin=0.25, gamma=32, embed_normed=False): +def circle_loss(embeddings, + labels, + sessions=None, + margin=0.25, + gamma=32, + embed_normed=False): """Paper: Circle Loss: A Unified Perspective of Pair Similarity Optimization. Link: http://arxiv.org/pdf/2002.10857.pdf @@ -19,8 +24,10 @@ def circle_loss(embeddings, labels, sessions=None, margin=0.25, gamma=32, embed_ gamma: parameter of circle loss embed_normed: bool, whether input embeddings l2 normalized """ - norm_embeddings = embeddings if embed_normed else tf.nn.l2_normalize(embeddings, axis=-1) - pair_wise_cosine_matrix = tf.matmul(norm_embeddings, norm_embeddings, transpose_b=True) + norm_embeddings = embeddings if embed_normed else tf.nn.l2_normalize( + embeddings, axis=-1) + pair_wise_cosine_matrix = tf.matmul( + norm_embeddings, norm_embeddings, transpose_b=True) positive_mask = get_anchor_positive_triplet_mask(labels, sessions) negative_mask = 1 - positive_mask - tf.eye(tf.shape(labels)[0]) @@ -28,11 +35,15 @@ def circle_loss(embeddings, labels, sessions=None, margin=0.25, gamma=32, embed_ delta_p = 1 - margin delta_n = margin - ap = tf.nn.relu(-tf.stop_gradient(pair_wise_cosine_matrix * positive_mask) + 1 + margin) - an = tf.nn.relu(tf.stop_gradient(pair_wise_cosine_matrix * negative_mask) + margin) + ap = tf.nn.relu(-tf.stop_gradient(pair_wise_cosine_matrix * positive_mask) + + 1 + margin) + an = tf.nn.relu( + tf.stop_gradient(pair_wise_cosine_matrix * negative_mask) + margin) - logit_p = -ap * (pair_wise_cosine_matrix - delta_p) * gamma * positive_mask - (1 - positive_mask) * 1e12 - logit_n = an * (pair_wise_cosine_matrix - delta_n) * gamma * negative_mask - (1 - negative_mask) * 1e12 + logit_p = -ap * (pair_wise_cosine_matrix - + delta_p) * gamma * positive_mask - (1 - positive_mask) * 1e12 + logit_n = an * (pair_wise_cosine_matrix - + delta_n) * gamma * negative_mask - (1 - negative_mask) * 1e12 joint_neg_loss = tf.reduce_logsumexp(logit_n, axis=-1) joint_pos_loss = tf.reduce_logsumexp(logit_p, axis=-1) @@ -62,7 +73,8 @@ def get_anchor_positive_triplet_mask(labels, sessions=None): if sessions is None or sessions is labels: class_equal = labels_equal else: - sessions_equal = tf.equal(tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1)) + sessions_equal = tf.equal( + tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1)) class_equal = tf.logical_and(sessions_equal, labels_equal) # Combine the three masks diff --git a/easy_rec/python/loss/contrastive_loss.py b/easy_rec/python/loss/contrastive_loss.py index 2e0c7fd2d..3fd2be645 100644 --- a/easy_rec/python/loss/contrastive_loss.py +++ b/easy_rec/python/loss/contrastive_loss.py @@ -30,7 +30,9 @@ def info_nce_loss(query, positive, temperature=0.1): raise ValueError(' must have 2 dimensions.') # Embedding vectors should have same number of components. if query.shape[-1] != positive.shape[-1]: - raise ValueError('Vectors of and should have the same number of components.') + raise ValueError( + 'Vectors of and should have the same number of components.' + ) # Negative keys are implicitly off-diagonal positive keys. @@ -62,8 +64,10 @@ def nce_loss(z_i, z_j, temperature=1.0): N = 2 * batch_size z = tf.concat((z_i, z_j), axis=0) sim = tf.matmul(z, tf.transpose(z)) / temperature - sim_i_j = tf.matrix_diag_part(tf.slice(sim, [batch_size, 0], [batch_size, batch_size])) - sim_j_i = tf.matrix_diag_part(tf.slice(sim, [0, batch_size], [batch_size, batch_size])) + sim_i_j = tf.matrix_diag_part( + tf.slice(sim, [batch_size, 0], [batch_size, batch_size])) + sim_j_i = tf.matrix_diag_part( + tf.slice(sim, [0, batch_size], [batch_size, batch_size])) positive_samples = tf.reshape(tf.concat((sim_i_j, sim_j_i), axis=0), (N, 1)) mask = get_mask_matrix(batch_size) negative_samples = tf.reshape(tf.boolean_mask(sim, mask), (N, -1)) diff --git a/easy_rec/python/loss/f1_reweight_loss.py b/easy_rec/python/loss/f1_reweight_loss.py index c1884a092..3f9689f4d 100644 --- a/easy_rec/python/loss/f1_reweight_loss.py +++ b/easy_rec/python/loss/f1_reweight_loss.py @@ -7,7 +7,11 @@ tf = tf.compat.v1 -def f1_reweight_sigmoid_cross_entropy(labels, logits, beta_square, label_smoothing=0, weights=None): +def f1_reweight_sigmoid_cross_entropy(labels, + logits, + beta_square, + label_smoothing=0, + weights=None): """Refer paper: Adaptive Scaling for Sparse Detection in Information Extraction.""" probs = tf.nn.sigmoid(logits) if len(logits.shape.as_list()) == 1: @@ -23,10 +27,12 @@ def f1_reweight_sigmoid_cross_entropy(labels, logits, beta_square, label_smoothi tn = batch_size_float - tp neg_weight = tp / (beta_square * num_pos + num_neg - tn + 1e-8) neg_weight_tile = tf.tile(tf.expand_dims(neg_weight, 0), [batch_size, 1]) - final_weights = tf.where(tf.equal(labels, 1.0), tf.ones_like(labels), neg_weight_tile) + final_weights = tf.where( + tf.equal(labels, 1.0), tf.ones_like(labels), neg_weight_tile) if weights is not None: weights = tf.cast(weights, tf.float32) if len(weights.shape.as_list()) == 1: weights = tf.expand_dims(weights, -1) final_weights *= weights - return tf.losses.sigmoid_cross_entropy(labels, logits, final_weights, label_smoothing=label_smoothing) + return tf.losses.sigmoid_cross_entropy( + labels, logits, final_weights, label_smoothing=label_smoothing) diff --git a/easy_rec/python/loss/focal_loss.py b/easy_rec/python/loss/focal_loss.py index 52a5fefad..5b681be95 100644 --- a/easy_rec/python/loss/focal_loss.py +++ b/easy_rec/python/loss/focal_loss.py @@ -9,14 +9,14 @@ def sigmoid_focal_loss_with_logits( - labels, - logits, - gamma=2.0, - alpha=None, - ohem_ratio=1.0, - sample_weights=None, - label_smoothing=0, - name='', + labels, + logits, + gamma=2.0, + alpha=None, + ohem_ratio=1.0, + sample_weights=None, + label_smoothing=0, + name='', ): """Implements the focal loss function. @@ -54,10 +54,8 @@ def sigmoid_focal_loss_with_logits( if gamma and gamma < 0: raise ValueError('Value of gamma should be greater than or equal to zero') logging.info( - '[{}] gamma: {}, alpha: {}, ohem_ratho: {}, label smoothing: {}'.format( - loss_name, gamma, alpha, ohem_ratio, label_smoothing - ) - ) + '[{}] gamma: {}, alpha: {}, ohem_ratho: {}, label smoothing: {}'.format( + loss_name, gamma, alpha, ohem_ratio, label_smoothing)) y_true = tf.cast(labels, logits.dtype) @@ -81,14 +79,15 @@ def sigmoid_focal_loss_with_logits( weights *= sample_weights if ohem_ratio == 1.0: - return tf.losses.sigmoid_cross_entropy(y_true, logits, weights=weights, label_smoothing=label_smoothing) + return tf.losses.sigmoid_cross_entropy( + y_true, logits, weights=weights, label_smoothing=label_smoothing) losses = tf.losses.sigmoid_cross_entropy( - y_true, - logits, - weights=weights, - label_smoothing=label_smoothing, - reduction=tf.losses.Reduction.NONE, + y_true, + logits, + weights=weights, + label_smoothing=label_smoothing, + reduction=tf.losses.Reduction.NONE, ) k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio) k = tf.to_int32(tf.math.rint(k)) diff --git a/easy_rec/python/loss/jrc_loss.py b/easy_rec/python/loss/jrc_loss.py index 625db1b5c..6d6c27059 100644 --- a/easy_rec/python/loss/jrc_loss.py +++ b/easy_rec/python/loss/jrc_loss.py @@ -10,14 +10,14 @@ def jrc_loss( - labels, - logits, - session_ids, - alpha=0.5, - loss_weight_strategy='fixed', - sample_weights=1.0, - same_label_loss=True, - name='', + labels, + logits, + session_ids, + alpha=0.5, + loss_weight_strategy='fixed', + sample_weights=1.0, + same_label_loss=True, + name='', ): """Joint Optimization of Ranking and Calibration with Contextualized Hybrid Model. @@ -35,9 +35,11 @@ def jrc_loss( name: the name of loss """ loss_name = name if name else 'jrc_loss' - logging.info('[{}] alpha: {}, loss_weight_strategy: {}'.format(loss_name, alpha, loss_weight_strategy)) + logging.info('[{}] alpha: {}, loss_weight_strategy: {}'.format( + loss_name, alpha, loss_weight_strategy)) - ce_loss = tf.losses.sparse_softmax_cross_entropy(labels, logits, weights=sample_weights) + ce_loss = tf.losses.sparse_softmax_cross_entropy( + labels, logits, weights=sample_weights) labels = tf.expand_dims(labels, 1) # [B, 1] labels = tf.concat([1 - labels, labels], axis=1) # [B, 2] @@ -46,7 +48,8 @@ def jrc_loss( # Mask: shape [B, B], mask[i,j]=1 indicates the i-th sample # and j-th sample are in the same context - mask = tf.equal(tf.expand_dims(session_ids, 1), tf.expand_dims(session_ids, 0)) + mask = tf.equal( + tf.expand_dims(session_ids, 1), tf.expand_dims(session_ids, 0)) mask = tf.to_float(mask) # Tile logits and label: [B, 2]->[B, B, 2] @@ -72,7 +75,8 @@ def jrc_loss( logging.info('[%s] enable same_label_loss' % loss_name) loss_pos = -tf.reduce_sum(y_pos * tf.nn.log_softmax(l_pos, axis=0), axis=0) loss_neg = -tf.reduce_sum(y_neg * tf.nn.log_softmax(l_neg, axis=0), axis=0) - ge_loss = tf.reduce_mean((loss_pos + loss_neg) / tf.reduce_sum(mask, axis=0)) + ge_loss = tf.reduce_mean( + (loss_pos + loss_neg) / tf.reduce_sum(mask, axis=0)) else: logging.info('[%s] disable same_label_loss' % loss_name) diag = tf.one_hot(tf.range(batch_size), batch_size) @@ -103,22 +107,26 @@ def jrc_loss( bern = tf.distributions.Bernoulli(probs=0.5, dtype=tf.float32) weights = bern.sample(2) loss_weight = tf.cond( - tf.equal(tf.reduce_sum(weights), 1), - lambda: weights, - lambda: tf.convert_to_tensor([0.5, 0.5]), + tf.equal(tf.reduce_sum(weights), 1), + lambda: weights, + lambda: tf.convert_to_tensor([0.5, 0.5]), ) loss = loss_weight[0] * ce_loss + loss_weight[1] * ge_loss tf.summary.scalar('loss/%s_ce_weight' % loss_name, loss_weight[0]) tf.summary.scalar('loss/%s_ge_weight' % loss_name, loss_weight[1]) elif loss_weight_strategy == 'uncertainty': - uncertainty1 = tf.Variable(0, name='%s_ranking_loss_weight' % loss_name, dtype=tf.float32) + uncertainty1 = tf.Variable( + 0, name='%s_ranking_loss_weight' % loss_name, dtype=tf.float32) tf.summary.scalar('loss/%s_ranking_uncertainty' % loss_name, uncertainty1) - uncertainty2 = tf.Variable(0, name='%s_calibration_loss_weight' % loss_name, dtype=tf.float32) - tf.summary.scalar('loss/%s_calibration_uncertainty' % loss_name, uncertainty2) + uncertainty2 = tf.Variable( + 0, name='%s_calibration_loss_weight' % loss_name, dtype=tf.float32) + tf.summary.scalar('loss/%s_calibration_uncertainty' % loss_name, + uncertainty2) loss = tf.exp(-uncertainty1) * ce_loss + 0.5 * uncertainty1 loss += tf.exp(-uncertainty2) * ge_loss + 0.5 * uncertainty2 else: - raise ValueError('Unsupported loss weight strategy `%s` for jrc loss' % loss_weight_strategy) + raise ValueError('Unsupported loss weight strategy `%s` for jrc loss' % + loss_weight_strategy) if np.isscalar(sample_weights) and sample_weights != 1.0: return loss * sample_weights return loss diff --git a/easy_rec/python/loss/listwise_loss.py b/easy_rec/python/loss/listwise_loss.py index 83bf47ace..39c8e898c 100644 --- a/easy_rec/python/loss/listwise_loss.py +++ b/easy_rec/python/loss/listwise_loss.py @@ -26,15 +26,15 @@ def _list_prob_loss(x, labels, logits, session_ids): def listwise_rank_loss( - labels, - logits, - session_ids, - transform_fn=None, - temperature=1.0, - label_is_logits=False, - scale_logits=False, - weights=1.0, - name='listwise_loss', + labels, + logits, + session_ids, + transform_fn=None, + temperature=1.0, + label_is_logits=False, + scale_logits=False, + weights=1.0, + name='listwise_loss', ): r"""Computes listwise softmax cross entropy loss between `labels` and `logits`. @@ -58,21 +58,22 @@ def listwise_rank_loss( name: the name of loss """ loss_name = name if name else 'listwise_rank_loss' - logging.info('[{}] temperature: {}, scale logits: {}'.format(loss_name, temperature, scale_logits)) + logging.info('[{}] temperature: {}, scale logits: {}'.format( + loss_name, temperature, scale_logits)) labels = tf.to_float(labels) if scale_logits: with tf.variable_scope(loss_name): w = tf.get_variable( - 'scale_w', - dtype=tf.float32, - shape=(1,), - initializer=tf.ones_initializer(), + 'scale_w', + dtype=tf.float32, + shape=(1,), + initializer=tf.ones_initializer(), ) b = tf.get_variable( - 'scale_b', - dtype=tf.float32, - shape=(1,), - initializer=tf.zeros_initializer(), + 'scale_b', + dtype=tf.float32, + shape=(1,), + initializer=tf.zeros_initializer(), ) logits = logits * tf.abs(w) + b if temperature != 1.0: @@ -86,9 +87,10 @@ def listwise_rank_loss( sessions, _ = tf.unique(tf.squeeze(session_ids)) tf.summary.scalar('loss/%s_num_of_group' % loss_name, tf.size(sessions)) losses = tf.map_fn( - lambda x: _list_wise_loss(x, labels, logits, session_ids, label_is_logits), - sessions, - dtype=tf.float32, + lambda x: _list_wise_loss(x, labels, logits, session_ids, label_is_logits + ), + sessions, + dtype=tf.float32, ) if tf.is_numeric_tensor(weights): logging.error('[%s] use unsupported sample weight' % loss_name) @@ -98,15 +100,15 @@ def listwise_rank_loss( def listwise_distill_loss( - labels, - logits, - session_ids, - transform_fn=None, - temperature=1.0, - label_clip_max_value=512, - scale_logits=False, - weights=1.0, - name='listwise_distill_loss', + labels, + logits, + session_ids, + transform_fn=None, + temperature=1.0, + label_clip_max_value=512, + scale_logits=False, + weights=1.0, + name='listwise_distill_loss', ): r"""Computes listwise softmax cross entropy loss between `labels` and `logits`. @@ -140,16 +142,16 @@ def listwise_distill_loss( if scale_logits: with tf.variable_scope(loss_name): w = tf.get_variable( - 'scale_w', - dtype=tf.float32, - shape=(1,), - initializer=tf.ones_initializer(), + 'scale_w', + dtype=tf.float32, + shape=(1,), + initializer=tf.ones_initializer(), ) b = tf.get_variable( - 'scale_b', - dtype=tf.float32, - shape=(1,), - initializer=tf.zeros_initializer(), + 'scale_b', + dtype=tf.float32, + shape=(1,), + initializer=tf.zeros_initializer(), ) logits = logits * tf.abs(w) + b if temperature != 1.0: @@ -158,9 +160,9 @@ def listwise_distill_loss( sessions, _ = tf.unique(tf.squeeze(session_ids)) tf.summary.scalar('loss/%s_num_of_group' % loss_name, tf.size(sessions)) losses = tf.map_fn( - lambda x: _list_prob_loss(x, labels, logits, session_ids), - sessions, - dtype=tf.float32, + lambda x: _list_prob_loss(x, labels, logits, session_ids), + sessions, + dtype=tf.float32, ) if tf.is_numeric_tensor(weights): logging.error('[%s] use unsupported sample weight' % loss_name) diff --git a/easy_rec/python/loss/multi_similarity.py b/easy_rec/python/loss/multi_similarity.py index 2b7f55e1e..04a863bee 100644 --- a/easy_rec/python/loss/multi_similarity.py +++ b/easy_rec/python/loss/multi_similarity.py @@ -9,15 +9,15 @@ def ms_loss( - embeddings, - labels, - session_ids=None, - alpha=2.0, - beta=50.0, - lamb=1.0, - eps=0.1, - ms_mining=False, - embed_normed=False, + embeddings, + labels, + session_ids=None, + alpha=2.0, + beta=50.0, + lamb=1.0, + eps=0.1, + ms_mining=False, + embed_normed=False, ): """Refer paper: Multi-Similarity Loss with General Pair Weighting for Deep Metric Learning. @@ -35,7 +35,8 @@ def ms_loss( mask_pos = get_anchor_positive_triplet_mask(labels, session_ids) mask_neg = 1 - mask_pos - tf.eye(batch_size) - sim_mat = tf.matmul(embeddings, embeddings, transpose_a=False, transpose_b=True) + sim_mat = tf.matmul( + embeddings, embeddings, transpose_a=False, transpose_b=True) sim_mat = tf.maximum(sim_mat, 0.0) pos_mat = tf.multiply(sim_mat, mask_pos) @@ -44,13 +45,17 @@ def ms_loss( if ms_mining: max_val = tf.reduce_max(neg_mat, axis=1, keepdims=True) tmp_max_val = tf.reduce_max(pos_mat, axis=1, keepdims=True) - min_val = tf.reduce_min(tf.multiply(sim_mat - tmp_max_val, mask_pos), axis=1, keepdims=True) + tmp_max_val + min_val = tf.reduce_min( + tf.multiply(sim_mat - tmp_max_val, mask_pos), axis=1, + keepdims=True) + tmp_max_val max_val = tf.tile(max_val, [1, batch_size]) min_val = tf.tile(min_val, [1, batch_size]) - mask_pos = tf.where(pos_mat < max_val + eps, mask_pos, tf.zeros_like(mask_pos)) - mask_neg = tf.where(neg_mat > min_val - eps, mask_neg, tf.zeros_like(mask_neg)) + mask_pos = tf.where(pos_mat < max_val + eps, mask_pos, + tf.zeros_like(mask_pos)) + mask_neg = tf.where(neg_mat > min_val - eps, mask_neg, + tf.zeros_like(mask_neg)) pos_exp = tf.exp(-alpha * (pos_mat - lamb)) pos_exp = tf.where(mask_pos > 0.0, pos_exp, tf.zeros_like(pos_exp)) diff --git a/easy_rec/python/loss/pairwise_loss.py b/easy_rec/python/loss/pairwise_loss.py index 3be51eb13..604f1ce2e 100644 --- a/easy_rec/python/loss/pairwise_loss.py +++ b/easy_rec/python/loss/pairwise_loss.py @@ -12,7 +12,13 @@ tf = tf.compat.v1 -def pairwise_loss(labels, logits, session_ids=None, margin=0, temperature=1.0, weights=1.0, name=''): +def pairwise_loss(labels, + logits, + session_ids=None, + margin=0, + temperature=1.0, + weights=1.0, + name=''): """Deprecated Pairwise loss. Also see `pairwise_logistic_loss` below. Args: @@ -25,19 +31,22 @@ def pairwise_loss(labels, logits, session_ids=None, margin=0, temperature=1.0, w name: the name of loss """ logging.warning( - 'The old `pairwise_loss` is being deprecated. ' - 'Please use the new `pairwise_logistic_loss` or `pairwise_focal_loss`' - ) + 'The old `pairwise_loss` is being deprecated. ' + 'Please use the new `pairwise_logistic_loss` or `pairwise_focal_loss`') loss_name = name if name else 'pairwise_loss' - logging.info('[{}] margin: {}, temperature: {}'.format(loss_name, margin, temperature)) + logging.info('[{}] margin: {}, temperature: {}'.format( + loss_name, margin, temperature)) if temperature != 1.0: logits /= temperature - pairwise_logits = tf.math.subtract(tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) - margin - pairwise_mask = tf.greater(tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) + pairwise_logits = tf.math.subtract( + tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) - margin + pairwise_mask = tf.greater( + tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) - group_equal = tf.equal(tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) + group_equal = tf.equal( + tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -54,43 +63,42 @@ def pairwise_loss(labels, logits, session_ids=None, margin=0, temperature=1.0, w pairwise_weights = weights pairwise_pseudo_labels = tf.ones_like(pairwise_logits) - loss = tf.losses.sigmoid_cross_entropy(pairwise_pseudo_labels, pairwise_logits, weights=pairwise_weights) + loss = tf.losses.sigmoid_cross_entropy( + pairwise_pseudo_labels, pairwise_logits, weights=pairwise_weights) # set rank loss to zero if a batch has no positive sample. # loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss) return loss -def pairwise_focal_loss( - labels, - logits, - session_ids=None, - hinge_margin=None, - gamma=2, - alpha=None, - ohem_ratio=1.0, - temperature=1.0, - weights=1.0, - name='', -): +def pairwise_focal_loss(labels, + logits, + session_ids=None, + hinge_margin=None, + gamma=2, + alpha=None, + ohem_ratio=1.0, + temperature=1.0, + weights=1.0, + name=''): loss_name = name if name else 'pairwise_focal_loss' assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]' logging.info( - '[{}] hinge margin: {}, gamma: {}, alpha: {}, ohem_ratio: {}, temperature: {}'.format( - loss_name, hinge_margin, gamma, alpha, ohem_ratio, temperature - ) - ) + '[{}] hinge margin: {}, gamma: {}, alpha: {}, ohem_ratio: {}, temperature: {}' + .format(loss_name, hinge_margin, gamma, alpha, ohem_ratio, temperature)) if temperature != 1.0: logits /= temperature pairwise_logits = tf.expand_dims(logits, -1) - tf.expand_dims(logits, 0) - pairwise_mask = tf.greater(tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) + pairwise_mask = tf.greater( + tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) if hinge_margin is not None: hinge_mask = tf.less(pairwise_logits, hinge_margin) pairwise_mask = tf.logical_and(pairwise_mask, hinge_mask) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) - group_equal = tf.equal(tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) + group_equal = tf.equal( + tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -108,27 +116,24 @@ def pairwise_focal_loss( pairwise_pseudo_labels = tf.ones_like(pairwise_logits) loss = sigmoid_focal_loss_with_logits( - pairwise_pseudo_labels, - pairwise_logits, - gamma=gamma, - alpha=alpha, - ohem_ratio=ohem_ratio, - sample_weights=pairwise_weights, - ) + pairwise_pseudo_labels, + pairwise_logits, + gamma=gamma, + alpha=alpha, + ohem_ratio=ohem_ratio, + sample_weights=pairwise_weights) return loss -def pairwise_logistic_loss( - labels, - logits, - session_ids=None, - temperature=1.0, - hinge_margin=None, - weights=1.0, - ohem_ratio=1.0, - use_label_margin=False, - name='', -): +def pairwise_logistic_loss(labels, + logits, + session_ids=None, + temperature=1.0, + hinge_margin=None, + weights=1.0, + ohem_ratio=1.0, + use_label_margin=False, + name=''): r"""Computes pairwise logistic loss between `labels` and `logits`, equivalent to RankNet loss. Definition: @@ -152,25 +157,28 @@ def pairwise_logistic_loss( """ loss_name = name if name else 'pairwise_logistic_loss' assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]' - logging.info( - '[{}] hinge margin: {}, ohem_ratio: {}, temperature: {}'.format(loss_name, hinge_margin, ohem_ratio, temperature) - ) + logging.info('[{}] hinge margin: {}, ohem_ratio: {}, temperature: {}'.format( + loss_name, hinge_margin, ohem_ratio, temperature)) if temperature != 1.0: logits /= temperature if use_label_margin: labels /= temperature - pairwise_logits = tf.math.subtract(tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) + pairwise_logits = tf.math.subtract( + tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) if use_label_margin: - pairwise_logits -= tf.math.subtract(tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) + pairwise_logits -= tf.math.subtract( + tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) elif hinge_margin is not None: pairwise_logits -= hinge_margin - pairwise_mask = tf.greater(tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) + pairwise_mask = tf.greater( + tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) - group_equal = tf.equal(tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) + group_equal = tf.equal( + tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -178,7 +186,8 @@ def pairwise_logistic_loss( tf.summary.scalar('loss/%s_num_of_pairs' % loss_name, num_pair) # The following is the same as log(1 + exp(-pairwise_logits)). - losses = tf.nn.relu(-pairwise_logits) + tf.math.log1p(tf.exp(-tf.abs(pairwise_logits))) + losses = tf.nn.relu(-pairwise_logits) + tf.math.log1p( + tf.exp(-tf.abs(pairwise_logits))) if tf.is_numeric_tensor(weights): logging.info('[%s] use sample weight' % loss_name) @@ -192,7 +201,8 @@ def pairwise_logistic_loss( if ohem_ratio == 1.0: return compute_weighted_loss(losses, pairwise_weights) - losses = compute_weighted_loss(losses, pairwise_weights, reduction=tf.losses.Reduction.NONE) + losses = compute_weighted_loss( + losses, pairwise_weights, reduction=tf.losses.Reduction.NONE) k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio) k = tf.to_int32(tf.math.rint(k)) topk = tf.nn.top_k(losses, k) @@ -200,19 +210,17 @@ def pairwise_logistic_loss( return tf.reduce_mean(losses) -def pairwise_hinge_loss( - labels, - logits, - session_ids=None, - temperature=1.0, - margin=1.0, - weights=1.0, - ohem_ratio=1.0, - label_is_logits=True, - use_label_margin=True, - use_exponent=False, - name='', -): +def pairwise_hinge_loss(labels, + logits, + session_ids=None, + temperature=1.0, + margin=1.0, + weights=1.0, + ohem_ratio=1.0, + label_is_logits=True, + use_label_margin=True, + use_exponent=False, + name=''): r"""Computes pairwise hinge loss between `labels` and `logits`. Definition: @@ -238,18 +246,9 @@ def pairwise_hinge_loss( loss_name = name if name else 'pairwise_hinge_loss' assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]' logging.info( - ( - '[{}] margin: {}, ohem_ratio: {}, temperature: {}, use_exponent: {},' ' label_is_logits: {}, use_label_margin: {}' - ).format( - loss_name, - margin, - ohem_ratio, - temperature, - use_exponent, - label_is_logits, - use_label_margin, - ) - ) + '[{}] margin: {}, ohem_ratio: {}, temperature: {}, use_exponent: {}, label_is_logits: {}, use_label_margin: {}' + .format(loss_name, margin, ohem_ratio, temperature, use_exponent, + label_is_logits, use_label_margin)) if temperature != 1.0: logits /= temperature @@ -259,13 +258,16 @@ def pairwise_hinge_loss( labels = tf.nn.sigmoid(labels) logits = tf.nn.sigmoid(labels) - pairwise_logits = tf.math.subtract(tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) - pairwise_labels = tf.math.subtract(tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) + pairwise_logits = tf.math.subtract( + tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) + pairwise_labels = tf.math.subtract( + tf.expand_dims(labels, -1), tf.expand_dims(labels, 0)) pairwise_mask = tf.greater(pairwise_labels, 0) if session_ids is not None: logging.info('[%s] use session ids' % loss_name) - group_equal = tf.equal(tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) + group_equal = tf.equal( + tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0)) pairwise_mask = tf.logical_and(pairwise_mask, group_equal) pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask) @@ -296,7 +298,8 @@ def pairwise_hinge_loss( if ohem_ratio == 1.0: return compute_weighted_loss(losses, pairwise_weights) - losses = compute_weighted_loss(losses, pairwise_weights, reduction=tf.losses.Reduction.NONE) + losses = compute_weighted_loss( + losses, pairwise_weights, reduction=tf.losses.Reduction.NONE) k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio) k = tf.to_int32(tf.math.rint(k)) topk = tf.nn.top_k(losses, k) diff --git a/easy_rec/python/loss/softmax_loss_with_negative_mining.py b/easy_rec/python/loss/softmax_loss_with_negative_mining.py index a64542db5..643b84689 100644 --- a/easy_rec/python/loss/softmax_loss_with_negative_mining.py +++ b/easy_rec/python/loss/softmax_loss_with_negative_mining.py @@ -6,33 +6,41 @@ tf = tf.compat.v1 -def support_vector_guided_softmax_loss(pos_score, neg_scores, margin=0, t=1, smooth=1.0, threshold=0, weights=1.0): +def support_vector_guided_softmax_loss(pos_score, + neg_scores, + margin=0, + t=1, + smooth=1.0, + threshold=0, + weights=1.0): """Refer paper: Support Vector Guided Softmax Loss for Face Recognition (https://128.84.21.199/abs/1812.11317).""" new_pos_score = pos_score - margin cond = tf.greater_equal(new_pos_score - neg_scores, threshold) - mask = tf.where(cond, tf.zeros_like(cond, tf.float32), tf.ones_like(cond, tf.float32)) # I_k + mask = tf.where(cond, tf.zeros_like(cond, tf.float32), + tf.ones_like(cond, tf.float32)) # I_k new_neg_scores = mask * (neg_scores * t + t - 1) + (1 - mask) * neg_scores logits = tf.concat([new_pos_score, new_neg_scores], axis=1) if 1.0 != smooth: logits *= smooth - loss = tf.losses.sparse_softmax_cross_entropy(tf.zeros_like(pos_score, dtype=tf.int32), logits, weights=weights) + loss = tf.losses.sparse_softmax_cross_entropy( + tf.zeros_like(pos_score, dtype=tf.int32), logits, weights=weights) # set rank loss to zero if a batch has no positive sample. loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss) return loss def softmax_loss_with_negative_mining( - user_emb, - item_emb, - labels, - num_negative_samples=4, - embed_normed=False, - weights=1.0, - gamma=1.0, - margin=0, - t=1, - seed=None, + user_emb, + item_emb, + labels, + num_negative_samples=4, + embed_normed=False, + weights=1.0, + gamma=1.0, + margin=0, + t=1, + seed=None, ): """Compute the softmax loss based on the cosine distance explained below. @@ -66,9 +74,9 @@ def softmax_loss_with_negative_mining( batch_size = tf.shape(item_emb)[0] is_valid = tf.assert_less( - num_negative_samples, - batch_size, - message='`num_negative_samples` should be less than batch_size', + num_negative_samples, + batch_size, + message='`num_negative_samples` should be less than batch_size', ) with tf.control_dependencies([is_valid]): if not embed_normed: @@ -90,9 +98,16 @@ def softmax_loss_with_negative_mining( weights = tf.boolean_mask(weights, mask) # sim_scores's shape: (num_of_pos_label_in_batch_size, num_negative_samples + 1) - sim_scores = tf.keras.backend.batch_dot(mask_user_emb, mask_item_emb, axes=(1, 2)) + sim_scores = tf.keras.backend.batch_dot( + mask_user_emb, mask_item_emb, axes=(1, 2)) pos_score = tf.slice(sim_scores, [0, 0], [-1, 1]) neg_scores = tf.slice(sim_scores, [0, 1], [-1, -1]) - loss = support_vector_guided_softmax_loss(pos_score, neg_scores, margin=margin, t=t, smooth=gamma, weights=weights) + loss = support_vector_guided_softmax_loss( + pos_score, + neg_scores, + margin=margin, + t=t, + smooth=gamma, + weights=weights) return loss diff --git a/easy_rec/python/loss/zero_inflated_lognormal.py b/easy_rec/python/loss/zero_inflated_lognormal.py index b99ea3d02..76072e115 100644 --- a/easy_rec/python/loss/zero_inflated_lognormal.py +++ b/easy_rec/python/loss/zero_inflated_lognormal.py @@ -17,7 +17,10 @@ def log_sigmoid(x): return -tf.nn.softplus(-x) # 兼容 TF 1.12 -def zero_inflated_lognormal_pred(logits, max_sigma=5.0, max_log_clip=20.0, return_log=False): +def zero_inflated_lognormal_pred(logits, + max_sigma=5.0, + max_log_clip=20.0, + return_log=False): """Calculates predicted mean of zero inflated lognormal logits. Arguments: @@ -36,7 +39,8 @@ def zero_inflated_lognormal_pred(logits, max_sigma=5.0, max_log_clip=20.0, retur log_positive_probs = log_sigmoid(logits[..., :1]) mu = logits[..., 1:2] sigma = tf.keras.backend.softplus(logits[..., 2:]) - sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), max_sigma) + sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), + max_sigma) log_mean_pos = mu + 0.5 * tf.keras.backend.square(sigma) log_preds = log_positive_probs + log_mean_pos if return_log: @@ -49,14 +53,14 @@ def zero_inflated_lognormal_pred(logits, max_sigma=5.0, max_log_clip=20.0, retur def zero_inflated_lognormal_loss( - labels, - logits, - max_sigma=5.0, - mu_reg=0.01, - sigma_reg=0.01, - class_weight=1.0, - reg_weight=1.0, - name='', + labels, + logits, + max_sigma=5.0, + mu_reg=0.01, + sigma_reg=0.01, + class_weight=1.0, + reg_weight=1.0, + name='', ): """Computes the zero inflated lognormal loss. @@ -82,27 +86,30 @@ def zero_inflated_lognormal_loss( """ loss_name = name if name else 'ziln_loss' logging.info( - '%s max_sigma=%f, mu_reg=%f, sigma_reg=%f, classify weight:%f, regression weight %f' - % (loss_name, max_sigma, mu_reg, sigma_reg, class_weight, reg_weight) - ) + '%s max_sigma=%f, mu_reg=%f, sigma_reg=%f, classify weight:%f, regression weight %f' + % (loss_name, max_sigma, mu_reg, sigma_reg, class_weight, reg_weight)) labels = tf.cast(labels, dtype=tf.float32) if labels.shape.ndims == 1: labels = tf.expand_dims(labels, 1) # [B, 1] positive = tf.cast(labels > 0, tf.float32) logits = tf.convert_to_tensor(logits, dtype=tf.float32) - logits.shape.assert_is_compatible_with(tf.TensorShape(labels.shape[:-1].as_list() + [3])) + logits.shape.assert_is_compatible_with( + tf.TensorShape(labels.shape[:-1].as_list() + [3])) positive_logits = logits[..., :1] - classification_loss = tf.keras.backend.binary_crossentropy(positive, positive_logits, from_logits=True) + classification_loss = tf.keras.backend.binary_crossentropy( + positive, positive_logits, from_logits=True) classification_loss = tf.keras.backend.mean(classification_loss) tf.summary.scalar('loss/%s_classify' % loss_name, classification_loss) mu = logits[..., 1:2] sigma = tf.keras.backend.softplus(logits[..., 2:]) - sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), max_sigma) + sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), + max_sigma) - safe_labels = positive * labels + (1 - positive) * tf.keras.backend.ones_like(labels) + safe_labels = positive * labels + ( + 1 - positive) * tf.keras.backend.ones_like(labels) logprob = tfd.LogNormal(loc=mu, scale=sigma).log_prob(safe_labels) num_pos = tf.reduce_sum(positive) + 1e-8 regression_loss = -(tf.reduce_sum(positive * logprob) / num_pos) diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index 145bfbf0c..325dabccb 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -1,7 +1,9 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import json import logging @@ -15,27 +17,22 @@ import easy_rec from easy_rec.python.builders import strategy_builder -from easy_rec.python.compat import estimator_train, exporter +from easy_rec.python.compat import estimator_train +from easy_rec.python.compat import exporter from easy_rec.python.input.input import Input from easy_rec.python.model.easy_rec_estimator import EasyRecEstimator from easy_rec.python.model.easy_rec_model import EasyRecModel from easy_rec.python.protos.train_pb2 import DistributionStrategy + from easy_rec.python.utils import ( # NOQA - config_util, - constant, - estimator_utils, - fg_util, - load_class, + config_util, constant, estimator_utils, fg_util, load_class, ) from easy_rec.python.utils.config_util import ( # NOQA - get_eval_input_path, - get_model_dir_path, - get_train_input_path, - set_eval_input_path, + get_eval_input_path, get_model_dir_path, get_train_input_path, + set_eval_input_path, ) from easy_rec.python.utils.export_big_model import ( # NOQA - export_big_model, - export_big_model_to_oss, + export_big_model, export_big_model_to_oss, ) try: @@ -68,12 +65,12 @@ def _get_input_fn( - data_config, - feature_configs, - data_path=None, - export_config=None, - check_mode=False, - **kwargs, + data_config, + feature_configs, + data_path=None, + export_config=None, + check_mode=False, + **kwargs, ): """Build estimator input function. @@ -93,13 +90,13 @@ def _get_input_fn( task_id, task_num = estimator_utils.get_task_index_and_num() input_obj = input_class( - data_config, - feature_configs, - data_path, - task_index=task_id, - task_num=task_num, - check_mode=check_mode, - **kwargs, + data_config, + feature_configs, + data_path, + task_index=task_id, + task_num=task_num, + check_mode=check_mode, + **kwargs, ) input_fn = input_obj.create_input(export_config) return input_fn @@ -110,19 +107,16 @@ def _create_estimator(pipeline_config, distribution=None, params={}): train_config = pipeline_config.train_config gpu_options = GPUOptions(allow_growth=True) # False) - logging.info( - 'train_config.train_distribute=%s[value=%d]' - % ( + logging.info('train_config.train_distribute=%s[value=%d]' % ( DistributionStrategy.Name(pipeline_config.train_config.train_distribute), pipeline_config.train_config.train_distribute, - ) - ) + )) # set gpu options only under hvd scenes if hvd is not None and pipeline_config.train_config.train_distribute in [ - DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy, - DistributionStrategy.HorovodStrategy, + DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy, + DistributionStrategy.HorovodStrategy, ]: local_rnk = hvd.local_rank() gpus = tf.config.experimental.list_physical_devices('GPU') @@ -132,18 +126,17 @@ def _create_estimator(pipeline_config, distribution=None, params={}): gpu_options.visible_device_list = str(local_rnk) session_config = ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=params.get('log_device_placement', False), - inter_op_parallelism_threads=train_config.inter_op_parallelism_threads, - intra_op_parallelism_threads=train_config.intra_op_parallelism_threads, + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=params.get('log_device_placement', False), + inter_op_parallelism_threads=train_config.inter_op_parallelism_threads, + intra_op_parallelism_threads=train_config.intra_op_parallelism_threads, ) if constant.NO_ARITHMETRIC_OPTI in os.environ: logging.info('arithmetic_optimization is closed to improve performance') session_config.graph_options.rewrite_options.arithmetic_optimization = ( - session_config.graph_options.rewrite_options.OFF - ) + session_config.graph_options.rewrite_options.OFF) session_config.device_filters.append('/job:ps') model_cls = EasyRecModel.create_class(model_config.model_class) @@ -160,18 +153,19 @@ def _create_estimator(pipeline_config, distribution=None, params={}): save_checkpoints_steps = train_config.save_checkpoints_steps run_config = tf.estimator.RunConfig( - model_dir=pipeline_config.model_dir, - log_step_count_steps=None, # train_config.log_step_count_steps, - save_summary_steps=train_config.save_summary_steps, - save_checkpoints_steps=save_checkpoints_steps, - save_checkpoints_secs=save_checkpoints_secs, - keep_checkpoint_max=train_config.keep_checkpoint_max, - train_distribute=distribution, - eval_distribute=distribution, - session_config=session_config, + model_dir=pipeline_config.model_dir, + log_step_count_steps=None, # train_config.log_step_count_steps, + save_summary_steps=train_config.save_summary_steps, + save_checkpoints_steps=save_checkpoints_steps, + save_checkpoints_secs=save_checkpoints_secs, + keep_checkpoint_max=train_config.keep_checkpoint_max, + train_distribute=distribution, + eval_distribute=distribution, + session_config=session_config, ) - estimator = EasyRecEstimator(pipeline_config, model_cls, run_config=run_config, params=params) + estimator = EasyRecEstimator( + pipeline_config, model_cls, run_config=run_config, params=params) return estimator, run_config @@ -182,7 +176,8 @@ def _create_eval_export_spec(pipeline_config, eval_data, check_mode=False): eval_config = pipeline_config.eval_config export_config = pipeline_config.export_config if eval_config.num_examples > 0: - eval_steps = int(math.ceil(float(eval_config.num_examples) / data_config.batch_size)) + eval_steps = int( + math.ceil(float(eval_config.num_examples) / data_config.batch_size)) logging.info('eval_steps = %d' % eval_steps) else: eval_steps = None @@ -191,47 +186,47 @@ def _create_eval_export_spec(pipeline_config, eval_data, check_mode=False): input_fn_kwargs['fg_json_path'] = pipeline_config.fg_json_path # create eval input export_input_fn = _get_input_fn( - data_config, - feature_configs, - None, - export_config, - check_mode=check_mode, - **input_fn_kwargs, + data_config, + feature_configs, + None, + export_config, + check_mode=check_mode, + **input_fn_kwargs, ) if export_config.exporter_type == 'final': - exporters = [FinalExporter(name='final', serving_input_receiver_fn=export_input_fn)] + exporters = [ + FinalExporter(name='final', serving_input_receiver_fn=export_input_fn) + ] elif export_config.exporter_type == 'latest': exporters = [ - LatestExporter( - name='latest', - serving_input_receiver_fn=export_input_fn, - exports_to_keep=export_config.exports_to_keep, - ) + LatestExporter( + name='latest', + serving_input_receiver_fn=export_input_fn, + exports_to_keep=export_config.exports_to_keep, + ) ] elif export_config.exporter_type == 'best': logging.info( - 'will use BestExporter, metric is %s, the bigger the better: %d' - % (export_config.best_exporter_metric, export_config.metric_bigger) - ) + 'will use BestExporter, metric is %s, the bigger the better: %d' % + (export_config.best_exporter_metric, export_config.metric_bigger)) def _metric_cmp_fn(best_eval_result, current_eval_result): - logging.info('metric: best = %s current = %s' % (str(best_eval_result), str(current_eval_result))) + logging.info('metric: best = %s current = %s' % + (str(best_eval_result), str(current_eval_result))) if export_config.metric_bigger: - return ( - best_eval_result[export_config.best_exporter_metric] < current_eval_result[export_config.best_exporter_metric] - ) + return (best_eval_result[export_config.best_exporter_metric] < + current_eval_result[export_config.best_exporter_metric]) else: - return ( - best_eval_result[export_config.best_exporter_metric] > current_eval_result[export_config.best_exporter_metric] - ) + return (best_eval_result[export_config.best_exporter_metric] > + current_eval_result[export_config.best_exporter_metric]) exporters = [ - BestExporter( - name='best', - serving_input_receiver_fn=export_input_fn, - compare_fn=_metric_cmp_fn, - exports_to_keep=export_config.exports_to_keep, - ) + BestExporter( + name='best', + serving_input_receiver_fn=export_input_fn, + compare_fn=_metric_cmp_fn, + exports_to_keep=export_config.exports_to_keep, + ) ] elif export_config.exporter_type == 'none': exporters = [] @@ -240,13 +235,14 @@ def _metric_cmp_fn(best_eval_result, current_eval_result): # set throttle_secs to a small number, so that we can control evaluation # interval steps by checkpoint saving steps - eval_input_fn = _get_input_fn(data_config, feature_configs, eval_data, **input_fn_kwargs) + eval_input_fn = _get_input_fn(data_config, feature_configs, eval_data, + **input_fn_kwargs) eval_spec = tf.estimator.EvalSpec( - name='val', - input_fn=eval_input_fn, - steps=eval_steps, - throttle_secs=10, - exporters=exporters, + name='val', + input_fn=eval_input_fn, + steps=eval_steps, + throttle_secs=10, + exporters=exporters, ) return eval_spec @@ -257,10 +253,10 @@ def _check_model_dir(model_dir, continue_train): tf.gfile.MakeDirs(model_dir) else: assert len(tf.gfile.Glob(model_dir + '/model.ckpt-*.meta')) == 0, ( - 'model_dir[=%s] already exists and not empty(if you ' - 'want to continue train on current model_dir please ' - 'delete dir %s or specify --continue_train[internal use only])' % (model_dir, model_dir) - ) + 'model_dir[=%s] already exists and not empty(if you ' + 'want to continue train on current model_dir please ' + 'delete dir %s or specify --continue_train[internal use only])' % + (model_dir, model_dir)) else: if not tf.gfile.IsDirectory(model_dir): logging.info('%s does not exists, create it automatically' % model_dir) @@ -275,10 +271,9 @@ def _get_ckpt_path(pipeline_config, checkpoint_path): ckpt_path = checkpoint_path elif tf.gfile.IsDirectory(pipeline_config.model_dir): ckpt_path = estimator_utils.latest_checkpoint(pipeline_config.model_dir) - logging.info( - 'checkpoint_path is not specified, ' - 'will use latest checkpoint %s from %s' % (ckpt_path, pipeline_config.model_dir) - ) + logging.info('checkpoint_path is not specified, ' + 'will use latest checkpoint %s from %s' % + (ckpt_path, pipeline_config.model_dir)) else: assert False, 'pipeline_config.model_dir(%s) does not exist' % pipeline_config.model_dir return ckpt_path @@ -297,8 +292,10 @@ def train_and_evaluate(pipeline_config_path, continue_train=False): Returns: None, the model will be saved into pipeline_config.model_dir """ - assert tf.gfile.Exists(pipeline_config_path), 'pipeline_config_path not exists' - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) + assert tf.gfile.Exists( + pipeline_config_path), 'pipeline_config_path not exists' + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path) _train_and_evaluate_impl(pipeline_config, continue_train) @@ -306,11 +303,11 @@ def train_and_evaluate(pipeline_config_path, continue_train=False): def _train_and_evaluate_impl( - pipeline_config, - continue_train=False, - check_mode=False, - fit_on_eval=False, - fit_on_eval_steps=None, + pipeline_config, + continue_train=False, + check_mode=False, + fit_on_eval=False, + fit_on_eval_steps=None, ): train_config = pipeline_config.train_config data_config = pipeline_config.data_config @@ -318,9 +315,8 @@ def _train_and_evaluate_impl( if train_config.train_distribute != DistributionStrategy.NoStrategy and train_config.sync_replicas: logging.warning( - 'will set sync_replicas to False, because train_distribute[%s] != NoStrategy' - % pipeline_config.train_config.train_distribute - ) + 'will set sync_replicas to False, because train_distribute[%s] != NoStrategy' + % pipeline_config.train_config.train_distribute) pipeline_config.train_config.sync_replicas = False train_data = get_train_input_path(pipeline_config) @@ -330,7 +326,8 @@ def _train_and_evaluate_impl( params = {} if train_config.is_profiling: params['log_device_placement'] = True - estimator, run_config = _create_estimator(pipeline_config, distribution=distribution, params=params) + estimator, run_config = _create_estimator( + pipeline_config, distribution=distribution, params=params) version_file = os.path.join(pipeline_config.model_dir, 'version') if estimator_utils.is_chief(): @@ -342,9 +339,8 @@ def _train_and_evaluate_impl( train_steps = None if train_config.HasField('num_steps') and train_config.num_steps > 0: train_steps = train_config.num_steps - assert ( - train_steps is not None or data_config.num_epochs > 0 - ), 'either num_steps and num_epochs must be set to an integer > 0.' + assert (train_steps is not None or data_config.num_epochs > 0 + ), 'either num_steps and num_epochs must be set to an integer > 0.' if train_steps and data_config.num_epochs: logging.info('Both num_steps and num_epochs are set.') @@ -362,51 +358,56 @@ def _train_and_evaluate_impl( # create train input train_input_fn = _get_input_fn( - data_config, - feature_configs, - train_data, - check_mode=check_mode, - **input_fn_kwargs, + data_config, + feature_configs, + train_data, + check_mode=check_mode, + **input_fn_kwargs, ) # Currently only a single Eval Spec is allowed. - train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn, max_steps=train_steps) + train_spec = tf.estimator.TrainSpec( + input_fn=train_input_fn, max_steps=train_steps) embedding_parallel = train_config.train_distribute in ( - DistributionStrategy.SokStrategy, - DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy, + DistributionStrategy.EmbeddingParallelStrategy, ) if embedding_parallel: estimator.train( - input_fn=train_input_fn, - max_steps=train_spec.max_steps, - hooks=list(train_spec.hooks), - saving_listeners=train_spec.saving_listeners, + input_fn=train_input_fn, + max_steps=train_spec.max_steps, + hooks=list(train_spec.hooks), + saving_listeners=train_spec.saving_listeners, ) train_input_fn.input_creator.stop() else: # create eval spec - eval_spec = _create_eval_export_spec(pipeline_config, eval_data, check_mode=check_mode) + eval_spec = _create_eval_export_spec( + pipeline_config, eval_data, check_mode=check_mode) estimator_train.train_and_evaluate(estimator, train_spec, eval_spec) logging.info('Train and evaluate finish') if fit_on_eval and (not estimator_utils.is_evaluator()): tf.reset_default_graph() logging.info('Start continue training on eval data') - eval_input_fn = _get_input_fn(data_config, feature_configs, eval_data, **input_fn_kwargs) + eval_input_fn = _get_input_fn(data_config, feature_configs, eval_data, + **input_fn_kwargs) if fit_on_eval_steps is not None: # wait estimator train done to get the correct train_steps while not estimator_train.estimator_train_done(estimator): time.sleep(1) train_steps = estimator_utils.get_trained_steps(estimator.model_dir) - logging.info('\ttrain_steps=%d fit_on_eval_steps=%d' % (train_steps, fit_on_eval_steps)) + logging.info('\ttrain_steps=%d fit_on_eval_steps=%d' % + (train_steps, fit_on_eval_steps)) fit_on_eval_steps += train_steps # Do not use estimator_train.train_and_evaluate as it starts tf.Server, # which is redundant and reports port not available error. estimator.train( - input_fn=eval_input_fn, - max_steps=fit_on_eval_steps, - hooks=list(train_spec.hooks), - saving_listeners=(train_spec.saving_listeners if hasattr(train_spec, 'saving_listeners') else None), + input_fn=eval_input_fn, + max_steps=fit_on_eval_steps, + hooks=list(train_spec.hooks), + saving_listeners=(train_spec.saving_listeners if hasattr( + train_spec, 'saving_listeners') else None), ) logging.info('Finished training on eval data') # return estimator for custom training using estimator.train @@ -414,10 +415,10 @@ def _train_and_evaluate_impl( def evaluate( - pipeline_config, - eval_checkpoint_path='', - eval_data_path=None, - eval_result_filename='eval_result.txt', + pipeline_config, + eval_checkpoint_path='', + eval_data_path=None, + eval_result_filename='eval_result.txt', ): """Evaluate a EasyRec model defined in pipeline_config_path. @@ -458,7 +459,8 @@ def evaluate( if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = server_lib.Server(cluster, job_name='ps', task_index=tf_config['task']['index']) + server = server_lib.Server( + cluster, job_name='ps', task_index=tf_config['task']['index']) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: @@ -474,29 +476,36 @@ def evaluate( if server_target: # evaluate with parameter server - input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() + input_iter = eval_spec.input_fn( + mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() input_feas, input_lbls = input_iter.get_next() from tensorflow.python.framework.ops import device from tensorflow.python.training.device_setter import replica_device_setter from tensorflow.python.training.monitored_session import ( # NOQA - ChiefSessionCreator, - MonitoredSession, + ChiefSessionCreator, MonitoredSession, ) - with device(replica_device_setter(worker_device='/job:master/task:0', cluster=cluster)): - estimator_spec = estimator._eval_model_fn(input_feas, input_lbls, run_config) + with device( + replica_device_setter( + worker_device='/job:master/task:0', cluster=cluster)): + estimator_spec = estimator._eval_model_fn(input_feas, input_lbls, + run_config) - session_config = ConfigProto(allow_soft_placement=True, log_device_placement=True) + session_config = ConfigProto( + allow_soft_placement=True, log_device_placement=True) chief_sess_creator = ChiefSessionCreator( - master=server_target, - checkpoint_filename_with_path=ckpt_path, - config=session_config, + master=server_target, + checkpoint_filename_with_path=ckpt_path, + config=session_config, ) eval_metric_ops = estimator_spec.eval_metric_ops update_ops = [eval_metric_ops[x][1] for x in eval_metric_ops.keys()] metric_ops = {x: eval_metric_ops[x][0] for x in eval_metric_ops.keys()} update_op = tf.group(update_ops) - with MonitoredSession(session_creator=chief_sess_creator, hooks=None, stop_grace_period_secs=120) as sess: + with MonitoredSession( + session_creator=chief_sess_creator, + hooks=None, + stop_grace_period_secs=120) as sess: while True: try: sess.run(update_op) @@ -509,7 +518,8 @@ def evaluate( # with tf.device( # replica_device_setter( # worker_device='/job:master/task:0', cluster=cluster)): - eval_result = estimator.evaluate(eval_spec.input_fn, eval_spec.steps, checkpoint_path=ckpt_path) + eval_result = estimator.evaluate( + eval_spec.input_fn, eval_spec.steps, checkpoint_path=ckpt_path) eval_spec.input_fn.input_creator.stop() logging.info('Evaluate finish') @@ -532,10 +542,10 @@ def evaluate( def distribute_evaluate( - pipeline_config, - eval_checkpoint_path='', - eval_data_path=None, - eval_result_filename='distribute_eval_result.txt', + pipeline_config, + eval_checkpoint_path='', + eval_data_path=None, + eval_result_filename='distribute_eval_result.txt', ): """Evaluate a EasyRec model defined in pipeline_config_path. @@ -567,7 +577,9 @@ def distribute_evaluate( eval_data = get_eval_input_path(pipeline_config) data_config = pipeline_config.data_config if data_config.HasField('sampler'): - logging.warning('It is not accuracy to use eval with negative sampler, recommand to use hitrate.py!') + logging.warning( + 'It is not accuracy to use eval with negative sampler, recommand to use hitrate.py!' + ) eval_result = {} return eval_result model_dir = get_model_dir_path(pipeline_config) @@ -575,7 +587,8 @@ def distribute_evaluate( if not tf.gfile.IsDirectory(eval_tmp_results_dir): logging.info('create eval tmp results dir {}'.format(eval_tmp_results_dir)) tf.gfile.MakeDirs(eval_tmp_results_dir) - assert tf.gfile.IsDirectory(eval_tmp_results_dir), 'tmp results dir not create success.' + assert tf.gfile.IsDirectory( + eval_tmp_results_dir), 'tmp results dir not create success.' os.environ['eval_tmp_results_dir'] = eval_tmp_results_dir server_target = None @@ -587,14 +600,16 @@ def distribute_evaluate( if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = server_lib.Server(cluster, job_name='ps', task_index=tf_config['task']['index']) + server = server_lib.Server( + cluster, job_name='ps', task_index=tf_config['task']['index']) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: cur_job_name = tf_config['task']['type'] cur_task_index = tf_config['task']['index'] cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = server_lib.Server(cluster, job_name=cur_job_name, task_index=cur_task_index) + server = server_lib.Server( + cluster, job_name=cur_job_name, task_index=cur_task_index) server_target = server.target print('server_target = %s' % server_target) elif tf_config['task']['type'] == 'worker': @@ -602,7 +617,8 @@ def distribute_evaluate( cur_job_name = tf_config['task']['type'] cur_task_index = tf_config['task']['index'] cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = server_lib.Server(cluster, job_name=cur_job_name, task_index=cur_task_index) + server = server_lib.Server( + cluster, job_name=cur_job_name, task_index=cur_task_index) server_target = server.target print('server_target = %s' % server_target) @@ -610,45 +626,53 @@ def distribute_evaluate( from tensorflow.python.framework.ops import device from tensorflow.python.training.device_setter import replica_device_setter from tensorflow.python.training.monitored_session import ( # NOQA - ChiefSessionCreator, - MonitoredSession, - WorkerSessionCreator, + ChiefSessionCreator, MonitoredSession, WorkerSessionCreator, ) from easy_rec.python.utils.estimator_utils import EvaluateExitBarrierHook cur_work_device = '/job:' + cur_job_name + '/task:' + str(cur_task_index) cur_ps_num = len(tf_config['cluster']['ps']) - with device(replica_device_setter(ps_tasks=cur_ps_num, worker_device=cur_work_device, cluster=cluster)): + with device( + replica_device_setter( + ps_tasks=cur_ps_num, worker_device=cur_work_device, + cluster=cluster)): distribution = strategy_builder.build(train_config) estimator, run_config = _create_estimator(pipeline_config, distribution) eval_spec = _create_eval_export_spec(pipeline_config, eval_data) ckpt_path = _get_ckpt_path(pipeline_config, eval_checkpoint_path) ckpt_dir = os.path.dirname(ckpt_path) - input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() + input_iter = eval_spec.input_fn( + mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() input_feas, input_lbls = input_iter.get_next() - estimator_spec = estimator._distribute_eval_model_fn(input_feas, input_lbls, run_config) + estimator_spec = estimator._distribute_eval_model_fn( + input_feas, input_lbls, run_config) session_config = ConfigProto( - allow_soft_placement=True, - log_device_placement=True, - device_filters=['/job:ps', '/job:worker/task:%d' % cur_task_index], + allow_soft_placement=True, + log_device_placement=True, + device_filters=['/job:ps', + '/job:worker/task:%d' % cur_task_index], ) if cur_job_name == 'master': metric_variables = tf.get_collection(tf.GraphKeys.METRIC_VARIABLES) model_ready_for_local_init_op = tf.variables_initializer(metric_variables) global_variables = tf.global_variables() - remain_variables = list(set(global_variables).difference(set(metric_variables))) + remain_variables = list( + set(global_variables).difference(set(metric_variables))) cur_saver = tf.train.Saver(var_list=remain_variables, sharded=True) - cur_scaffold = tf.train.Scaffold(saver=cur_saver, ready_for_local_init_op=model_ready_for_local_init_op) + cur_scaffold = tf.train.Scaffold( + saver=cur_saver, + ready_for_local_init_op=model_ready_for_local_init_op) cur_sess_creator = ChiefSessionCreator( - scaffold=cur_scaffold, - master=server_target, - checkpoint_filename_with_path=ckpt_path, - config=session_config, + scaffold=cur_scaffold, + master=server_target, + checkpoint_filename_with_path=ckpt_path, + config=session_config, ) else: - cur_sess_creator = WorkerSessionCreator(master=server_target, config=session_config) + cur_sess_creator = WorkerSessionCreator( + master=server_target, config=session_config) eval_metric_ops = estimator_spec.eval_metric_ops update_ops = [eval_metric_ops[x][1] for x in eval_metric_ops.keys()] metric_ops = {x: eval_metric_ops[x][0] for x in eval_metric_ops.keys()} @@ -656,14 +680,16 @@ def distribute_evaluate( cur_worker_num = len(tf_config['cluster']['worker']) + 1 if cur_job_name == 'master': cur_stop_grace_period_sesc = 120 - cur_hooks = EvaluateExitBarrierHook(cur_worker_num, True, ckpt_dir, metric_ops) + cur_hooks = EvaluateExitBarrierHook(cur_worker_num, True, ckpt_dir, + metric_ops) else: cur_stop_grace_period_sesc = 10 - cur_hooks = EvaluateExitBarrierHook(cur_worker_num, False, ckpt_dir, metric_ops) + cur_hooks = EvaluateExitBarrierHook(cur_worker_num, False, ckpt_dir, + metric_ops) with MonitoredSession( - session_creator=cur_sess_creator, - hooks=[cur_hooks], - stop_grace_period_secs=cur_stop_grace_period_sesc, + session_creator=cur_sess_creator, + hooks=[cur_hooks], + stop_grace_period_secs=cur_stop_grace_period_sesc, ) as sess: while True: try: @@ -734,12 +760,12 @@ def predict(pipeline_config, checkpoint_path='', data_path=None): def export( - export_dir, - pipeline_config, - checkpoint_path='', - asset_files=None, - verbose=False, - **extra_params, + export_dir, + pipeline_config, + checkpoint_path='', + asset_files=None, + verbose=False, + **extra_params, ): """Export model defined in pipeline_config_path. @@ -785,7 +811,8 @@ def export( logging.info('will add asset files: %s' % asset_files) for asset_file in asset_files.split(','): asset_file = asset_file.strip() - if ':' not in asset_file or asset_file.startswith('oss:') or asset_file.startswith('hdfs:'): + if ':' not in asset_file or asset_file.startswith( + 'oss:') or asset_file.startswith('hdfs:'): _, asset_name = os.path.split(asset_file) else: asset_name, asset_file = asset_file.split(':', 1) @@ -798,7 +825,8 @@ def export( input_fn_kwargs = {'pipeline_config': pipeline_config} if data_config.input_type == data_config.InputType.OdpsRTPInputV2: input_fn_kwargs['fg_json_path'] = pipeline_config.fg_json_path - serving_input_fn = _get_input_fn(data_config, feature_configs, None, export_config, **input_fn_kwargs) + serving_input_fn = _get_input_fn(data_config, feature_configs, None, + export_config, **input_fn_kwargs) ckpt_path = _get_ckpt_path(pipeline_config, checkpoint_path) if 'oss_path' in extra_params: if pipeline_config.train_config.HasField('incr_save_config'): @@ -807,40 +835,43 @@ def export( incr_save_type = incr_save_config.WhichOneof('incr_update') logging.info('incr_save_type=%s' % incr_save_type) if incr_save_type: - extra_params['incr_update'][incr_save_type] = getattr(incr_save_config, incr_save_type) + extra_params['incr_update'][incr_save_type] = getattr( + incr_save_config, incr_save_type) return export_big_model_to_oss( - export_dir, - pipeline_config, - extra_params, - serving_input_fn, - estimator, - ckpt_path, - verbose, + export_dir, + pipeline_config, + extra_params, + serving_input_fn, + estimator, + ckpt_path, + verbose, ) if 'redis_url' in extra_params: return export_big_model( - export_dir, - pipeline_config, - extra_params, - serving_input_fn, - estimator, - ckpt_path, - verbose, + export_dir, + pipeline_config, + extra_params, + serving_input_fn, + estimator, + ckpt_path, + verbose, ) final_export_dir = estimator.export_savedmodel( - export_dir_base=export_dir, - serving_input_receiver_fn=serving_input_fn, - checkpoint_path=ckpt_path, - strip_default_attrs=True, + export_dir_base=export_dir, + serving_input_receiver_fn=serving_input_fn, + checkpoint_path=ckpt_path, + strip_default_attrs=True, ) # add export ts as version info saved_model = saved_model_pb2.SavedModel() if type(final_export_dir) not in [type(''), type('')]: final_export_dir = final_export_dir.decode('utf-8') - export_ts = [x for x in final_export_dir.split('/') if x != '' and x is not None] + export_ts = [ + x for x in final_export_dir.split('/') if x != '' and x is not None + ] export_ts = export_ts[-1] saved_pb_path = os.path.join(final_export_dir, 'saved_model.pb') with tf.gfile.GFile(saved_pb_path, 'rb') as fin: @@ -870,12 +901,12 @@ def export( def export_checkpoint( - pipeline_config=None, - export_path='', - checkpoint_path='', - asset_files=None, - verbose=False, - mode=tf.estimator.ModeKeys.PREDICT, + pipeline_config=None, + export_path='', + checkpoint_path='', + asset_files=None, + verbose=False, + mode=tf.estimator.ModeKeys.PREDICT, ): """Export the EasyRec model as checkpoint.""" pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config) @@ -897,13 +928,14 @@ def export_checkpoint( # construct serving input fn export_config = pipeline_config.export_config - serving_input_fn = _get_input_fn(data_config, feature_configs, None, export_config, **input_fn_kwargs) + serving_input_fn = _get_input_fn(data_config, feature_configs, None, + export_config, **input_fn_kwargs) ckpt_path = _get_ckpt_path(pipeline_config, checkpoint_path) estimator.export_checkpoint( - export_path=export_path, - serving_input_receiver_fn=serving_input_fn, - checkpoint_path=ckpt_path, - mode=mode, + export_path=export_path, + serving_input_receiver_fn=serving_input_fn, + checkpoint_path=ckpt_path, + mode=mode, ) logging.info('model checkpoint has been exported successfully') diff --git a/easy_rec/python/model/autoint.py b/easy_rec/python/model/autoint.py index 5156426fa..cf63ef4fd 100644 --- a/easy_rec/python/model/autoint.py +++ b/easy_rec/python/model/autoint.py @@ -6,6 +6,7 @@ from easy_rec.python.layers import multihead_attention from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.autoint_pb2 import AutoInt as AutoIntConfig # NOQA if tf.__version__ >= '2.0': @@ -13,11 +14,17 @@ class AutoInt(RankModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(AutoInt, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(AutoInt, self).__init__(model_config, feature_configs, features, + labels, is_training) assert self._model_config.WhichOneof('model') == 'autoint', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._features, _ = self._input_layer(self._feature_dict, 'all') self._feature_num = len(self._model_config.feature_groups[0].feature_names) self._seq_key_num = 0 @@ -33,7 +40,8 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai for feature_config in feature_configs: fea_emb_dim_list.append(feature_config.embedding_dim) assert ( - len(set(fea_emb_dim_list)) == 1 and len(fea_emb_dim_list) == self._feature_num + len(set(fea_emb_dim_list)) == 1 and + len(fea_emb_dim_list) == self._feature_num ), 'AutoInt requires that all feature dimensions must be consistent.' self._d_model = fea_emb_dim_list[0] @@ -44,21 +52,23 @@ def build_predict_graph(self): logging.info('feature_num: {0}'.format(self._feature_num)) attention_fea = tf.reshape( - self._features, - shape=[-1, self._feature_num + self._seq_key_num, self._d_model], + self._features, + shape=[-1, self._feature_num + self._seq_key_num, self._d_model], ) for i in range(self._model_config.interacting_layer_num): attention_layer = multihead_attention.MultiHeadAttention( - head_num=self._head_num, - head_size=self._head_size, - l2_reg=self._l2_reg, - use_res=True, - name='multi_head_self_attention_layer_%d' % i, + head_num=self._head_num, + head_size=self._head_size, + l2_reg=self._l2_reg, + use_res=True, + name='multi_head_self_attention_layer_%d' % i, ) attention_fea = attention_layer(attention_fea) - attention_fea = tf.reshape(attention_fea, shape=[-1, attention_fea.shape[1] * attention_fea.shape[2]]) + attention_fea = tf.reshape( + attention_fea, + shape=[-1, attention_fea.shape[1] * attention_fea.shape[2]]) final = tf.layers.dense(attention_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/cmbf.py b/easy_rec/python/model/cmbf.py index e65178322..6c8dd6b0d 100644 --- a/easy_rec/python/model/cmbf.py +++ b/easy_rec/python/model/cmbf.py @@ -2,8 +2,10 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import cmbf, dnn +from easy_rec.python.layers import cmbf +from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.cmbf_pb2 import CMBF as CMBFConfig # NOQA if tf.__version__ >= '2.0': @@ -18,24 +20,32 @@ class CMBF(RankModel): https://www.mdpi.com/1424-8220/21/16/5275 """ - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(CMBF, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'cmbf', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model' - ) + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(CMBF, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert self._model_config.WhichOneof( + 'model' + ) == 'cmbf', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._cmbf_layer = cmbf.CMBF( - model_config, - feature_configs, - features, - self._model_config.cmbf.config, - self._input_layer, + model_config, + feature_configs, + features, + self._model_config.cmbf.config, + self._input_layer, ) self._model_config = self._model_config.cmbf def build_predict_graph(self): hidden = self._cmbf_layer(self._is_training, l2_reg=self._l2_reg) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, + 'final_dnn', self._is_training) all_fea = final_dnn_layer(hidden) final = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/collaborative_metric_learning.py b/easy_rec/python/model/collaborative_metric_learning.py index 066b946f8..d2857d91c 100644 --- a/easy_rec/python/model/collaborative_metric_learning.py +++ b/easy_rec/python/model/collaborative_metric_learning.py @@ -1,35 +1,36 @@ import tensorflow as tf -from easy_rec.python.core.metrics import ( # NOQA - metric_learning_average_precision_at_k, - metric_learning_recall_at_k, -) from easy_rec.python.layers import dnn from easy_rec.python.layers.common_layers import highway from easy_rec.python.loss.circle_loss import circle_loss from easy_rec.python.loss.multi_similarity import ms_loss from easy_rec.python.model.easy_rec_model import EasyRecModel -from easy_rec.python.protos.collaborative_metric_learning_pb2 import ( - CoMetricLearningI2I as MetricLearningI2IConfig, # NOQA -) from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.utils.activation import gelu from easy_rec.python.utils.proto_util import copy_obj +from easy_rec.python.protos.collaborative_metric_learning_pb2 import CoMetricLearningI2I as MetricLearningI2IConfig # NOQA + +from easy_rec.python.core.metrics import ( # NOQA + metric_learning_average_precision_at_k, metric_learning_recall_at_k, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 class CoMetricLearningI2I(EasyRecModel): + def __init__( - self, - model_config, # pipeline.model_config - feature_configs, # pipeline.feature_configs - features, # same as model_fn input - labels=None, - is_training=False, + self, + model_config, # pipeline.model_config + feature_configs, # pipeline.feature_configs + features, # same as model_fn input + labels=None, + is_training=False, ): - super(CoMetricLearningI2I, self).__init__(model_config, feature_configs, features, labels, is_training) + super(CoMetricLearningI2I, self).__init__(model_config, feature_configs, + features, labels, is_training) model = self._model_config.WhichOneof('model') assert model == 'metric_learning', 'invalid model config: %s' % model @@ -47,19 +48,22 @@ def __init__( elif self._loss_type == LossType.MULTI_SIMILARITY_LOSS: self.loss = self._model_config.multi_similarity_loss else: - raise ValueError('unsupported loss type: %s' % LossType.Name(self._loss_type)) + raise ValueError('unsupported loss type: %s' % + LossType.Name(self._loss_type)) if not self.has_backbone: self._highway_features = {} self._highway_num = len(self._model_config.highway) for _id in range(self._highway_num): highway_cfg = self._model_config.highway[_id] - highway_feature, _ = self._input_layer(self._feature_dict, highway_cfg.input) + highway_feature, _ = self._input_layer(self._feature_dict, + highway_cfg.input) self._highway_features[highway_cfg.input] = highway_feature self.input_features = [] if self._model_config.HasField('input'): - input_feature, _ = self._input_layer(self._feature_dict, self._model_config.input) + input_feature, _ = self._input_layer(self._feature_dict, + self._model_config.input) self.input_features.append(input_feature) self.dnn = copy_obj(self._model_config.dnn) @@ -85,16 +89,16 @@ def build_predict_graph(self): for _id in range(self._highway_num): highway_cfg = self._model_config.highway[_id] highway_fea = tf.layers.batch_normalization( - self._highway_features[highway_cfg.input], - training=self._is_training, - trainable=True, - name='highway_%s_bn' % highway_cfg.input, + self._highway_features[highway_cfg.input], + training=self._is_training, + trainable=True, + name='highway_%s_bn' % highway_cfg.input, ) highway_fea = highway( - highway_fea, - highway_cfg.emb_size, - activation=gelu, - scope='highway_%s' % _id, + highway_fea, + highway_cfg.emb_size, + activation=gelu, + scope='highway_%s' % _id, ) print('highway_fea: ', highway_fea) self.input_features.append(highway_fea) @@ -106,21 +110,24 @@ def build_predict_graph(self): dnn_net = dnn.DNN(self.dnn, self._l2_reg, 'dnn', self._is_training) net_output = dnn_net(feature) tower_emb = tf.layers.dense( - inputs=net_output, - units=last_hidden, - kernel_regularizer=self._l2_reg, - name='dnn/dnn_%d' % (num_dnn_layer - 1), + inputs=net_output, + units=last_hidden, + kernel_regularizer=self._l2_reg, + name='dnn/dnn_%d' % (num_dnn_layer - 1), ) if self._model_config.output_l2_normalized_emb: norm_emb = tf.nn.l2_normalize(tower_emb, axis=-1) self._prediction_dict['norm_emb'] = norm_emb - self._prediction_dict['norm_embedding'] = tf.reduce_join(tf.as_string(norm_emb), axis=-1, separator=',') + self._prediction_dict['norm_embedding'] = tf.reduce_join( + tf.as_string(norm_emb), axis=-1, separator=',') self._prediction_dict['float_emb'] = tower_emb - self._prediction_dict['embedding'] = tf.reduce_join(tf.as_string(tower_emb), axis=-1, separator=',') + self._prediction_dict['embedding'] = tf.reduce_join( + tf.as_string(tower_emb), axis=-1, separator=',') if self.sample_id is not None and self.sample_id in self._feature_dict: - self._prediction_dict['sample_id'] = tf.identity(self._feature_dict[self.sample_id]) + self._prediction_dict['sample_id'] = tf.identity( + self._feature_dict[self.sample_id]) return self._prediction_dict def build_loss_graph(self): @@ -129,23 +136,23 @@ def build_loss_graph(self): norm_emb = self._prediction_dict['norm_emb'] if emb_normed else emb if self._loss_type == LossType.CIRCLE_LOSS: self._loss_dict['circle_loss'] = circle_loss( - norm_emb, - self.labels, - self.session_ids, - self.loss.margin, - self.loss.gamma, - embed_normed=emb_normed, + norm_emb, + self.labels, + self.session_ids, + self.loss.margin, + self.loss.gamma, + embed_normed=emb_normed, ) elif self._loss_type == LossType.MULTI_SIMILARITY_LOSS: self._loss_dict['ms_loss'] = ms_loss( - norm_emb, - self.labels, - self.session_ids, - self.loss.alpha, - self.loss.beta, - self.loss.lamb, - self.loss.eps, - embed_normed=emb_normed, + norm_emb, + self.labels, + self.session_ids, + self.loss.alpha, + self.loss.beta, + self.loss.lamb, + self.loss.eps, + embed_normed=emb_normed, ) else: raise ValueError('invalid loss type: %s' % LossType.Name(self._loss_type)) @@ -173,7 +180,11 @@ def build_metric_graph(self, eval_config): emb = self._prediction_dict['float_emb'] if len(recall_at_k) > 0: - metric_dict.update(metric_learning_recall_at_k(recall_at_k, emb, self.labels, self.session_ids)) + metric_dict.update( + metric_learning_recall_at_k(recall_at_k, emb, self.labels, + self.session_ids)) if len(precision_at_k) > 0: - metric_dict.update(metric_learning_average_precision_at_k(precision_at_k, emb, self.labels, self.session_ids)) + metric_dict.update( + metric_learning_average_precision_at_k(precision_at_k, emb, + self.labels, self.session_ids)) return metric_dict diff --git a/easy_rec/python/model/dat.py b/easy_rec/python/model/dat.py index 32adc7e44..a622beee4 100644 --- a/easy_rec/python/model/dat.py +++ b/easy_rec/python/model/dat.py @@ -15,13 +15,22 @@ class DAT(MatchModel): """Dual Augmented Two-tower Model.""" - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(DAT, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'dat', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model' - ) - - feature_group_names = [fg.group_name for fg in self._model_config.feature_groups] + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(DAT, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert self._model_config.WhichOneof( + 'model' + ) == 'dat', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') + + feature_group_names = [ + fg.group_name for fg in self._model_config.feature_groups + ] assert 'user' in feature_group_names, 'user feature group not found' assert 'item' in feature_group_names, 'item feature group not found' assert 'user_id_augment' in feature_group_names, 'user_id_augment feature group not found' @@ -32,11 +41,13 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai self.user_tower = copy_obj(self._model_config.user_tower) self.user_deep_feature, _ = self._input_layer(self._feature_dict, 'user') - self.user_augmented_vec, _ = self._input_layer(self._feature_dict, 'user_id_augment') + self.user_augmented_vec, _ = self._input_layer(self._feature_dict, + 'user_id_augment') self.item_tower = copy_obj(self._model_config.item_tower) self.item_deep_feature, _ = self._input_layer(self._feature_dict, 'item') - self.item_augmented_vec, _ = self._input_layer(self._feature_dict, 'item_id_augment') + self.item_augmented_vec, _ = self._input_layer(self._feature_dict, + 'item_id_augment') self._user_tower_emb = None self._item_tower_emb = None @@ -44,28 +55,32 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai def build_predict_graph(self): num_user_dnn_layer = len(self.user_tower.dnn.hidden_units) last_user_hidden = self.user_tower.dnn.hidden_units.pop() - user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training) + user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', + self._is_training) - user_tower_feature = tf.concat([self.user_deep_feature, self.user_augmented_vec], axis=-1) + user_tower_feature = tf.concat( + [self.user_deep_feature, self.user_augmented_vec], axis=-1) user_tower_emb = user_dnn(user_tower_feature) user_tower_emb = tf.layers.dense( - inputs=user_tower_emb, - units=last_user_hidden, - kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), + inputs=user_tower_emb, + units=last_user_hidden, + kernel_regularizer=self._l2_reg, + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), ) num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() - item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training) + item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', + self._is_training) - item_tower_feature = tf.concat([self.item_deep_feature, self.item_augmented_vec], axis=-1) + item_tower_feature = tf.concat( + [self.item_deep_feature, self.item_augmented_vec], axis=-1) item_tower_emb = item_dnn(item_tower_feature) item_tower_emb = tf.layers.dense( - inputs=item_tower_emb, - units=last_item_hidden, - kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), + inputs=item_tower_emb, + units=last_item_hidden, + kernel_regularizer=self._l2_reg, + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), ) user_tower_emb = self.norm(user_tower_emb) @@ -78,7 +93,8 @@ def build_predict_graph(self): raise ValueError('Currently DAT model only supports list wise mode.') if self._loss_type == LossType.CLASSIFICATION: - raise ValueError('Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.') + raise ValueError( + 'Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.') elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: y_pred = self._mask_in_batch(y_pred) self._prediction_dict['logits'] = y_pred @@ -88,8 +104,10 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb - self._prediction_dict['user_emb'] = tf.reduce_join(tf.as_string(user_tower_emb), axis=-1, separator=',') - self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_tower_emb), axis=-1, separator=',') + self._prediction_dict['user_emb'] = tf.reduce_join( + tf.as_string(user_tower_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join( + tf.as_string(item_tower_emb), axis=-1, separator=',') augmented_p_u = tf.stop_gradient(user_tower_emb) augmented_p_i = tf.stop_gradient(item_tower_emb) @@ -104,21 +122,24 @@ def build_predict_graph(self): def get_outputs(self): if self._loss_type == LossType.CLASSIFICATION: - raise ValueError('Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.') + raise ValueError( + 'Currently DAT model only supports SOFTMAX_CROSS_ENTROPY loss.') elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: - self._prediction_dict['logits'] = tf.squeeze(self._prediction_dict['logits'], axis=-1) - self._prediction_dict['probs'] = tf.nn.sigmoid(self._prediction_dict['logits']) + self._prediction_dict['logits'] = tf.squeeze( + self._prediction_dict['logits'], axis=-1) + self._prediction_dict['probs'] = tf.nn.sigmoid( + self._prediction_dict['logits']) return [ - 'logits', - 'probs', - 'user_emb', - 'item_emb', - 'user_tower_emb', - 'item_tower_emb', - 'augmented_p_u', - 'augmented_p_i', - 'augmented_a_u', - 'augmented_a_i', + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_tower_emb', + 'item_tower_emb', + 'augmented_p_u', + 'augmented_p_i', + 'augmented_a_u', + 'augmented_a_i', ] else: raise ValueError('invalid loss type: %s' % str(self._loss_type)) diff --git a/easy_rec/python/model/dbmtl.py b/easy_rec/python/model/dbmtl.py index b421bfdd3..8424d125e 100644 --- a/easy_rec/python/model/dbmtl.py +++ b/easy_rec/python/model/dbmtl.py @@ -2,7 +2,10 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import cmbf, dnn, mmoe, uniter +from easy_rec.python.layers import cmbf +from easy_rec.python.layers import dnn +from easy_rec.python.layers import mmoe +from easy_rec.python.layers import uniter from easy_rec.python.model.multi_task_model import MultiTaskModel from easy_rec.python.protos.dbmtl_pb2 import DBMTL as DBMTLConfig @@ -11,32 +14,39 @@ class DBMTL(MultiTaskModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(DBMTL, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(DBMTL, self).__init__(model_config, feature_configs, features, labels, + is_training) assert self._model_config.WhichOneof('model') == 'dbmtl', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.dbmtl assert isinstance(self._model_config, DBMTLConfig) if self._model_config.HasField('bottom_cmbf'): self._cmbf_layer = cmbf.CMBF( - model_config, - feature_configs, - features, - self._model_config.bottom_cmbf, - self._input_layer, + model_config, + feature_configs, + features, + self._model_config.bottom_cmbf, + self._input_layer, ) elif self._model_config.HasField('bottom_uniter'): self._uniter_layer = uniter.Uniter( - model_config, - feature_configs, - features, - self._model_config.bottom_uniter, - self._input_layer, + model_config, + feature_configs, + features, + self._model_config.bottom_uniter, + self._input_layer, ) elif not self.has_backbone: - self._features, self._feature_list = self._input_layer(self._feature_dict, 'all') + self._features, self._feature_list = self._input_layer( + self._feature_dict, 'all') else: assert False, 'invalid code branch' self._init_towers(self._model_config.task_towers) @@ -50,10 +60,10 @@ def build_predict_graph(self): bottom_fea = self._uniter_layer(self._is_training, l2_reg=self._l2_reg) elif self._model_config.HasField('bottom_dnn'): bottom_dnn = dnn.DNN( - self._model_config.bottom_dnn, - self._l2_reg, - name='bottom_dnn', - is_training=self._is_training, + self._model_config.bottom_dnn, + self._l2_reg, + name='bottom_dnn', + is_training=self._is_training, ) bottom_fea = bottom_dnn(self._features) else: @@ -62,10 +72,10 @@ def build_predict_graph(self): # MMOE block if self._model_config.HasField('expert_dnn'): mmoe_layer = mmoe.MMOE( - self._model_config.expert_dnn, - l2_reg=self._l2_reg, - num_task=self._task_num, - num_expert=self._model_config.num_expert, + self._model_config.expert_dnn, + l2_reg=self._l2_reg, + num_task=self._task_num, + num_expert=self._model_config.num_expert, ) task_input_list = mmoe_layer(bottom_fea) else: @@ -77,10 +87,10 @@ def build_predict_graph(self): tower_name = task_tower_cfg.tower_name if task_tower_cfg.HasField('dnn'): tower_dnn = dnn.DNN( - task_tower_cfg.dnn, - self._l2_reg, - name=tower_name + '/dnn', - is_training=self._is_training, + task_tower_cfg.dnn, + self._l2_reg, + name=tower_name + '/dnn', + is_training=self._is_training, ) tower_fea = tower_dnn(task_input_list[i]) tower_features[tower_name] = tower_fea @@ -93,23 +103,24 @@ def build_predict_graph(self): for task_tower_cfg in self._model_config.task_towers: tower_name = task_tower_cfg.tower_name relation_dnn = dnn.DNN( - task_tower_cfg.relation_dnn, - self._l2_reg, - name=tower_name + '/relation_dnn', - is_training=self._is_training, + task_tower_cfg.relation_dnn, + self._l2_reg, + name=tower_name + '/relation_dnn', + is_training=self._is_training, ) tower_inputs = [tower_features[tower_name]] for relation_tower_name in task_tower_cfg.relation_tower_names: tower_inputs.append(relation_features[relation_tower_name]) - relation_input = tf.concat(tower_inputs, axis=-1, name=tower_name + '/relation_input') + relation_input = tf.concat( + tower_inputs, axis=-1, name=tower_name + '/relation_input') relation_fea = relation_dnn(relation_input) relation_features[tower_name] = relation_fea output_logits = tf.layers.dense( - relation_fea, - task_tower_cfg.num_class, - kernel_regularizer=self._l2_reg, - name=tower_name + '/output', + relation_fea, + task_tower_cfg.num_class, + kernel_regularizer=self._l2_reg, + name=tower_name + '/output', ) tower_outputs[tower_name] = output_logits diff --git a/easy_rec/python/model/dcn.py b/easy_rec/python/model/dcn.py index 7e44545c9..aca03fe8e 100644 --- a/easy_rec/python/model/dcn.py +++ b/easy_rec/python/model/dcn.py @@ -5,6 +5,7 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.dcn_pb2 import DCN as DCNConfig # NOQA if tf.__version__ >= '2.0': @@ -12,11 +13,19 @@ class DCN(RankModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(DCN, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'dcn', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model' - ) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(DCN, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert self._model_config.WhichOneof( + 'model' + ) == 'dcn', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.dcn assert isinstance(self._model_config, DCNConfig) @@ -28,9 +37,9 @@ def _cross_net(self, tensor, num_cross_layers): for i in range(num_cross_layers): name = 'cross_layer_%s' % i w = tf.get_variable( - name=name + '_w', - dtype=tf.float32, - shape=(input_dim), + name=name + '_w', + dtype=tf.float32, + shape=(input_dim), ) b = tf.get_variable(name=name + '_b', dtype=tf.float32, shape=(input_dim)) xw = tf.reduce_sum(x * w, axis=1, keepdims=True) # (B, 1) @@ -42,7 +51,8 @@ def build_predict_graph(self): # deep tower deep_tower_config = self._model_config.deep_tower - dnn_layer = dnn.DNN(deep_tower_config.dnn, self._l2_reg, 'dnn', self._is_training) + dnn_layer = dnn.DNN(deep_tower_config.dnn, self._l2_reg, 'dnn', + self._is_training) deep_tensor = dnn_layer(self._features) tower_fea_arr.append(deep_tensor) # cross tower @@ -52,7 +62,8 @@ def build_predict_graph(self): tower_fea_arr.append(cross_tensor) # final tower all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, + 'final_dnn', self._is_training) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/deepfm.py b/easy_rec/python/model/deepfm.py index c3a41dae3..821174a0f 100644 --- a/easy_rec/python/model/deepfm.py +++ b/easy_rec/python/model/deepfm.py @@ -4,7 +4,8 @@ import tensorflow as tf -from easy_rec.python.layers import dnn, fm +from easy_rec.python.layers import dnn +from easy_rec.python.layers import fm from easy_rec.python.model.rank_model import RankModel from easy_rec.python.protos.deepfm_pb2 import DeepFM as DeepFMConfig @@ -13,20 +14,28 @@ class DeepFM(RankModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(DeepFM, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(DeepFM, self).__init__(model_config, feature_configs, features, + labels, is_training) assert self._model_config.WhichOneof('model') == 'deepfm', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.deepfm assert isinstance(self._model_config, DeepFMConfig) # backward compatibility if self._model_config.HasField('wide_regularization'): - tf.logging.warn('wide_regularization is deprecated, please use l2_regularization') + tf.logging.warn( + 'wide_regularization is deprecated, please use l2_regularization') self._wide_features, _ = self._input_layer(self._feature_dict, 'wide') - self._deep_features, self._fm_features = self._input_layer(self._feature_dict, 'deep') + self._deep_features, self._fm_features = self._input_layer( + self._feature_dict, 'deep') if 'fm' in self._input_layer._feature_groups: _, self._fm_features = self._input_layer(self._feature_dict, 'fm') @@ -37,53 +46,61 @@ def build_input_layer(self, model_config, feature_configs): assert model_config.deepfm.wide_output_dim == model_config.num_class self._wide_output_dim = model_config.deepfm.wide_output_dim if self._wide_output_dim != self._num_class: - logging.warning('wide_output_dim not equal to 1, it is not a standard model') + logging.warning( + 'wide_output_dim not equal to 1, it is not a standard model') super(DeepFM, self).build_input_layer(model_config, feature_configs) def build_predict_graph(self): # Wide if self._num_class > 1 and self._wide_output_dim == self._num_class: wide_shape = tf.shape(self._wide_features) - new_shape = tf.stack([-1, wide_shape[1] // self._num_class, self._num_class]) + new_shape = tf.stack( + [-1, wide_shape[1] // self._num_class, self._num_class]) wide_fea = tf.reshape(self._wide_features, new_shape) wide_fea = tf.reduce_sum(wide_fea, axis=1, name='wide_feature') else: - wide_fea = tf.reduce_sum(self._wide_features, axis=1, keepdims=True, name='wide_feature') + wide_fea = tf.reduce_sum( + self._wide_features, axis=1, keepdims=True, name='wide_feature') # FM fm_fea = fm.FM(name='fm_feature')(self._fm_features) self._fm_outputs = fm_fea # Deep - deep_layer = dnn.DNN(self._model_config.dnn, self._l2_reg, 'deep_feature', self._is_training) + deep_layer = dnn.DNN(self._model_config.dnn, self._l2_reg, 'deep_feature', + self._is_training) deep_fea = deep_layer(self._deep_features) # Final if len(self._model_config.final_dnn.hidden_units) > 0: all_fea = tf.concat([wide_fea, fm_fea, deep_fea], axis=1) final_dnn_layer = dnn.DNN( - self._model_config.final_dnn, - self._l2_reg, - 'final_dnn', - self._is_training, + self._model_config.final_dnn, + self._l2_reg, + 'final_dnn', + self._is_training, ) all_fea = final_dnn_layer(all_fea) - output = tf.layers.dense(all_fea, self._num_class, kernel_regularizer=self._l2_reg, name='output') + output = tf.layers.dense( + all_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='output') else: if self._num_class > 1: fm_fea = tf.layers.dense( - fm_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='fm_logits', + fm_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='fm_logits', ) else: fm_fea = tf.reduce_sum(fm_fea, 1, keepdims=True) deep_fea = tf.layers.dense( - deep_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='deep_logits', + deep_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='deep_logits', ) output = wide_fea + fm_fea + deep_fea @@ -93,11 +110,15 @@ def build_predict_graph(self): def build_feature_output_dict(self): outputs = super(DeepFM, self).build_feature_output_dict() - outputs.update( - { - 'wide_features': tf.reduce_join(tf.as_string(self._wide_features), axis=-1, separator=','), - 'deep_features': tf.reduce_join(tf.as_string(self._deep_features), axis=-1, separator=','), - 'fm_outputs': tf.reduce_join(tf.as_string(self._fm_outputs), axis=-1, separator=','), - } - ) + outputs.update({ + 'wide_features': + tf.reduce_join( + tf.as_string(self._wide_features), axis=-1, separator=','), + 'deep_features': + tf.reduce_join( + tf.as_string(self._deep_features), axis=-1, separator=','), + 'fm_outputs': + tf.reduce_join( + tf.as_string(self._fm_outputs), axis=-1, separator=','), + }) return outputs diff --git a/easy_rec/python/model/dlrm.py b/easy_rec/python/model/dlrm.py index 7d2a8df95..a0d0f77ca 100755 --- a/easy_rec/python/model/dlrm.py +++ b/easy_rec/python/model/dlrm.py @@ -6,6 +6,7 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.dlrm_pb2 import DLRM as DLRMConfig # NOQA if tf.__version__ >= '2.0': @@ -15,30 +16,39 @@ class DLRM(RankModel): """Implements Deep Learning Recommendation Model for Personalization and Recommendation Systems(FaceBook).""" - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(DLRM, self).__init__(model_config, feature_configs, features, labels, is_training) - assert model_config.WhichOneof('model') == 'dlrm', 'invalid model config: %s' % model_config.WhichOneof('model') + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(DLRM, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert model_config.WhichOneof( + 'model' + ) == 'dlrm', 'invalid model config: %s' % model_config.WhichOneof('model') self._model_config = model_config.dlrm assert isinstance(self._model_config, DLRMConfig) - assert self._input_layer.has_group('sparse'), 'sparse group is not specified' + assert self._input_layer.has_group( + 'sparse'), 'sparse group is not specified' _, self._sparse_features = self._input_layer(self._feature_dict, 'sparse') assert self._input_layer.has_group('dense'), 'dense group is not specified' self._dense_feature, _ = self._input_layer(self._feature_dict, 'dense') def build_predict_graph(self): - bot_dnn = dnn.DNN(self._model_config.bot_dnn, self._l2_reg, 'bot_dnn', self._is_training) + bot_dnn = dnn.DNN(self._model_config.bot_dnn, self._l2_reg, 'bot_dnn', + self._is_training) dense_fea = bot_dnn(self._dense_feature) - logging.info('arch_interaction_op = %s' % self._model_config.arch_interaction_op) + logging.info('arch_interaction_op = %s' % + self._model_config.arch_interaction_op) if self._model_config.arch_interaction_op == 'cat': all_fea = tf.concat([dense_fea] + self._sparse_features, axis=1) elif self._model_config.arch_interaction_op == 'dot': - assert dense_fea.get_shape()[1] == self._sparse_features[0].get_shape()[1], ( - 'bot_dnn last hidden[%d] != sparse feature embedding_dim[%d]' - % ( + assert dense_fea.get_shape()[1] == self._sparse_features[0].get_shape( + )[1], ('bot_dnn last hidden[%d] != sparse feature embedding_dim[%d]' % ( dense_fea.get_shape()[1], self._sparse_features[0].get_shape()[1], - ) - ) + )) all_feas = [dense_fea] + self._sparse_features all_feas = [x[:, None, :] for x in all_feas] @@ -48,16 +58,18 @@ def build_predict_graph(self): offset = 0 if self._model_config.arch_interaction_itself else 1 upper_tri = [] for i in range(num_fea): - upper_tri.append(interaction[:, i, (i + offset) : num_fea]) + upper_tri.append(interaction[:, i, (i + offset):num_fea]) upper_tri = tf.concat(upper_tri, axis=1) concat_feas = [upper_tri] + self._sparse_features if self._model_config.arch_with_dense_feature: concat_feas.append(dense_fea) all_fea = tf.concat(concat_feas, axis=1) - top_dnn = dnn.DNN(self._model_config.top_dnn, self._l2_reg, 'top_dnn', self._is_training) + top_dnn = dnn.DNN(self._model_config.top_dnn, self._l2_reg, 'top_dnn', + self._is_training) all_fea = top_dnn(all_fea) - logits = tf.layers.dense(all_fea, 1, kernel_regularizer=self._l2_reg, name='output') + logits = tf.layers.dense( + all_fea, 1, kernel_regularizer=self._l2_reg, name='output') self._add_to_prediction_dict(logits) diff --git a/easy_rec/python/model/dropoutnet.py b/easy_rec/python/model/dropoutnet.py index d77d4d709..e3e420c76 100644 --- a/easy_rec/python/model/dropoutnet.py +++ b/easy_rec/python/model/dropoutnet.py @@ -4,21 +4,22 @@ from easy_rec.python.layers import dnn from easy_rec.python.loss.pairwise_loss import pairwise_loss -from easy_rec.python.loss.softmax_loss_with_negative_mining import ( # NOQA - softmax_loss_with_negative_mining, -) from easy_rec.python.model.easy_rec_model import EasyRecModel -from easy_rec.python.protos.dropoutnet_pb2 import DropoutNet as DropoutNetConfig # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.utils.proto_util import copy_obj +from easy_rec.python.loss.softmax_loss_with_negative_mining import ( # NOQA + softmax_loss_with_negative_mining,) +from easy_rec.python.protos.dropoutnet_pb2 import DropoutNet as DropoutNetConfig # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses def cosine_similarity(user_emb, item_emb): - user_item_sim = tf.reduce_sum(tf.multiply(user_emb, item_emb), axis=1, name='cosine') + user_item_sim = tf.reduce_sum( + tf.multiply(user_emb, item_emb), axis=1, name='cosine') return user_item_sim @@ -32,12 +33,18 @@ def bernoulli_dropout(x, rate, training=False): class DropoutNet(EasyRecModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(DropoutNet, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(DropoutNet, self).__init__(model_config, feature_configs, features, + labels, is_training) self._losses = self._model_config.losses assert self._model_config.WhichOneof('model') == 'dropoutnet', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.dropoutnet assert isinstance(self._model_config, DropoutNetConfig) @@ -47,9 +54,11 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai self.user_tower_layers = copy_obj(self._model_config.user_tower) self.user_content_feature, self.user_preference_feature = None, None if self._input_layer.has_group('user_content'): - self.user_content_feature, _ = self._input_layer(self._feature_dict, 'user_content') + self.user_content_feature, _ = self._input_layer(self._feature_dict, + 'user_content') if self._input_layer.has_group('user_preference'): - self.user_preference_feature, _ = self._input_layer(self._feature_dict, 'user_preference') + self.user_preference_feature, _ = self._input_layer( + self._feature_dict, 'user_preference') assert self.user_content_feature is not None or self.user_preference_feature is not None, 'no user feature' # copy_obj so that any modification will not affect original config @@ -58,9 +67,11 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai self.item_tower_layers = copy_obj(self._model_config.item_tower) self.item_content_feature, self.item_preference_feature = None, None if self._input_layer.has_group('item_content'): - self.item_content_feature, _ = self._input_layer(self._feature_dict, 'item_content') + self.item_content_feature, _ = self._input_layer(self._feature_dict, + 'item_content') if self._input_layer.has_group('item_preference'): - self.item_preference_feature, _ = self._input_layer(self._feature_dict, 'item_preference') + self.item_preference_feature, _ = self._input_layer( + self._feature_dict, 'item_preference') assert self.item_content_feature is not None or self.item_preference_feature is not None, 'no item feature' def build_predict_graph(self): @@ -75,37 +86,38 @@ def build_predict_graph(self): user_features = [] if self.user_content_feature is not None: user_content_dnn = dnn.DNN( - self.user_content_layers, - self._l2_reg, - 'user_content', - self._is_training, + self.user_content_layers, + self._l2_reg, + 'user_content', + self._is_training, ) content_feature = user_content_dnn(self.user_content_feature) user_features.append(content_feature) if self.user_preference_feature is not None: user_prefer_feature = bernoulli_dropout( - self.user_preference_feature, - self._model_config.user_dropout_rate, - self._is_training, + self.user_preference_feature, + self._model_config.user_dropout_rate, + self._is_training, ) user_prefer_dnn = dnn.DNN( - self.user_preference_layers, - self._l2_reg, - 'user_preference', - self._is_training, + self.user_preference_layers, + self._l2_reg, + 'user_preference', + self._is_training, ) prefer_feature = user_prefer_dnn(user_prefer_feature) user_features.append(prefer_feature) user_tower_feature = tf.concat(user_features, axis=-1) - user_dnn = dnn.DNN(self.user_tower_layers, self._l2_reg, 'user_dnn', self._is_training) + user_dnn = dnn.DNN(self.user_tower_layers, self._l2_reg, 'user_dnn', + self._is_training) user_hidden = user_dnn(user_tower_feature) user_tower_emb = tf.layers.dense( - inputs=user_hidden, - units=last_user_hidden, - kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), + inputs=user_hidden, + units=last_user_hidden, + kernel_regularizer=self._l2_reg, + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), ) # --------------------------build item tower----------------------------------- @@ -113,37 +125,38 @@ def build_predict_graph(self): item_features = [] if self.item_content_feature is not None: item_content_dnn = dnn.DNN( - self.item_content_layers, - self._l2_reg, - 'item_content', - self._is_training, + self.item_content_layers, + self._l2_reg, + 'item_content', + self._is_training, ) content_feature = item_content_dnn(self.item_content_feature) item_features.append(content_feature) if self.item_preference_feature is not None: item_prefer_feature = bernoulli_dropout( - self.item_preference_feature, - self._model_config.item_dropout_rate, - self._is_training, + self.item_preference_feature, + self._model_config.item_dropout_rate, + self._is_training, ) item_prefer_dnn = dnn.DNN( - self.item_preference_layers, - self._l2_reg, - 'item_preference', - self._is_training, + self.item_preference_layers, + self._l2_reg, + 'item_preference', + self._is_training, ) prefer_feature = item_prefer_dnn(item_prefer_feature) item_features.append(prefer_feature) item_tower_feature = tf.concat(item_features, axis=-1) - item_dnn = dnn.DNN(self.item_tower_layers, self._l2_reg, 'item_dnn', self._is_training) + item_dnn = dnn.DNN(self.item_tower_layers, self._l2_reg, 'item_dnn', + self._is_training) item_hidden = item_dnn(item_tower_feature) item_tower_emb = tf.layers.dense( - inputs=item_hidden, - units=last_item_hidden, - kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), + inputs=item_hidden, + units=last_item_hidden, + kernel_regularizer=self._l2_reg, + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), ) user_emb = tf.nn.l2_normalize(user_tower_emb, axis=-1) @@ -152,8 +165,10 @@ def build_predict_graph(self): self._prediction_dict['similarity'] = cosine self._prediction_dict['float_user_emb'] = user_emb self._prediction_dict['float_item_emb'] = item_emb - self._prediction_dict['user_emb'] = tf.reduce_join(tf.as_string(user_emb), axis=-1, separator=',') - self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_emb), axis=-1, separator=',') + self._prediction_dict['user_emb'] = tf.reduce_join( + tf.as_string(user_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join( + tf.as_string(item_emb), axis=-1, separator=',') return self._prediction_dict def build_loss_graph(self): @@ -161,26 +176,28 @@ def build_loss_graph(self): logits = self._prediction_dict['similarity'] for loss in self._losses: if loss.loss_type == LossType.SOFTMAX_CROSS_ENTROPY_WITH_NEGATIVE_MINING: - assert self._model_config.HasField('softmax_loss'), '`softmax_loss` must be configured' + assert self._model_config.HasField( + 'softmax_loss'), '`softmax_loss` must be configured' user_emb = self._prediction_dict['float_user_emb'] item_emb = self._prediction_dict['float_item_emb'] loss_value = softmax_loss_with_negative_mining( - user_emb, - item_emb, - labels, - self._model_config.softmax_loss.num_negative_samples, - embed_normed=True, - weights=self._sample_weight, - margin=self._model_config.softmax_loss.margin, - gamma=self._model_config.softmax_loss.gamma, - t=self._model_config.softmax_loss.coefficient_of_support_vector, + user_emb, + item_emb, + labels, + self._model_config.softmax_loss.num_negative_samples, + embed_normed=True, + weights=self._sample_weight, + margin=self._model_config.softmax_loss.margin, + gamma=self._model_config.softmax_loss.gamma, + t=self._model_config.softmax_loss.coefficient_of_support_vector, ) self._loss_dict['softmax_loss'] = loss_value * loss.weight elif loss.loss_type == LossType.PAIR_WISE_LOSS: loss_value = pairwise_loss(labels, logits) self._loss_dict['pairwise_loss'] = loss_value * loss.weight elif loss.loss_type == LossType.CLASSIFICATION: - loss_value = tf.losses.sigmoid_cross_entropy(labels, logits, self._sample_weight) + loss_value = tf.losses.sigmoid_cross_entropy(labels, logits, + self._sample_weight) self._loss_dict['sigmoid_loss'] = loss_value * loss.weight return self._loss_dict @@ -194,13 +211,17 @@ def build_metric_graph(self, eval_config): predict = tf.greater(prob, 0.5) for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'auc': - metric_dict['auc'] = metrics.auc(labels, prob, weights=self._sample_weight) + metric_dict['auc'] = metrics.auc( + labels, prob, weights=self._sample_weight) elif metric.WhichOneof('metric') == 'accuracy': - metric_dict['accuracy'] = metrics.accuracy(tf.cast(labels, tf.bool), predict, weights=self._sample_weight) + metric_dict['accuracy'] = metrics.accuracy( + tf.cast(labels, tf.bool), predict, weights=self._sample_weight) elif metric.WhichOneof('metric') == 'precision': - metric_dict['precision'] = metrics.precision(labels, predict, weights=self._sample_weight) + metric_dict['precision'] = metrics.precision( + labels, predict, weights=self._sample_weight) elif metric.WhichOneof('metric') == 'recall': - metric_dict['recall'] = metrics.recall(labels, predict, weights=self._sample_weight) + metric_dict['recall'] = metrics.recall( + labels, predict, weights=self._sample_weight) else: ValueError('invalid metric type: %s' % str(metric)) return metric_dict diff --git a/easy_rec/python/model/dssm.py b/easy_rec/python/model/dssm.py index dba07d170..740e02257 100644 --- a/easy_rec/python/model/dssm.py +++ b/easy_rec/python/model/dssm.py @@ -15,11 +15,19 @@ class DSSM(MatchModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(DSSM, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'dssm', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model' - ) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(DSSM, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert self._model_config.WhichOneof( + 'model' + ) == 'dssm', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.dssm assert isinstance(self._model_config, DSSMConfig) @@ -35,24 +43,26 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai def build_predict_graph(self): num_user_dnn_layer = len(self.user_tower.dnn.hidden_units) last_user_hidden = self.user_tower.dnn.hidden_units.pop() - user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training) + user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', + self._is_training) user_tower_emb = user_dnn(self.user_tower_feature) user_tower_emb = tf.layers.dense( - inputs=user_tower_emb, - units=last_user_hidden, - kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), + inputs=user_tower_emb, + units=last_user_hidden, + kernel_regularizer=self._l2_reg, + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), ) num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() - item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training) + item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', + self._is_training) item_tower_emb = item_dnn(self.item_tower_feature) item_tower_emb = tf.layers.dense( - inputs=item_tower_emb, - units=last_item_hidden, - kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), + inputs=item_tower_emb, + units=last_item_hidden, + kernel_regularizer=self._l2_reg, + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), ) if self._model_config.simi_func == Similarity.COSINE: @@ -64,8 +74,16 @@ def build_predict_graph(self): user_item_sim = self.sim(user_tower_emb, item_tower_emb) / temperature if self._model_config.scale_simi: - sim_w = tf.get_variable('sim_w', dtype=tf.float32, shape=(1), initializer=tf.ones_initializer()) - sim_b = tf.get_variable('sim_b', dtype=tf.float32, shape=(1), initializer=tf.zeros_initializer()) + sim_w = tf.get_variable( + 'sim_w', + dtype=tf.float32, + shape=(1), + initializer=tf.ones_initializer()) + sim_b = tf.get_variable( + 'sim_b', + dtype=tf.float32, + shape=(1), + initializer=tf.zeros_initializer()) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -85,30 +103,34 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb - self._prediction_dict['user_emb'] = tf.reduce_join(tf.as_string(user_tower_emb), axis=-1, separator=',') - self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_tower_emb), axis=-1, separator=',') + self._prediction_dict['user_emb'] = tf.reduce_join( + tf.as_string(user_tower_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join( + tf.as_string(item_tower_emb), axis=-1, separator=',') return self._prediction_dict def get_outputs(self): if self._loss_type == LossType.CLASSIFICATION: return [ - 'logits', - 'probs', - 'user_emb', - 'item_emb', - 'user_tower_emb', - 'item_tower_emb', + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_tower_emb', + 'item_tower_emb', ] elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: - self._prediction_dict['logits'] = tf.squeeze(self._prediction_dict['logits'], axis=-1) - self._prediction_dict['probs'] = tf.nn.sigmoid(self._prediction_dict['logits']) + self._prediction_dict['logits'] = tf.squeeze( + self._prediction_dict['logits'], axis=-1) + self._prediction_dict['probs'] = tf.nn.sigmoid( + self._prediction_dict['logits']) return [ - 'logits', - 'probs', - 'user_emb', - 'item_emb', - 'user_tower_emb', - 'item_tower_emb', + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_tower_emb', + 'item_tower_emb', ] elif self._loss_type == LossType.L2_LOSS: return ['y', 'user_emb', 'item_emb', 'user_tower_emb', 'item_tower_emb'] @@ -117,24 +139,28 @@ def get_outputs(self): def build_output_dict(self): output_dict = super(DSSM, self).build_output_dict() - output_dict['user_tower_feature'] = tf.reduce_join(tf.as_string(self.user_tower_feature), axis=-1, separator=',') - output_dict['item_tower_feature'] = tf.reduce_join(tf.as_string(self.item_tower_feature), axis=-1, separator=',') + output_dict['user_tower_feature'] = tf.reduce_join( + tf.as_string(self.user_tower_feature), axis=-1, separator=',') + output_dict['item_tower_feature'] = tf.reduce_join( + tf.as_string(self.item_tower_feature), axis=-1, separator=',') return output_dict def build_rtp_output_dict(self): output_dict = super(DSSM, self).build_rtp_output_dict() if 'user_tower_emb' not in self._prediction_dict: - raise ValueError('User tower embedding does not exist. Please checking predict graph.') + raise ValueError( + 'User tower embedding does not exist. Please checking predict graph.') output_dict['user_embedding_output'] = tf.identity( - self._prediction_dict['user_tower_emb'], name='user_embedding_output' - ) + self._prediction_dict['user_tower_emb'], name='user_embedding_output') if 'item_tower_emb' not in self._prediction_dict: - raise ValueError('Item tower embedding does not exist. Please checking predict graph.') + raise ValueError( + 'Item tower embedding does not exist. Please checking predict graph.') output_dict['item_embedding_output'] = tf.identity( - self._prediction_dict['item_tower_emb'], name='item_embedding_output' - ) + self._prediction_dict['item_tower_emb'], name='item_embedding_output') if self._loss_type == LossType.CLASSIFICATION: if 'probs' not in self._prediction_dict: - raise ValueError('Probs output does not exist. Please checking predict graph.') - output_dict['rank_predict'] = tf.identity(self._prediction_dict['probs'], name='rank_predict') + raise ValueError( + 'Probs output does not exist. Please checking predict graph.') + output_dict['rank_predict'] = tf.identity( + self._prediction_dict['probs'], name='rank_predict') return output_dict diff --git a/easy_rec/python/model/dssm_senet.py b/easy_rec/python/model/dssm_senet.py index 3737c897e..4676e72eb 100644 --- a/easy_rec/python/model/dssm_senet.py +++ b/easy_rec/python/model/dssm_senet.py @@ -2,26 +2,34 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import dnn, senet +from easy_rec.python.layers import dnn +from easy_rec.python.layers import senet from easy_rec.python.model.dssm import DSSM from easy_rec.python.model.match_model import MatchModel -from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.protos.simi_pb2 import Similarity from easy_rec.python.utils.proto_util import copy_obj +from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses class DSSM_SENet(DSSM): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - MatchModel.__init__(self, model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + MatchModel.__init__(self, model_config, feature_configs, features, labels, + is_training) assert self._model_config.WhichOneof('model') == 'dssm_senet', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.dssm_senet assert isinstance(self._model_config, DSSM_SENet_Config) @@ -29,20 +37,22 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai self.user_tower = copy_obj(self._model_config.user_tower) ( - self.user_seq_features, - self.user_plain_features, - self.user_feature_list, - ) = self._input_layer(self._feature_dict, 'user', is_combine=False) + self.user_seq_features, + self.user_plain_features, + self.user_feature_list, + ) = self._input_layer( + self._feature_dict, 'user', is_combine=False) self.user_num_fields = len(self.user_feature_list) # copy_obj so that any modification will not affect original config self.item_tower = copy_obj(self._model_config.item_tower) ( - self.item_seq_features, - self.item_plain_features, - self.item_feature_list, - ) = self._input_layer(self._feature_dict, 'item', is_combine=False) + self.item_seq_features, + self.item_plain_features, + self.item_feature_list, + ) = self._input_layer( + self._feature_dict, 'item', is_combine=False) self.item_num_fields = len(self.item_feature_list) self._user_tower_emb = None @@ -50,32 +60,33 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai def build_predict_graph(self): user_senet = senet.SENet( - num_fields=self.user_num_fields, - num_squeeze_group=self.user_tower.senet.num_squeeze_group, - reduction_ratio=self.user_tower.senet.reduction_ratio, - l2_reg=self._l2_reg, - name='user_senet', + num_fields=self.user_num_fields, + num_squeeze_group=self.user_tower.senet.num_squeeze_group, + reduction_ratio=self.user_tower.senet.reduction_ratio, + l2_reg=self._l2_reg, + name='user_senet', ) user_senet_output_list = user_senet(self.user_feature_list) user_senet_output = tf.concat(user_senet_output_list, axis=-1) num_user_dnn_layer = len(self.user_tower.dnn.hidden_units) last_user_hidden = self.user_tower.dnn.hidden_units.pop() - user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', self._is_training) + user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', + self._is_training) user_tower_emb = user_dnn(user_senet_output) user_tower_emb = tf.layers.dense( - inputs=user_tower_emb, - units=last_user_hidden, - kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), + inputs=user_tower_emb, + units=last_user_hidden, + kernel_regularizer=self._l2_reg, + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), ) item_senet = senet.SENet( - num_fields=self.item_num_fields, - num_squeeze_group=self.item_tower.senet.num_squeeze_group, - reduction_ratio=self.item_tower.senet.reduction_ratio, - l2_reg=self._l2_reg, - name='item_senet', + num_fields=self.item_num_fields, + num_squeeze_group=self.item_tower.senet.num_squeeze_group, + reduction_ratio=self.item_tower.senet.reduction_ratio, + l2_reg=self._l2_reg, + name='item_senet', ) item_senet_output_list = item_senet(self.item_feature_list) @@ -83,13 +94,14 @@ def build_predict_graph(self): num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() - item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', self._is_training) + item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', + self._is_training) item_tower_emb = item_dnn(item_senet_output) item_tower_emb = tf.layers.dense( - inputs=item_tower_emb, - units=last_item_hidden, - kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), + inputs=item_tower_emb, + units=last_item_hidden, + kernel_regularizer=self._l2_reg, + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), ) if self._model_config.simi_func == Similarity.COSINE: @@ -101,8 +113,16 @@ def build_predict_graph(self): user_item_sim = self.sim(user_tower_emb, item_tower_emb) / temperature if self._model_config.scale_simi: - sim_w = tf.get_variable('sim_w', dtype=tf.float32, shape=(1), initializer=tf.ones_initializer()) - sim_b = tf.get_variable('sim_b', dtype=tf.float32, shape=(1), initializer=tf.zeros_initializer()) + sim_w = tf.get_variable( + 'sim_w', + dtype=tf.float32, + shape=(1), + initializer=tf.ones_initializer()) + sim_b = tf.get_variable( + 'sim_b', + dtype=tf.float32, + shape=(1), + initializer=tf.zeros_initializer()) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -122,8 +142,10 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb - self._prediction_dict['user_emb'] = tf.reduce_join(tf.as_string(user_tower_emb), axis=-1, separator=',') - self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_tower_emb), axis=-1, separator=',') + self._prediction_dict['user_emb'] = tf.reduce_join( + tf.as_string(user_tower_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join( + tf.as_string(item_tower_emb), axis=-1, separator=',') return self._prediction_dict def build_output_dict(self): diff --git a/easy_rec/python/model/dummy_model.py b/easy_rec/python/model/dummy_model.py index 9e34c82a3..d32f9d811 100644 --- a/easy_rec/python/model/dummy_model.py +++ b/easy_rec/python/model/dummy_model.py @@ -7,8 +7,15 @@ class DummyModel(EasyRecModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(DummyModel, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(DummyModel, self).__init__(model_config, feature_configs, features, + labels, is_training) if self._labels is not None: self._labels = list(self._labels.values()) @@ -28,7 +35,11 @@ def build_predict_graph(self): return self._prediction_dict def build_loss_graph(self): - return {'cross_ent': tf.reduce_sum(tf.square(self._prediction_dict['output'] - self._labels[0]))} + return { + 'cross_ent': + tf.reduce_sum( + tf.square(self._prediction_dict['output'] - self._labels[0])) + } def get_outputs(self): return ['output'] diff --git a/easy_rec/python/model/easy_rec_estimator.py b/easy_rec/python/model/easy_rec_estimator.py index 5305f08c1..074f43623 100644 --- a/easy_rec/python/model/easy_rec_estimator.py +++ b/easy_rec/python/model/easy_rec_estimator.py @@ -16,34 +16,28 @@ from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.saved_model import signature_constants -from tensorflow.python.training import basic_session_run_hooks, saver +from tensorflow.python.training import basic_session_run_hooks +from tensorflow.python.training import saver from easy_rec.python.builders import optimizer_builder -from easy_rec.python.compat import optimizers, sync_replicas_optimizer -from easy_rec.python.compat.early_stopping import ( # NOQA - custom_early_stop_hook, - deadline_stop_hook, - find_early_stop_var, - oss_stop_hook, - stop_if_no_decrease_hook, - stop_if_no_increase_hook, -) -from easy_rec.python.compat.embedding_parallel_saver import ( # NOQA - EmbeddingParallelSaver, -) +from easy_rec.python.compat import optimizers +from easy_rec.python.compat import sync_replicas_optimizer from easy_rec.python.compat.ops import GraphKeys from easy_rec.python.input.input import Input from easy_rec.python.layers.utils import _tensor_to_tensorinfo from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig from easy_rec.python.protos.train_pb2 import DistributionStrategy +from easy_rec.python.utils.multi_optimizer import MultiOptimizer + +from easy_rec.python.compat.early_stopping import ( # NOQA + custom_early_stop_hook, deadline_stop_hook, find_early_stop_var, + oss_stop_hook, stop_if_no_decrease_hook, stop_if_no_increase_hook, +) +from easy_rec.python.compat.embedding_parallel_saver import ( # NOQA + EmbeddingParallelSaver,) from easy_rec.python.utils import ( # NOQA - constant, - embedding_utils, - estimator_utils, - hvd_utils, - pai_util, + constant, embedding_utils, estimator_utils, hvd_utils, pai_util, ) -from easy_rec.python.utils.multi_optimizer import MultiOptimizer try: import horovod.tensorflow as hvd @@ -64,45 +58,59 @@ class EasyRecEstimator(tf.estimator.Estimator): + def __init__(self, pipeline_config, model_cls, run_config, params): self._pipeline_config = pipeline_config self._model_cls = model_cls assert isinstance(self._pipeline_config, EasyRecConfig) super(EasyRecEstimator, self).__init__( - model_fn=self._model_fn, - model_dir=pipeline_config.model_dir, - config=run_config, - params=params, + model_fn=self._model_fn, + model_dir=pipeline_config.model_dir, + config=run_config, + params=params, ) - def evaluate(self, input_fn, steps=None, hooks=None, checkpoint_path=None, name=None): + def evaluate(self, + input_fn, + steps=None, + hooks=None, + checkpoint_path=None, + name=None): # support for datahub/kafka offset restore input_fn.input_creator.restore(checkpoint_path) - return super(EasyRecEstimator, self).evaluate(input_fn, steps, hooks, checkpoint_path, name) - - def train(self, input_fn, hooks=None, steps=None, max_steps=None, saving_listeners=None): + return super(EasyRecEstimator, self).evaluate(input_fn, steps, hooks, + checkpoint_path, name) + + def train(self, + input_fn, + hooks=None, + steps=None, + max_steps=None, + saving_listeners=None): # support for datahub/kafka offset restore checkpoint_path = estimator_utils.latest_checkpoint(self.model_dir) if checkpoint_path is not None: input_fn.input_creator.restore(checkpoint_path) elif self.train_config.HasField('fine_tune_checkpoint'): fine_tune_ckpt = self.train_config.fine_tune_checkpoint - if fine_tune_ckpt.endswith('/') or gfile.IsDirectory(fine_tune_ckpt + '/'): + if fine_tune_ckpt.endswith('/') or gfile.IsDirectory(fine_tune_ckpt + + '/'): fine_tune_ckpt = estimator_utils.latest_checkpoint(fine_tune_ckpt) print( - 'fine_tune_checkpoint[%s] is directory, will use the latest checkpoint: %s' - % (self.train_config.fine_tune_checkpoint, fine_tune_ckpt) - ) + 'fine_tune_checkpoint[%s] is directory, will use the latest checkpoint: %s' + % (self.train_config.fine_tune_checkpoint, fine_tune_ckpt)) self.train_config.fine_tune_checkpoint = fine_tune_ckpt input_fn.input_creator.restore(fine_tune_ckpt) - return super(EasyRecEstimator, self).train(input_fn, hooks, steps, max_steps, saving_listeners) + return super(EasyRecEstimator, self).train(input_fn, hooks, steps, + max_steps, saving_listeners) @property def feature_configs(self): if len(self._pipeline_config.feature_configs) > 0: return self._pipeline_config.feature_configs - elif self._pipeline_config.feature_config and len(self._pipeline_config.feature_config.features) > 0: + elif self._pipeline_config.feature_config and len( + self._pipeline_config.feature_config.features) > 0: return self._pipeline_config.feature_config.features else: assert False, 'One of feature_configs and feature_config.features must be configured.' @@ -121,7 +129,8 @@ def train_config(self): @property def incr_save_config(self): - return self.train_config.incr_save_config if self.train_config.HasField('incr_save_config') else None + return self.train_config.incr_save_config if self.train_config.HasField( + 'incr_save_config') else None @property def export_config(self): @@ -130,8 +139,8 @@ def export_config(self): @property def embedding_parallel(self): return self.train_config.train_distribute in ( - DistributionStrategy.SokStrategy, - DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy, + DistributionStrategy.EmbeddingParallelStrategy, ) @property @@ -145,21 +154,30 @@ def saver_cls(self): def _train_model_fn(self, features, labels, run_config): tf.keras.backend.set_learning_phase(1) - model = self._model_cls(self.model_config, self.feature_configs, features, labels, is_training=True) + model = self._model_cls( + self.model_config, + self.feature_configs, + features, + labels, + is_training=True) predict_dict = model.build_predict_graph() loss_dict = model.build_loss_graph() - regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) + regularization_losses = tf.get_collection( + tf.GraphKeys.REGULARIZATION_LOSSES) if regularization_losses: regularization_losses = [ - reg_loss.get() if hasattr(reg_loss, 'get') else reg_loss for reg_loss in regularization_losses + reg_loss.get() if hasattr(reg_loss, 'get') else reg_loss + for reg_loss in regularization_losses ] - regularization_losses = tf.add_n(regularization_losses, name='regularization_loss') + regularization_losses = tf.add_n( + regularization_losses, name='regularization_loss') loss_dict['regularization_loss'] = regularization_losses variational_dropout_loss = tf.get_collection('variational_dropout_loss') if variational_dropout_loss: - variational_dropout_loss = tf.add_n(variational_dropout_loss, name='variational_dropout_loss') + variational_dropout_loss = tf.add_n( + variational_dropout_loss, name='variational_dropout_loss') loss_dict['variational_dropout_loss'] = variational_dropout_loss loss = tf.add_n(list(loss_dict.values())) @@ -170,13 +188,14 @@ def _train_model_fn(self, features, labels, run_config): if Input.DATA_OFFSET in features: task_index, task_num = estimator_utils.get_task_index_and_num() data_offset_var = tf.get_variable( - name=Input.DATA_OFFSET, - dtype=tf.string, - shape=[task_num], - collections=[tf.GraphKeys.GLOBAL_VARIABLES, Input.DATA_OFFSET], - trainable=False, + name=Input.DATA_OFFSET, + dtype=tf.string, + shape=[task_num], + collections=[tf.GraphKeys.GLOBAL_VARIABLES, Input.DATA_OFFSET], + trainable=False, ) - update_offset = tf.assign(data_offset_var[task_index], features[Input.DATA_OFFSET]) + update_offset = tf.assign(data_offset_var[task_index], + features[Input.DATA_OFFSET]) ops.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_offset) else: data_offset_var = None @@ -188,7 +207,8 @@ def _train_model_fn(self, features, labels, run_config): global_vars = {x.name: x for x in tf.global_variables()} for x in update_ops: if isinstance(x, ops.Operation) and x.inputs[0].name in global_vars: - ops.add_to_collection(constant.DENSE_UPDATE_VARIABLES, global_vars[x.inputs[0].name]) + ops.add_to_collection(constant.DENSE_UPDATE_VARIABLES, + global_vars[x.inputs[0].name]) update_op = tf.group(*update_ops, name='update_barrier') with tf.control_dependencies([update_op]): loss = tf.identity(loss, name='total_loss') @@ -208,12 +228,10 @@ def _train_model_fn(self, features, labels, run_config): all_opts.append(opt) grouped_vars = model.get_grouped_vars(len(all_opts)) assert len(grouped_vars) == len(optimizer_config), ( - 'the number of var group(%d) != the number of optimizers(%d)' - % ( - len(grouped_vars), - len(optimizer_config), - ) - ) + 'the number of var group(%d) != the number of optimizers(%d)' % ( + len(grouped_vars), + len(optimizer_config), + )) optimizer = MultiOptimizer(all_opts, grouped_vars) if self.train_config.train_distribute == DistributionStrategy.SokStrategy: @@ -227,56 +245,57 @@ def _train_model_fn(self, features, labels, run_config): # for distributed and synced training if self.train_config.sync_replicas and run_config.num_worker_replicas > 1: - logging.info('sync_replicas: num_worker_replias = %d' % run_config.num_worker_replicas) + logging.info('sync_replicas: num_worker_replias = %d' % + run_config.num_worker_replicas) if pai_util.is_on_pai(): optimizer = tf.train.SyncReplicasOptimizer( - optimizer, - replicas_to_aggregate=run_config.num_worker_replicas, - total_num_replicas=run_config.num_worker_replicas, - sparse_accumulator_type=self.train_config.sparse_accumulator_type, + optimizer, + replicas_to_aggregate=run_config.num_worker_replicas, + total_num_replicas=run_config.num_worker_replicas, + sparse_accumulator_type=self.train_config.sparse_accumulator_type, ) else: optimizer = sync_replicas_optimizer.SyncReplicasOptimizer( - optimizer, - replicas_to_aggregate=run_config.num_worker_replicas, - total_num_replicas=run_config.num_worker_replicas, + optimizer, + replicas_to_aggregate=run_config.num_worker_replicas, + total_num_replicas=run_config.num_worker_replicas, ) - hooks.append(optimizer.make_session_run_hook(run_config.is_chief, num_tokens=0)) + hooks.append( + optimizer.make_session_run_hook(run_config.is_chief, num_tokens=0)) # add barrier for no strategy case if run_config.num_worker_replicas > 1 and self.train_config.train_distribute == DistributionStrategy.NoStrategy: - hooks.append(estimator_utils.ExitBarrierHook(run_config.num_worker_replicas, run_config.is_chief, self.model_dir)) + hooks.append( + estimator_utils.ExitBarrierHook(run_config.num_worker_replicas, + run_config.is_chief, self.model_dir)) if self.export_config.enable_early_stop: eval_dir = os.path.join(self._model_dir, 'eval_val') logging.info('will use early stop, eval_events_dir=%s' % eval_dir) if self.export_config.HasField('early_stop_func'): hooks.append( - custom_early_stop_hook( - self, - eval_dir=eval_dir, - custom_stop_func=self.export_config.early_stop_func, - custom_stop_func_params=self.export_config.early_stop_params, - ) - ) + custom_early_stop_hook( + self, + eval_dir=eval_dir, + custom_stop_func=self.export_config.early_stop_func, + custom_stop_func_params=self.export_config.early_stop_params, + )) elif self.export_config.metric_bigger: hooks.append( - stop_if_no_increase_hook( - self, - self.export_config.best_exporter_metric, - self.export_config.max_check_steps, - eval_dir=eval_dir, - ) - ) + stop_if_no_increase_hook( + self, + self.export_config.best_exporter_metric, + self.export_config.max_check_steps, + eval_dir=eval_dir, + )) else: hooks.append( - stop_if_no_decrease_hook( - self, - self.export_config.best_exporter_metric, - self.export_config.max_check_steps, - eval_dir=eval_dir, - ) - ) + stop_if_no_decrease_hook( + self, + self.export_config.best_exporter_metric, + self.export_config.max_check_steps, + eval_dir=eval_dir, + )) if self.train_config.enable_oss_stop_signal: hooks.append(oss_stop_hook(self)) @@ -293,11 +312,14 @@ def _train_model_fn(self, features, labels, run_config): gradient_clipping_by_norm = None gradient_multipliers = None - if self.train_config.optimizer_config[0].HasField('embedding_learning_rate_multiplier'): + if self.train_config.optimizer_config[0].HasField( + 'embedding_learning_rate_multiplier'): gradient_multipliers = { - var: self.train_config.optimizer_config[0].embedding_learning_rate_multiplier - for var in tf.trainable_variables() - if 'embedding_weights:' in var.name or '/embedding_weights/part_' in var.name + var: self.train_config.optimizer_config[0] + .embedding_learning_rate_multiplier + for var in tf.trainable_variables() + if 'embedding_weights:' in var.name or + '/embedding_weights/part_' in var.name } # optimize loss @@ -321,19 +343,20 @@ def _train_model_fn(self, features, labels, run_config): logging.info('embedding_parallel is enabled') train_op = optimizers.optimize_loss( - loss=loss, - global_step=tf.train.get_global_step(), - learning_rate=None, - clip_gradients=gradient_clipping_by_norm, - optimizer=optimizer, - gradient_multipliers=gradient_multipliers, - variables=all_train_vars, - summaries=summaries, - colocate_gradients_with_ops=True, - not_apply_grad_after_first_step=run_config.is_chief and self._pipeline_config.data_config.chief_redundant, - name='', # Preventing scope prefix on all variables. - incr_save=(self.incr_save_config is not None), - embedding_parallel=self.embedding_parallel, + loss=loss, + global_step=tf.train.get_global_step(), + learning_rate=None, + clip_gradients=gradient_clipping_by_norm, + optimizer=optimizer, + gradient_multipliers=gradient_multipliers, + variables=all_train_vars, + summaries=summaries, + colocate_gradients_with_ops=True, + not_apply_grad_after_first_step=run_config.is_chief and + self._pipeline_config.data_config.chief_redundant, + name='', # Preventing scope prefix on all variables. + incr_save=(self.incr_save_config is not None), + embedding_parallel=self.embedding_parallel, ) # online evaluation @@ -347,7 +370,9 @@ def _train_model_fn(self, features, labels, run_config): tf.summary.scalar('%s/batch' % k, v[1]) train_op = tf.group([train_op] + list(metric_update_op_dict.values())) if estimator_utils.is_chief(): - hooks.append(estimator_utils.OnlineEvaluationHook(metric_dict=metric_dict, output_dir=self.model_dir)) + hooks.append( + estimator_utils.OnlineEvaluationHook( + metric_dict=metric_dict, output_dir=self.model_dir)) if self.train_config.HasField('fine_tune_checkpoint'): fine_tune_ckpt = self.train_config.fine_tune_checkpoint @@ -355,10 +380,10 @@ def _train_model_fn(self, features, labels, run_config): fine_tune_ckpt_var_map = self.train_config.fine_tune_ckpt_var_map force_restore = self.train_config.force_restore_shape_compatible restore_hook = model.restore( - fine_tune_ckpt, - include_global_step=False, - ckpt_var_map_path=fine_tune_ckpt_var_map, - force_restore_shape_compatible=force_restore, + fine_tune_ckpt, + include_global_step=False, + ckpt_var_map_path=fine_tune_ckpt_var_map, + force_restore_shape_compatible=force_restore, ) if restore_hook is not None: hooks.append(restore_hook) @@ -373,22 +398,24 @@ def _train_model_fn(self, features, labels, run_config): log_step_count_steps = self.train_config.log_step_count_steps logging_hook = basic_session_run_hooks.LoggingTensorHook( - logging_dict, - every_n_iter=log_step_count_steps, - formatter=estimator_utils.tensor_log_format_func, + logging_dict, + every_n_iter=log_step_count_steps, + formatter=estimator_utils.tensor_log_format_func, ) hooks.append(logging_hook) if self.train_config.train_distribute in [ - DistributionStrategy.CollectiveAllReduceStrategy, - DistributionStrategy.MirroredStrategy, - DistributionStrategy.MultiWorkerMirroredStrategy, + DistributionStrategy.CollectiveAllReduceStrategy, + DistributionStrategy.MirroredStrategy, + DistributionStrategy.MultiWorkerMirroredStrategy, ]: # for multi worker strategy, we could not replace the # inner CheckpointSaverHook, so just use it. scaffold = tf.train.Scaffold() else: - var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) + tf.get_collection(tf.GraphKeys.SAVEABLE_OBJECTS) + var_list = tf.get_collection( + tf.GraphKeys.GLOBAL_VARIABLES) + tf.get_collection( + tf.GraphKeys.SAVEABLE_OBJECTS) # exclude data_offset_var var_list = [x for x in var_list if x != data_offset_var] @@ -397,7 +424,9 @@ def _train_model_fn(self, features, labels, run_config): early_stop_var = find_early_stop_var(var_list) var_list = [x for x in var_list if x != early_stop_var] - initialize_var_list = [x for x in var_list if 'WorkQueue' not in str(type(x))] + initialize_var_list = [ + x for x in var_list if 'WorkQueue' not in str(type(x)) + ] # incompatiable shape restore will not be saved in checkpoint # but must be able to restore from checkpoint @@ -409,53 +438,61 @@ def _train_model_fn(self, features, labels, run_config): if early_stop_var is not None and estimator_utils.is_chief(): local_init_ops.append(tf.initializers.variables([early_stop_var])) if len(incompatiable_shape_restore) > 0: - local_init_ops.append(tf.initializers.variables(incompatiable_shape_restore)) + local_init_ops.append( + tf.initializers.variables(incompatiable_shape_restore)) scaffold = tf.train.Scaffold( - saver=self.saver_cls( - var_list=var_list, - sharded=True, - max_to_keep=self.train_config.keep_checkpoint_max, - save_relative_paths=True, - ), - local_init_op=tf.group(local_init_ops), - ready_for_local_init_op=tf.report_uninitialized_variables(var_list=initialize_var_list), + saver=self.saver_cls( + var_list=var_list, + sharded=True, + max_to_keep=self.train_config.keep_checkpoint_max, + save_relative_paths=True, + ), + local_init_op=tf.group(local_init_ops), + ready_for_local_init_op=tf.report_uninitialized_variables( + var_list=initialize_var_list), ) # saver hook saver_hook = estimator_utils.CheckpointSaverHook( - checkpoint_dir=self.model_dir, - save_secs=self._config.save_checkpoints_secs, - save_steps=self._config.save_checkpoints_steps, - scaffold=scaffold, - write_graph=self.train_config.write_graph, - data_offset_var=data_offset_var, - increment_save_config=self.incr_save_config, + checkpoint_dir=self.model_dir, + save_secs=self._config.save_checkpoints_secs, + save_steps=self._config.save_checkpoints_steps, + scaffold=scaffold, + write_graph=self.train_config.write_graph, + data_offset_var=data_offset_var, + increment_save_config=self.incr_save_config, ) if estimator_utils.is_chief() or self.embedding_parallel: hooks.append(saver_hook) if estimator_utils.is_chief(): hooks.append( - basic_session_run_hooks.StepCounterHook(every_n_steps=log_step_count_steps, output_dir=self.model_dir) - ) + basic_session_run_hooks.StepCounterHook( + every_n_steps=log_step_count_steps, output_dir=self.model_dir)) # profiling hook if self.train_config.is_profiling and estimator_utils.is_chief(): - profile_hook = tf.train.ProfilerHook(save_steps=log_step_count_steps, output_dir=self.model_dir) + profile_hook = tf.train.ProfilerHook( + save_steps=log_step_count_steps, output_dir=self.model_dir) hooks.append(profile_hook) return tf.estimator.EstimatorSpec( - mode=tf.estimator.ModeKeys.TRAIN, - loss=loss, - predictions=predict_dict, - train_op=train_op, - scaffold=scaffold, - training_hooks=hooks, + mode=tf.estimator.ModeKeys.TRAIN, + loss=loss, + predictions=predict_dict, + train_op=train_op, + scaffold=scaffold, + training_hooks=hooks, ) def _eval_model_fn(self, features, labels, run_config): tf.keras.backend.set_learning_phase(0) start = time.time() - model = self._model_cls(self.model_config, self.feature_configs, features, labels, is_training=False) + model = self._model_cls( + self.model_config, + self.feature_configs, + features, + labels, + is_training=False) predict_dict = model.build_predict_graph() loss_dict = model.build_loss_graph() loss = tf.add_n(list(loss_dict.values())) @@ -468,28 +505,36 @@ def _eval_model_fn(self, features, labels, run_config): metric_dict['loss/loss/' + loss_key] = tf.metrics.mean(loss_tensor) tf.logging.info('metric_dict keys: %s' % metric_dict.keys()) - var_list = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS) + var_list = ops.get_collection( + ops.GraphKeys.GLOBAL_VARIABLES) + ops.get_collection( + ops.GraphKeys.SAVEABLE_OBJECTS) metric_variables = ops.get_collection(ops.GraphKeys.METRIC_VARIABLES) model_ready_for_local_init_op = tf.variables_initializer(metric_variables) scaffold = tf.train.Scaffold( - saver=self.saver_cls(var_list=var_list, sharded=True, save_relative_paths=True), - ready_for_local_init_op=model_ready_for_local_init_op, + saver=self.saver_cls( + var_list=var_list, sharded=True, save_relative_paths=True), + ready_for_local_init_op=model_ready_for_local_init_op, ) end = time.time() tf.logging.info('eval graph construct finished. Time %.3fs' % (end - start)) return tf.estimator.EstimatorSpec( - mode=tf.estimator.ModeKeys.EVAL, - loss=loss, - scaffold=scaffold, - predictions=predict_dict, - eval_metric_ops=metric_dict, + mode=tf.estimator.ModeKeys.EVAL, + loss=loss, + scaffold=scaffold, + predictions=predict_dict, + eval_metric_ops=metric_dict, ) def _distribute_eval_model_fn(self, features, labels, run_config): tf.keras.backend.set_learning_phase(0) start = time.time() - model = self._model_cls(self.model_config, self.feature_configs, features, labels, is_training=False) + model = self._model_cls( + self.model_config, + self.feature_configs, + features, + labels, + is_training=False) predict_dict = model.build_predict_graph() loss_dict = model.build_loss_graph() loss = tf.add_n(list(loss_dict.values())) @@ -522,25 +567,27 @@ def _distribute_eval_model_fn(self, features, labels, run_config): global_variables = tf.global_variables() metric_variables = tf.get_collection(tf.GraphKeys.METRIC_VARIABLES) model_ready_for_local_init_op = tf.variables_initializer(metric_variables) - remain_variables = list(set(global_variables).difference(set(metric_variables))) + remain_variables = list( + set(global_variables).difference(set(metric_variables))) cur_saver = tf.train.Saver(var_list=remain_variables, sharded=True) - scaffold = tf.train.Scaffold(saver=cur_saver, ready_for_local_init_op=model_ready_for_local_init_op) + scaffold = tf.train.Scaffold( + saver=cur_saver, ready_for_local_init_op=model_ready_for_local_init_op) return tf.estimator.EstimatorSpec( - mode=tf.estimator.ModeKeys.EVAL, - loss=loss, - predictions=predict_dict, - eval_metric_ops=metric_dict, - scaffold=scaffold, + mode=tf.estimator.ModeKeys.EVAL, + loss=loss, + predictions=predict_dict, + eval_metric_ops=metric_dict, + scaffold=scaffold, ) def _export_model_fn(self, features, labels, run_config, params): tf.keras.backend.set_learning_phase(0) model = self._model_cls( - self.model_config, - self.feature_configs, - features, - labels=None, - is_training=False, + self.model_config, + self.feature_configs, + features, + labels=None, + is_training=False, ) model.build_predict_graph() @@ -556,50 +603,60 @@ def _export_model_fn(self, features, labels, run_config, params): outputs.update(model.build_rtp_output_dict()) for out in outputs: - tf.logging.info('output %s shape: %s type: %s' % (out, outputs[out].get_shape().as_list(), outputs[out].dtype)) - export_outputs = {signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: tf.estimator.export.PredictOutput(outputs)} + tf.logging.info( + 'output %s shape: %s type: %s' % + (out, outputs[out].get_shape().as_list(), outputs[out].dtype)) + export_outputs = { + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + tf.estimator.export.PredictOutput(outputs) + } # save train pipeline.config for debug purpose pipeline_path = os.path.join(self._model_dir, 'pipeline.config') if gfile.Exists(pipeline_path): ops.add_to_collection( - tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(pipeline_path, dtype=tf.string, name='pipeline.config'), + tf.GraphKeys.ASSET_FILEPATHS, + tf.constant(pipeline_path, dtype=tf.string, name='pipeline.config'), ) else: print('train pipeline_path(%s) does not exist' % pipeline_path) # restore DENSE_UPDATE_VARIABLES collection - dense_train_var_path = os.path.join(self.model_dir, constant.DENSE_UPDATE_VARIABLES) + dense_train_var_path = os.path.join(self.model_dir, + constant.DENSE_UPDATE_VARIABLES) if gfile.Exists(dense_train_var_path): with gfile.GFile(dense_train_var_path, 'r') as fin: var_name_to_id_map = json.load(fin) - var_name_id_lst = [(x, var_name_to_id_map[x]) for x in var_name_to_id_map] + var_name_id_lst = [ + (x, var_name_to_id_map[x]) for x in var_name_to_id_map + ] var_name_id_lst.sort(key=lambda x: x[1]) all_vars = {x.op.name: x for x in tf.global_variables()} for var_name, var_id in var_name_id_lst: assert var_name in all_vars, 'dense_train_var[%s] is not found' % var_name - ops.add_to_collection(constant.DENSE_UPDATE_VARIABLES, all_vars[var_name]) + ops.add_to_collection(constant.DENSE_UPDATE_VARIABLES, + all_vars[var_name]) # add more asset files if len(export_config.asset_files) > 0: for asset_file in export_config.asset_files: if asset_file.startswith('!'): asset_file = asset_file[1:] - if ':' not in asset_file or asset_file.startswith('oss:') or asset_file.startswith('hdfs:'): + if ':' not in asset_file or asset_file.startswith( + 'oss:') or asset_file.startswith('hdfs:'): _, asset_name = os.path.split(asset_file) else: asset_name, asset_file = asset_file.split(':', 1) ops.add_to_collection( - ops.GraphKeys.ASSET_FILEPATHS, - tf.constant(asset_file, dtype=tf.string, name=asset_name), + ops.GraphKeys.ASSET_FILEPATHS, + tf.constant(asset_file, dtype=tf.string, name=asset_name), ) elif 'asset_files' in params: for asset_name in params['asset_files']: asset_file = params['asset_files'][asset_name] ops.add_to_collection( - tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(asset_file, dtype=tf.string, name=asset_name), + tf.GraphKeys.ASSET_FILEPATHS, + tf.constant(asset_file, dtype=tf.string, name=asset_name), ) if self._pipeline_config.HasField('fg_json_path'): @@ -607,20 +664,24 @@ def _export_model_fn(self, features, labels, run_config, params): if fg_path[0] == '!': fg_path = fg_path[1:] ops.add_to_collection( - tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(fg_path, dtype=tf.string, name='fg.json'), + tf.GraphKeys.ASSET_FILEPATHS, + tf.constant(fg_path, dtype=tf.string, name='fg.json'), ) - var_list = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS) + var_list = ops.get_collection( + ops.GraphKeys.GLOBAL_VARIABLES) + ops.get_collection( + ops.GraphKeys.SAVEABLE_OBJECTS) - scaffold = tf.train.Scaffold(saver=self.saver_cls(var_list=var_list, sharded=True, save_relative_paths=True)) + scaffold = tf.train.Scaffold( + saver=self.saver_cls( + var_list=var_list, sharded=True, save_relative_paths=True)) return tf.estimator.EstimatorSpec( - mode=tf.estimator.ModeKeys.PREDICT, - loss=None, - scaffold=scaffold, - predictions=outputs, - export_outputs=export_outputs, + mode=tf.estimator.ModeKeys.PREDICT, + loss=None, + scaffold=scaffold, + predictions=outputs, + export_outputs=export_outputs, ) def _model_fn(self, features, labels, mode, config, params): @@ -629,7 +690,8 @@ def _model_fn(self, features, labels, mode, config, params): if self._pipeline_config.feature_config.embedding_on_cpu: os.environ['place_embedding_on_cpu'] = 'True' if self._pipeline_config.fg_json_path: - EasyRecEstimator._write_rtp_fg_config_to_col(fg_config_path=self._pipeline_config.fg_json_path) + EasyRecEstimator._write_rtp_fg_config_to_col( + fg_config_path=self._pipeline_config.fg_json_path) EasyRecEstimator._write_rtp_inputs_to_col(features) if self.embedding_parallel: @@ -679,11 +741,11 @@ def _write_rtp_inputs_to_col(features): col[0] = json.dumps(feature_info_map) def export_checkpoint( - self, - export_path=None, - serving_input_receiver_fn=None, - checkpoint_path=None, - mode=tf.estimator.ModeKeys.PREDICT, + self, + export_path=None, + serving_input_receiver_fn=None, + checkpoint_path=None, + mode=tf.estimator.ModeKeys.PREDICT, ): with context.graph_mode(): if not checkpoint_path: @@ -694,12 +756,13 @@ def export_checkpoint( with ops.Graph().as_default(): input_receiver = serving_input_receiver_fn() estimator_spec = self._call_model_fn( - features=input_receiver.features, - labels=getattr(input_receiver, 'labels', None), - mode=mode, - config=self.config, + features=input_receiver.features, + labels=getattr(input_receiver, 'labels', None), + mode=mode, + config=self.config, ) with tf_session.Session(config=self._session_config) as session: - graph_saver = estimator_spec.scaffold.saver or saver.Saver(sharded=True) + graph_saver = estimator_spec.scaffold.saver or saver.Saver( + sharded=True) graph_saver.restore(session, checkpoint_path) graph_saver.save(session, export_path) diff --git a/easy_rec/python/model/easy_rec_model.py b/easy_rec/python/model/easy_rec_model.py index 4ea3dd77b..8087f5e68 100644 --- a/easy_rec/python/model/easy_rec_model.py +++ b/easy_rec/python/model/easy_rec_model.py @@ -8,14 +8,17 @@ import six import tensorflow as tf -from tensorflow.python.framework import ops, tensor_shape +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from easy_rec.python.compat import regularizers from easy_rec.python.layers import input_layer from easy_rec.python.layers.backbone import Backbone -from easy_rec.python.utils import constant, estimator_utils, restore_filter +from easy_rec.python.utils import constant +from easy_rec.python.utils import estimator_utils +from easy_rec.python.utils import restore_filter from easy_rec.python.utils.load_class import get_register_class_meta try: @@ -41,11 +44,18 @@ tf = tf.compat.v1 _EASY_REC_MODEL_CLASS_MAP = {} -_meta_type = get_register_class_meta(_EASY_REC_MODEL_CLASS_MAP, have_abstract_class=True) +_meta_type = get_register_class_meta( + _EASY_REC_MODEL_CLASS_MAP, have_abstract_class=True) class EasyRecModel(six.with_metaclass(_meta_type, object)): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): self._base_model_config = model_config self._model_config = model_config self._is_training = is_training @@ -94,10 +104,10 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai def build_backbone_network(self): if self.has_backbone: return Backbone( - self._base_model_config.backbone, - self._feature_dict, - input_layer=self._input_layer, - l2_reg=self._l2_reg, + self._base_model_config.backbone, + self._feature_dict, + input_layer=self._input_layer, + l2_reg=self._l2_reg, ) return None @@ -111,11 +121,11 @@ def backbone(self): return self._backbone_output if self._backbone_net: kwargs = { - 'loss_dict': self._loss_dict, - 'metric_dict': self._metric_dict, - 'prediction_dict': self._prediction_dict, - 'labels': self._labels, - constant.SAMPLE_WEIGHT: self._sample_weight, + 'loss_dict': self._loss_dict, + 'metric_dict': self._metric_dict, + 'prediction_dict': self._prediction_dict, + 'labels': self._labels, + constant.SAMPLE_WEIGHT: self._sample_weight, } return self._backbone_net(self._is_training, **kwargs) return None @@ -134,11 +144,14 @@ def feature_groups(self): @property def l2_regularization(self): - model_config = getattr(self._base_model_config, self._base_model_config.WhichOneof('model')) + model_config = getattr(self._base_model_config, + self._base_model_config.WhichOneof('model')) l2_regularization = 0.0 - if hasattr(model_config, 'dense_regularization') and model_config.HasField('dense_regularization'): + if hasattr(model_config, 'dense_regularization') and model_config.HasField( + 'dense_regularization'): # backward compatibility - logging.warn('dense_regularization is deprecated, please use l2_regularization') + logging.warn( + 'dense_regularization is deprecated, please use l2_regularization') l2_regularization = model_config.dense_regularization elif hasattr(model_config, 'l2_regularization'): l2_regularization = model_config.l2_regularization @@ -146,17 +159,17 @@ def l2_regularization(self): def build_input_layer(self, model_config, feature_configs): self._input_layer = input_layer.InputLayer( - feature_configs, - model_config.feature_groups, - wide_output_dim=self._wide_output_dim, - ev_params=self._global_ev_params, - embedding_regularizer=self._emb_reg, - kernel_regularizer=self._l2_reg, - variational_dropout_config=( - model_config.variational_dropout if model_config.HasField('variational_dropout') else None - ), - is_training=self._is_training, - is_predicting=self._is_predicting, + feature_configs, + model_config.feature_groups, + wide_output_dim=self._wide_output_dim, + ev_params=self._global_ev_params, + embedding_regularizer=self._emb_reg, + kernel_regularizer=self._l2_reg, + variational_dropout_config=(model_config.variational_dropout if + model_config.HasField('variational_dropout') + else None), + is_training=self._is_training, + is_predicting=self._is_predicting, ) @abstractmethod @@ -179,7 +192,9 @@ def build_output_dict(self): outputs = {} for name in self.get_outputs(): if name not in self._prediction_dict: - raise KeyError('output node {} not in prediction_dict, can not be exported'.format(name)) + raise KeyError( + 'output node {} not in prediction_dict, can not be exported'.format( + name)) outputs[name] = self._prediction_dict[name] return outputs @@ -193,7 +208,9 @@ def build_feature_output_dict(self): sparse_values = feature_value.values if sparse_values.dtype != tf.string: sparse_values = tf.as_string(sparse_values) - feature_value = tf.sparse_to_dense(feature_value.indices, feature_value.dense_shape, sparse_values, '') + feature_value = tf.sparse_to_dense(feature_value.indices, + feature_value.dense_shape, + sparse_values, '') elif feature_value.dtype != tf.string: feature_value = tf.as_string(feature_value) feature_value = tf.reduce_join(feature_value, axis=-1, separator=',') @@ -205,11 +222,11 @@ def build_rtp_output_dict(self): return {} def restore( - self, - ckpt_path, - include_global_step=False, - ckpt_var_map_path='', - force_restore_shape_compatible=False, + self, + ckpt_path, + include_global_step=False, + ckpt_var_map_path='', + force_restore_shape_compatible=False, ): """Restore variables from ckpt_path. @@ -252,44 +269,48 @@ def restore( var_shape[0] += x[0] var_shape = tensor_shape.TensorShape(var_shape) variable = variables.PartitionedVariable( - variable_name, - var_shape, - variable[0].dtype, - variable, - partitions=[len(variable)] + [1] * (len(var_shape) - 1), + variable_name, + var_shape, + variable[0].dtype, + variable, + partitions=[len(variable)] + [1] * (len(var_shape) - 1), ) else: var_shape = variable.shape.as_list() if ckpt_var_shape == var_shape: vars_in_ckpt[variable_name] = ( - list(variable) if isinstance(variable, variables.PartitionedVariable) else variable - ) + list(variable) if isinstance( + variable, variables.PartitionedVariable) else variable) elif len(ckpt_var_shape) == len(var_shape): if force_restore_shape_compatible: # create a variable compatible with checkpoint to restore - dtype = variable[0].dtype if isinstance(variable, list) else variable.dtype + dtype = variable[0].dtype if isinstance(variable, + list) else variable.dtype with tf.variable_scope('incompatible_shape_restore'): tmp_var = tf.get_variable( - name=variable_name + '_T_E_M_P', - shape=ckpt_var_shape, - trainable=False, - # add to a special collection for easy reference - # by tf.get_collection('T_E_M_P_RESTROE') - collections=['T_E_M_P_RESTROE'], - dtype=dtype, + name=variable_name + '_T_E_M_P', + shape=ckpt_var_shape, + trainable=False, + # add to a special collection for easy reference + # by tf.get_collection('T_E_M_P_RESTROE') + collections=['T_E_M_P_RESTROE'], + dtype=dtype, ) vars_in_ckpt[variable_name] = tmp_var incompatible_shape_var_map[variable] = tmp_var - print('incompatible restore %s[%s, %s]' % (variable_name, str(var_shape), str(ckpt_var_shape))) + print('incompatible restore %s[%s, %s]' % + (variable_name, str(var_shape), str(ckpt_var_shape))) else: logging.warning( - 'Variable [%s] is available in checkpoint, but ' 'incompatible shape with model variable.', - variable_name, + 'Variable [%s] is available in checkpoint, but ' + 'incompatible shape with model variable.', + variable_name, ) else: logging.warning( - 'Variable [%s] is available in checkpoint, but ' 'incompatible shape dims with model variable.', - variable_name, + 'Variable [%s] is available in checkpoint, but ' + 'incompatible shape dims with model variable.', + variable_name, ) elif 'EmbeddingVariable' in str(type(variable)): if '%s-keys' % variable_name not in ckpt_var2shape_map: @@ -304,7 +325,8 @@ def restore( saveable_objects.append(s) init_op = saveable_objects[0].restore([ckpt_path], None) variable._initializer_op = init_op - elif type(variable) == list and 'EmbeddingVariable' in str(type(variable[0])): + elif type(variable) == list and 'EmbeddingVariable' in str( + type(variable[0])): if '%s/part_0-keys' % variable_name not in ckpt_var2shape_map: continue print('restore partitioned embedding_variable %s' % variable_name) @@ -321,24 +343,27 @@ def restore( elif sok is not None and isinstance(variable, sok.DynamicVariable): print('restore dynamic_variable %s' % variable_name) keys, vals = load_embed_lib.load_kv_embed( - task_index=hvd.rank(), - task_num=hvd.size(), - embed_dim=variable._dimension, - var_name='embed-' + variable.name.replace('/', '__'), - ckpt_path=ckpt_path, + task_index=hvd.rank(), + task_num=hvd.size(), + embed_dim=variable._dimension, + var_name='embed-' + variable.name.replace('/', '__'), + ckpt_path=ckpt_path, ) with ops.control_dependencies([variable._initializer_op]): - variable._initializer_op = dynamic_variable_ops.dummy_var_assign(variable.handle, keys, vals) + variable._initializer_op = dynamic_variable_ops.dummy_var_assign( + variable.handle, keys, vals) else: fail_restore_vars.append(variable_name) for variable_name in fail_restore_vars: if 'Momentum' not in variable_name: - logging.warning('Variable [%s] is not available in checkpoint', variable_name) + logging.warning('Variable [%s] is not available in checkpoint', + variable_name) tf.train.init_from_checkpoint(ckpt_path, vars_in_ckpt) if force_restore_shape_compatible: - return estimator_utils.IncompatibleShapeRestoreHook(incompatible_shape_var_map) + return estimator_utils.IncompatibleShapeRestoreHook( + incompatible_shape_var_map) else: return None @@ -361,7 +386,8 @@ def _get_restore_vars(self, ckpt_var_map_path): name2var = {} for one_var in all_vars: var_name = re.sub(VAR_SUFIX_PATTERN, '', one_var.name) - if re.search(PARTITION_PATTERN, var_name) and one_var._save_slice_info is not None: + if re.search(PARTITION_PATTERN, + var_name) and one_var._save_slice_info is not None: var_name = re.sub(PARTITION_PATTERN, '', var_name) is_part = True else: @@ -401,10 +427,16 @@ def _get_restore_vars(self, ckpt_var_map_path): else: var_filter, scope_update = self.get_restore_filter() if var_filter is not None: - name2var = {var_name: name2var[var_name] for var in name2var if var_filter.keep(var.name)} + name2var = { + var_name: name2var[var_name] + for var in name2var + if var_filter.keep(var.name) + } # drop scope prefix if necessary if scope_update is not None: - name2var = {scope_update(var_name): name2var[var_name] for var_name in name2var} + name2var = { + scope_update(var_name): name2var[var_name] for var_name in name2var + } return name2var def get_restore_filter(self): @@ -420,11 +452,14 @@ def get_restore_filter(self): for x in self._base_model_config.restore_filters: logging.info('restore will filter out pattern %s' % x) - all_filters = [restore_filter.KeywordFilter(x, True) for x in self._base_model_config.restore_filters] + all_filters = [ + restore_filter.KeywordFilter(x, True) + for x in self._base_model_config.restore_filters + ] return ( - restore_filter.CombineFilter(all_filters, restore_filter.Logical.AND), - None, + restore_filter.CombineFilter(all_filters, restore_filter.Logical.AND), + None, ) def get_grouped_vars(self, opt_num): @@ -443,7 +478,8 @@ def get_grouped_vars(self, opt_num): embedding_vars = [] deep_vars = [] for tmp_var in variables.trainable_variables(): - if tmp_var.name.startswith('input_layer') or '/embedding_weights' in tmp_var.name: + if tmp_var.name.startswith( + 'input_layer') or '/embedding_weights' in tmp_var.name: embedding_vars.append(tmp_var) else: deep_vars.append(tmp_var) diff --git a/easy_rec/python/model/esmm.py b/easy_rec/python/model/esmm.py index 62e338ca6..f998f9734 100644 --- a/easy_rec/python/model/esmm.py +++ b/easy_rec/python/model/esmm.py @@ -15,11 +15,19 @@ class ESMM(MultiTaskModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(ESMM, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'esmm', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model' - ) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(ESMM, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert self._model_config.WhichOneof( + 'model' + ) == 'esmm', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.esmm assert isinstance(self._model_config, ESMMConfig) @@ -59,27 +67,35 @@ def build_loss_graph(self): cvr_label_name = self._label_name_dict[cvr_tower_name] ctr_label_name = self._label_name_dict[ctr_tower_name] if self._cvr_tower_cfg.loss_type == LossType.CLASSIFICATION: - ctcvr_label = tf.cast(self._labels[cvr_label_name] * self._labels[ctr_label_name], tf.float32) - cvr_losses = tf.keras.backend.binary_crossentropy(ctcvr_label, self._prediction_dict['probs_ctcvr']) + ctcvr_label = tf.cast( + self._labels[cvr_label_name] * self._labels[ctr_label_name], + tf.float32) + cvr_losses = tf.keras.backend.binary_crossentropy( + ctcvr_label, self._prediction_dict['probs_ctcvr']) cvr_loss = tf.reduce_sum(cvr_losses, name='ctcvr_loss') # The weight defaults to 1. - self._loss_dict['weighted_cross_entropy_loss_%s' % cvr_tower_name] = self._cvr_tower_cfg.weight * cvr_loss + self._loss_dict['weighted_cross_entropy_loss_%s' % + cvr_tower_name] = self._cvr_tower_cfg.weight * cvr_loss elif self._cvr_tower_cfg.loss_type == LossType.L2_LOSS: logging.info('l2 loss is used') cvr_dtype = self._labels[cvr_label_name].dtype - ctcvr_label = self._labels[cvr_label_name] * tf.cast(self._labels[ctr_label_name], cvr_dtype) + ctcvr_label = self._labels[cvr_label_name] * tf.cast( + self._labels[ctr_label_name], cvr_dtype) cvr_loss = tf.losses.mean_squared_error( - labels=ctcvr_label, - predictions=self._prediction_dict['y_ctcvr'], - weights=self._sample_weight, + labels=ctcvr_label, + predictions=self._prediction_dict['y_ctcvr'], + weights=self._sample_weight, ) - self._loss_dict['weighted_l2_loss_%s' % cvr_tower_name] = self._cvr_tower_cfg.weight * cvr_loss + self._loss_dict['weighted_l2_loss_%s' % + cvr_tower_name] = self._cvr_tower_cfg.weight * cvr_loss _labels = tf.cast(self._labels[ctr_label_name], tf.float32) _logits = self._prediction_dict['logits_%s' % ctr_tower_name] - cross = tf.nn.sigmoid_cross_entropy_with_logits(labels=_labels, logits=_logits, name='ctr_loss') + cross = tf.nn.sigmoid_cross_entropy_with_logits( + labels=_labels, logits=_logits, name='ctr_loss') ctr_loss = tf.reduce_sum(cross) - self._loss_dict['weighted_cross_entropy_loss_%s' % ctr_tower_name] = self._ctr_tower_cfg.weight * ctr_loss + self._loss_dict['weighted_cross_entropy_loss_%s' % + ctr_tower_name] = self._ctr_tower_cfg.weight * ctr_loss return self._loss_dict def build_metric_graph(self, eval_config): @@ -101,61 +117,61 @@ def build_metric_graph(self, eval_config): # CTCVR metric ctcvr_label_name = cvr_label_name + '_ctcvr' cvr_dtype = self._labels[cvr_label_name].dtype - self._labels[ctcvr_label_name] = self._labels[cvr_label_name] * tf.cast(self._labels[ctr_label_name], cvr_dtype) + self._labels[ctcvr_label_name] = self._labels[cvr_label_name] * tf.cast( + self._labels[ctr_label_name], cvr_dtype) metric_dict.update( - self._build_metric_impl( - metric, - loss_type=self._cvr_tower_cfg.loss_type, - label_name=ctcvr_label_name, - num_class=self._cvr_tower_cfg.num_class, - suffix='_ctcvr', - ) - ) + self._build_metric_impl( + metric, + loss_type=self._cvr_tower_cfg.loss_type, + label_name=ctcvr_label_name, + num_class=self._cvr_tower_cfg.num_class, + suffix='_ctcvr', + )) # CVR metric cvr_label_masked_name = cvr_label_name + '_masked' ctr_mask = self._labels[ctr_label_name] > 0 - self._labels[cvr_label_masked_name] = tf.boolean_mask(self._labels[cvr_label_name], ctr_mask) + self._labels[cvr_label_masked_name] = tf.boolean_mask( + self._labels[cvr_label_name], ctr_mask) pred_prefix = 'probs' if self._cvr_tower_cfg.loss_type == LossType.CLASSIFICATION else 'y' pred_name = '%s_%s' % (pred_prefix, cvr_tower_name) - self._prediction_dict[pred_name + '_masked'] = tf.boolean_mask(self._prediction_dict[pred_name], ctr_mask) + self._prediction_dict[pred_name + '_masked'] = tf.boolean_mask( + self._prediction_dict[pred_name], ctr_mask) metric_dict.update( - self._build_metric_impl( - metric, - loss_type=self._cvr_tower_cfg.loss_type, - label_name=cvr_label_masked_name, - num_class=self._cvr_tower_cfg.num_class, - suffix='_%s_masked' % cvr_tower_name, - ) - ) + self._build_metric_impl( + metric, + loss_type=self._cvr_tower_cfg.loss_type, + label_name=cvr_label_masked_name, + num_class=self._cvr_tower_cfg.num_class, + suffix='_%s_masked' % cvr_tower_name, + )) for metric in self._ctr_tower_cfg.metrics_set: # CTR metric metric_dict.update( - self._build_metric_impl( - metric, - loss_type=self._ctr_tower_cfg.loss_type, - label_name=ctr_label_name, - num_class=self._ctr_tower_cfg.num_class, - suffix='_%s' % ctr_tower_name, - ) - ) + self._build_metric_impl( + metric, + loss_type=self._ctr_tower_cfg.loss_type, + label_name=ctr_label_name, + num_class=self._ctr_tower_cfg.num_class, + suffix='_%s' % ctr_tower_name, + )) return metric_dict def _add_to_prediction_dict(self, output): super(ESMM, self)._add_to_prediction_dict(output) if self._cvr_tower_cfg.loss_type == LossType.CLASSIFICATION: prob = tf.multiply( - self._prediction_dict['probs_%s' % self._cvr_tower_cfg.tower_name], - self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name], + self._prediction_dict['probs_%s' % self._cvr_tower_cfg.tower_name], + self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name], ) # pctcvr = pctr * pcvr self._prediction_dict['probs_ctcvr'] = prob else: prob = tf.multiply( - self._prediction_dict['y_%s' % self._cvr_tower_cfg.tower_name], - self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name], + self._prediction_dict['y_%s' % self._cvr_tower_cfg.tower_name], + self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name], ) # pctcvr = pctr * pcvr self._prediction_dict['y_ctcvr'] = prob @@ -175,7 +191,8 @@ def build_predict_graph(self): group_fea = self._group_features[group_id] group = self._model_config.groups[group_id] group_name = group.input - dnn_model = dnn.DNN(group.dnn, self._l2_reg, group_name, self._is_training) + dnn_model = dnn.DNN(group.dnn, self._l2_reg, group_name, + self._is_training) group_fea = dnn_model(group_fea) group_fea_arr.append(group_fea) all_fea = tf.concat(group_fea_arr, axis=1) @@ -184,37 +201,37 @@ def build_predict_graph(self): cvr_tower_name = self._cvr_tower_cfg.tower_name dnn_model = dnn.DNN( - self._cvr_tower_cfg.dnn, - self._l2_reg, - name=cvr_tower_name, - is_training=self._is_training, + self._cvr_tower_cfg.dnn, + self._l2_reg, + name=cvr_tower_name, + is_training=self._is_training, ) cvr_tower_output = dnn_model(all_fea) cvr_tower_output = tf.layers.dense( - inputs=cvr_tower_output, - units=1, - kernel_regularizer=self._l2_reg, - name='%s/dnn_output' % cvr_tower_name, + inputs=cvr_tower_output, + units=1, + kernel_regularizer=self._l2_reg, + name='%s/dnn_output' % cvr_tower_name, ) ctr_tower_name = self._ctr_tower_cfg.tower_name dnn_model = dnn.DNN( - self._ctr_tower_cfg.dnn, - self._l2_reg, - name=ctr_tower_name, - is_training=self._is_training, + self._ctr_tower_cfg.dnn, + self._l2_reg, + name=ctr_tower_name, + is_training=self._is_training, ) ctr_tower_output = dnn_model(all_fea) ctr_tower_output = tf.layers.dense( - inputs=ctr_tower_output, - units=1, - kernel_regularizer=self._l2_reg, - name='%s/dnn_output' % ctr_tower_name, + inputs=ctr_tower_output, + units=1, + kernel_regularizer=self._l2_reg, + name='%s/dnn_output' % ctr_tower_name, ) tower_outputs = { - cvr_tower_name: cvr_tower_output, - ctr_tower_name: ctr_tower_output, + cvr_tower_name: cvr_tower_output, + ctr_tower_name: ctr_tower_output, } self._add_to_prediction_dict(tower_outputs) return self._prediction_dict @@ -231,5 +248,6 @@ def get_outputs(self): elif self._cvr_tower_cfg.loss_type == LossType.L2_LOSS: outputs.append('y_ctcvr') else: - raise ValueError('invalid cvr_tower loss type: %s' % str(self._cvr_tower_cfg.loss_type)) + raise ValueError('invalid cvr_tower loss type: %s' % + str(self._cvr_tower_cfg.loss_type)) return outputs diff --git a/easy_rec/python/model/fm.py b/easy_rec/python/model/fm.py index 6f92e6e95..c54042684 100644 --- a/easy_rec/python/model/fm.py +++ b/easy_rec/python/model/fm.py @@ -13,16 +13,25 @@ class FM(RankModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(FM, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'fm', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model' - ) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(FM, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert self._model_config.WhichOneof( + 'model' + ) == 'fm', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.fm assert isinstance(self._model_config, FMConfig) self._wide_features, _ = self._input_layer(self._feature_dict, 'wide') - self._deep_features, self._fm_features = self._input_layer(self._feature_dict, 'deep') + self._deep_features, self._fm_features = self._input_layer( + self._feature_dict, 'deep') def build_input_layer(self, model_config, feature_configs): # overwrite create input_layer to support wide_output_dim @@ -30,25 +39,26 @@ def build_input_layer(self, model_config, feature_configs): super(FM, self).build_input_layer(model_config, feature_configs) def build_predict_graph(self): - wide_fea = tf.reduce_sum(self._wide_features, axis=1, keepdims=True, name='wide_feature') + wide_fea = tf.reduce_sum( + self._wide_features, axis=1, keepdims=True, name='wide_feature') fm_fea = fm.FM(name='fm_feature')(self._fm_features) if self._num_class > 1: fm_fea = tf.layers.dense( - fm_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='fm_logits', + fm_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='fm_logits', ) else: fm_fea = tf.reduce_sum(fm_fea, 1, keepdims=True) bias = tf.get_variable( - 'fm_bias', - [self._num_class], - initializer=tf.zeros_initializer(), - trainable=True, + 'fm_bias', + [self._num_class], + initializer=tf.zeros_initializer(), + trainable=True, ) output = wide_fea + fm_fea diff --git a/easy_rec/python/model/match_model.py b/easy_rec/python/model/match_model.py index a28c82453..e9c4d2d44 100644 --- a/easy_rec/python/model/match_model.py +++ b/easy_rec/python/model/match_model.py @@ -16,8 +16,15 @@ class MatchModel(EasyRecModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(MatchModel, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(MatchModel, self).__init__(model_config, feature_configs, features, + labels, is_training) self._loss_type = self._model_config.loss_type self._num_class = self._model_config.num_class @@ -43,24 +50,21 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai def _mask_in_batch(self, logits): batch_size = tf.shape(logits)[0] if getattr(self._model_config, 'ignore_in_batch_neg_sam', False): - in_batch = logits[:, :batch_size] - (1 - tf.diag(tf.ones([batch_size], dtype=tf.float32))) * 1e32 + in_batch = logits[:, :batch_size] - ( + 1 - tf.diag(tf.ones([batch_size], dtype=tf.float32))) * 1e32 return tf.concat([in_batch, logits[:, batch_size:]], axis=1) else: if self._item_ids is not None: mask_in_batch_neg = tf.to_float( - tf.equal( - self._item_ids[None, :batch_size], - self._item_ids[:batch_size, None], - ) - ) - tf.diag(tf.ones([batch_size], dtype=tf.float32)) - tf.summary.scalar('in_batch_neg_conflict', tf.reduce_sum(mask_in_batch_neg)) - return tf.concat( - [ - logits[:, :batch_size] - mask_in_batch_neg * 1e32, - logits[:, batch_size:], - ], - axis=1, - ) # yapf: disable + tf.equal(self._item_ids[None, :batch_size], + self._item_ids[:batch_size, None])) - tf.diag( + tf.ones([batch_size], dtype=tf.float32)) + tf.summary.scalar('in_batch_neg_conflict', + tf.reduce_sum(mask_in_batch_neg)) + return tf.concat([ + logits[:, :batch_size] - mask_in_batch_neg * 1e32, + logits[:, batch_size:]], + axis=1) # yapf: disable else: return logits @@ -73,7 +77,8 @@ def _list_wise_sim(self, user_emb, item_emb): noclk_size = tf.shape(hard_neg_indices)[0] # pos_item_emb, neg_item_emb, hard_neg_item_emb = tf.split( # item_emb, [batch_size, -1, noclk_size], axis=0) - simple_item_emb, hard_neg_item_emb = tf.split(item_emb, [-1, noclk_size], axis=0) + simple_item_emb, hard_neg_item_emb = tf.split( + item_emb, [-1, noclk_size], axis=0) else: # pos_item_emb = item_emb[:batch_size] # neg_item_emb = item_emb[batch_size:] @@ -86,7 +91,8 @@ def _list_wise_sim(self, user_emb, item_emb): _mode = os.environ['tf.estimator.mode'] if _mode == tf.estimator.ModeKeys.PREDICT: - simple_user_item_sim = tf.reduce_sum(tf.multiply(user_emb, simple_item_emb), axis=1, keep_dims=True) + simple_user_item_sim = tf.reduce_sum( + tf.multiply(user_emb, simple_item_emb), axis=1, keep_dims=True) else: simple_user_item_sim = tf.matmul(user_emb, tf.transpose(simple_item_emb)) @@ -94,15 +100,16 @@ def _list_wise_sim(self, user_emb, item_emb): return simple_user_item_sim else: user_emb_expand = tf.gather(user_emb, hard_neg_indices[:, 0]) - hard_neg_user_item_sim = tf.reduce_sum(tf.multiply(user_emb_expand, hard_neg_item_emb), axis=1) + hard_neg_user_item_sim = tf.reduce_sum( + tf.multiply(user_emb_expand, hard_neg_item_emb), axis=1) max_num_neg = tf.reduce_max(hard_neg_indices[:, 1]) + 1 hard_neg_shape = tf.stack([tf.to_int64(batch_size), max_num_neg]) - hard_neg_sim = tf.scatter_nd(hard_neg_indices, hard_neg_user_item_sim, hard_neg_shape) + hard_neg_sim = tf.scatter_nd(hard_neg_indices, hard_neg_user_item_sim, + hard_neg_shape) hard_neg_mask = tf.scatter_nd( - hard_neg_indices, - tf.ones_like(hard_neg_user_item_sim, dtype=tf.float32), - shape=hard_neg_shape, - ) + hard_neg_indices, + tf.ones_like(hard_neg_user_item_sim, dtype=tf.float32), + shape=hard_neg_shape) # set tail positions to -1e32, so that after exp(x), will be zero hard_neg_user_item_sim = hard_neg_sim - (1 - hard_neg_mask) * 1e32 @@ -114,7 +121,8 @@ def _list_wise_sim(self, user_emb, item_emb): return tf.concat([simple_user_item_sim, hard_neg_user_item_sim], axis=1) def _point_wise_sim(self, user_emb, item_emb): - user_item_sim = tf.reduce_sum(tf.multiply(user_emb, item_emb), axis=1, keep_dims=True) + user_item_sim = tf.reduce_sum( + tf.multiply(user_emb, item_emb), axis=1, keep_dims=True) return user_item_sim def sim(self, user_emb, item_emb): @@ -136,7 +144,9 @@ def norm(self, fea): def build_predict_graph(self): if not self.has_backbone: - raise NotImplementedError('method `build_predict_graph` must be implemented when you donot use backbone network') + raise NotImplementedError( + 'method `build_predict_graph` must be implemented when you donot use backbone network' + ) model = self._model_config.WhichOneof('model') assert model == 'model_params', '`model_params` must be configured' model_params = self._model_config.model_params @@ -158,8 +168,16 @@ def build_predict_graph(self): user_item_sim = self.sim(user_tower_emb, item_tower_emb) / temperature if model_params.scale_simi: - sim_w = tf.get_variable('sim_w', dtype=tf.float32, shape=(1), initializer=tf.ones_initializer()) - sim_b = tf.get_variable('sim_b', dtype=tf.float32, shape=(1), initializer=tf.zeros_initializer()) + sim_w = tf.get_variable( + 'sim_w', + dtype=tf.float32, + shape=(1), + initializer=tf.ones_initializer()) + sim_b = tf.get_variable( + 'sim_b', + dtype=tf.float32, + shape=(1), + initializer=tf.zeros_initializer()) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -179,8 +197,10 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb - self._prediction_dict['user_emb'] = tf.reduce_join(tf.as_string(user_tower_emb), axis=-1, separator=',') - self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_tower_emb), axis=-1, separator=',') + self._prediction_dict['user_emb'] = tf.reduce_join( + tf.as_string(user_tower_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join( + tf.as_string(item_tower_emb), axis=-1, separator=',') return self._prediction_dict @@ -195,12 +215,13 @@ def _build_list_wise_loss_graph(self): batch_size = tf.shape(self._prediction_dict['probs'])[0] indices = tf.range(batch_size) indices = tf.concat([indices[:, None], indices[:, None]], axis=1) - hit_prob = tf.gather_nd(self._prediction_dict['probs'][:batch_size, :batch_size], indices) + hit_prob = tf.gather_nd( + self._prediction_dict['probs'][:batch_size, :batch_size], indices) sample_weights = tf.cast(tf.squeeze(self._sample_weight), tf.float32) self._loss_dict['cross_entropy_loss'] = -tf.reduce_mean( - tf.log(hit_prob + 1e-12) * sample_weights - ) / tf.reduce_mean(sample_weights) + tf.log(hit_prob + 1e-12) * + sample_weights) / tf.reduce_mean(sample_weights) logging.info('softmax cross entropy loss is used') @@ -209,36 +230,24 @@ def _build_list_wise_loss_graph(self): pos_simi = tf.reduce_sum(user_features * pos_item_emb, axis=1) # if pos_simi < 0, produce loss reg_pos_loss = tf.nn.relu(-pos_simi) - self._loss_dict['reg_pos_loss'] = tf.reduce_mean(reg_pos_loss * sample_weights) / tf.reduce_mean(sample_weights) + self._loss_dict['reg_pos_loss'] = tf.reduce_mean( + reg_pos_loss * sample_weights) / tf.reduce_mean(sample_weights) # the AMM loss for DAT model - if all( - [ - k in self._prediction_dict.keys() - for k in [ - 'augmented_p_u', - 'augmented_p_i', - 'augmented_a_u', - 'augmented_a_i', - ] - ] - ): - self._loss_dict['amm_loss_u'] = ( - self._model_config.amm_u_weight - * tf.reduce_mean( - tf.square(self._prediction_dict['augmented_a_u'] - self._prediction_dict['augmented_p_i'][:batch_size]) - * sample_weights - ) - / tf.reduce_mean(sample_weights) - ) - self._loss_dict['amm_loss_i'] = ( - self._model_config.amm_i_weight - * tf.reduce_mean( - tf.square(self._prediction_dict['augmented_a_i'][:batch_size] - self._prediction_dict['augmented_p_u']) - * sample_weights - ) - / tf.reduce_mean(sample_weights) - ) + if all([ + k in self._prediction_dict.keys() for k in + ['augmented_p_u', 'augmented_p_i', 'augmented_a_u', 'augmented_a_i'] + ]): + self._loss_dict[ + 'amm_loss_u'] = self._model_config.amm_u_weight * tf.reduce_mean( + tf.square(self._prediction_dict['augmented_a_u'] - + self._prediction_dict['augmented_p_i'][:batch_size]) * + sample_weights) / tf.reduce_mean(sample_weights) + self._loss_dict[ + 'amm_loss_i'] = self._model_config.amm_i_weight * tf.reduce_mean( + tf.square(self._prediction_dict['augmented_a_i'][:batch_size] - + self._prediction_dict['augmented_p_u']) * + sample_weights) / tf.reduce_mean(sample_weights) else: raise ValueError('invalid loss type: %s' % str(self._loss_type)) @@ -257,15 +266,15 @@ def _build_point_wise_loss_graph(self): kwargs = {'loss_name': loss_name} self._loss_dict[loss_name] = loss_builder.build( - self._loss_type, - label=label, - pred=pred, - loss_weight=self._sample_weight, - **kwargs, - ) + self._loss_type, + label=label, + pred=pred, + loss_weight=self._sample_weight, + **kwargs) # build kd loss - kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, self._labels, self._feature_dict) + kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, + self._labels, self._feature_dict) self._loss_dict.update(kd_loss_dict) return self._loss_dict @@ -277,7 +286,6 @@ def build_metric_graph(self, eval_config): def _build_list_wise_metric_graph(self, eval_config): from easy_rec.python.core.easyrec_metrics import metrics_tf - logits = self._prediction_dict['logits'] # label = tf.zeros_like(logits[:, :1], dtype=tf.int64) batch_size = tf.shape(logits)[0] @@ -289,63 +297,59 @@ def _build_list_wise_metric_graph(self, eval_config): metric_dict = {} for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'recall_at_topk': - metric_dict['recall@%d' % metric.recall_at_topk.topk] = metrics_tf.recall_at_k( - label, logits, metric.recall_at_topk.topk - ) + metric_dict['recall@%d' % + metric.recall_at_topk.topk] = metrics_tf.recall_at_k( + label, logits, metric.recall_at_topk.topk) - logits_v2 = tf.concat([pos_item_sim[:, None], logits[:, batch_size:]], axis=1) + logits_v2 = tf.concat([pos_item_sim[:, None], logits[:, batch_size:]], + axis=1) labels_v2 = tf.zeros_like(logits_v2[:, :1], dtype=tf.int64) - metric_dict['recall_neg_sam@%d' % metric.recall_at_topk.topk] = metrics_tf.recall_at_k( - labels_v2, logits_v2, metric.recall_at_topk.topk - ) - - metric_dict['recall_in_batch@%d' % metric.recall_at_topk.topk] = metrics_tf.recall_at_k( - label, logits[:, :batch_size], metric.recall_at_topk.topk - ) + metric_dict['recall_neg_sam@%d' % + metric.recall_at_topk.topk] = metrics_tf.recall_at_k( + labels_v2, logits_v2, metric.recall_at_topk.topk) + + metric_dict['recall_in_batch@%d' % + metric.recall_at_topk.topk] = metrics_tf.recall_at_k( + label, logits[:, :batch_size], + metric.recall_at_topk.topk) else: raise ValueError('invalid metric type: %s' % str(metric)) return metric_dict def _build_point_wise_metric_graph(self, eval_config): from easy_rec.python.core.easyrec_metrics import metrics_tf - metric_dict = {} label = list(self._labels.values())[0] for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'auc': assert self._loss_type == LossType.CLASSIFICATION - metric_dict['auc'] = metrics_tf.auc(label, self._prediction_dict['probs']) + metric_dict['auc'] = metrics_tf.auc(label, + self._prediction_dict['probs']) elif metric.WhichOneof('metric') == 'mean_absolute_error': assert self._loss_type == LossType.L2_LOSS metric_dict['mean_absolute_error'] = metrics_tf.mean_absolute_error( - tf.to_float(label), self._prediction_dict['y'] - ) + tf.to_float(label), self._prediction_dict['y']) else: raise ValueError('invalid metric type: %s' % str(metric)) return metric_dict def get_outputs(self): if not self.has_backbone: - raise NotImplementedError('could not call get_outputs on abstract class MatchModel') + raise NotImplementedError( + 'could not call get_outputs on abstract class MatchModel') if self._loss_type == LossType.CLASSIFICATION: return [ - 'logits', - 'probs', - 'user_emb', - 'item_emb', - 'user_tower_emb', - 'item_tower_emb', + 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', + 'item_tower_emb' ] elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: - self._prediction_dict['logits'] = tf.squeeze(self._prediction_dict['logits'], axis=-1) - self._prediction_dict['probs'] = tf.nn.sigmoid(self._prediction_dict['logits']) + self._prediction_dict['logits'] = tf.squeeze( + self._prediction_dict['logits'], axis=-1) + self._prediction_dict['probs'] = tf.nn.sigmoid( + self._prediction_dict['logits']) return [ - 'logits', - 'probs', - 'user_emb', - 'item_emb', - 'user_tower_emb', - 'item_tower_emb', + 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', + 'item_tower_emb' ] elif self._loss_type == LossType.L2_LOSS: return ['y', 'user_emb', 'item_emb', 'user_tower_emb', 'item_tower_emb'] diff --git a/easy_rec/python/model/mind.py b/easy_rec/python/model/mind.py index 8b43ca8b1..747f829c4 100644 --- a/easy_rec/python/model/mind.py +++ b/easy_rec/python/model/mind.py @@ -19,14 +19,23 @@ class MIND(MatchModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(MIND, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'mind', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model' - ) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(MIND, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert self._model_config.WhichOneof( + 'model' + ) == 'mind', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.mind - self._hist_seq_features, _, _ = self._input_layer(self._feature_dict, 'hist', is_combine=False) + self._hist_seq_features, _, _ = self._input_layer( + self._feature_dict, 'hist', is_combine=False) self._user_features, _ = self._input_layer(self._feature_dict, 'user') self._item_features, _ = self._input_layer(self._feature_dict, 'item') @@ -37,20 +46,31 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai # copy obj so that any modification will not affect original config self.concat_dnn = copy_obj(self._model_config.concat_dnn) - self._l2_reg = regularizers.l2_regularizer(self._model_config.l2_regularization) + self._l2_reg = regularizers.l2_regularizer( + self._model_config.l2_regularization) def build_predict_graph(self): - capsule_layer = CapsuleLayer(self._model_config.capsule_config, self._is_training) + capsule_layer = CapsuleLayer(self._model_config.capsule_config, + self._is_training) if self._model_config.time_id_fea: - time_id_fea = [x[0] for x in self._hist_seq_features if self._model_config.time_id_fea in x[0].name] - logging.info('time_id_fea is set(%s), find num: %d' % (self._model_config.time_id_fea, len(time_id_fea))) + time_id_fea = [ + x[0] + for x in self._hist_seq_features + if self._model_config.time_id_fea in x[0].name + ] + logging.info('time_id_fea is set(%s), find num: %d' % + (self._model_config.time_id_fea, len(time_id_fea))) else: time_id_fea = [] time_id_fea = time_id_fea[0] if len(time_id_fea) > 0 else None if time_id_fea is not None: - hist_seq_feas = [x[0] for x in self._hist_seq_features if self._model_config.time_id_fea not in x[0].name] + hist_seq_feas = [ + x[0] + for x in self._hist_seq_features + if self._model_config.time_id_fea not in x[0].name + ] else: hist_seq_feas = [x[0] for x in self._hist_seq_features] @@ -62,23 +82,25 @@ def build_predict_graph(self): hist_embed_dims = [x.get_shape()[-1] for x in hist_seq_feas] for i in range(1, len(hist_embed_dims)): assert hist_embed_dims[i] == hist_embed_dims[0], ( - 'all hist seq must have the same embedding shape, but: %s' % str(hist_embed_dims) - ) + 'all hist seq must have the same embedding shape, but: %s' % + str(hist_embed_dims)) hist_seq_feas = tf.add_n(hist_seq_feas) / len(hist_seq_feas) else: hist_seq_feas = tf.concat(hist_seq_feas, axis=2) - if self._model_config.HasField('pre_capsule_dnn') and len(self._model_config.pre_capsule_dnn.hidden_units) > 0: + if self._model_config.HasField('pre_capsule_dnn') and len( + self._model_config.pre_capsule_dnn.hidden_units) > 0: pre_dnn_layer = dnn.DNN( - self._model_config.pre_capsule_dnn, - self._l2_reg, - 'pre_capsule_dnn', - self._is_training, + self._model_config.pre_capsule_dnn, + self._l2_reg, + 'pre_capsule_dnn', + self._is_training, ) hist_seq_feas = pre_dnn_layer(hist_seq_feas) if time_id_fea is not None: - assert time_id_fea.get_shape()[-1] == 1, 'time_id must have only embedding_size of 1' + assert time_id_fea.get_shape( + )[-1] == 1, 'time_id must have only embedding_size of 1' time_id_mask = tf.sequence_mask(hist_seq_len, tf.shape(time_id_fea)[1]) time_id_mask = (tf.cast(time_id_mask, tf.float32) * 2 - 1) * 1e32 time_id_fea = tf.minimum(time_id_fea, time_id_mask[:, :, None]) @@ -87,7 +109,8 @@ def build_predict_graph(self): tf.summary.histogram('hist_seq_len', hist_seq_len) # batch_size x max_k x high_capsule_dim - high_capsules, num_high_capsules = capsule_layer(hist_seq_feas, hist_seq_len) + high_capsules, num_high_capsules = capsule_layer(hist_seq_feas, + hist_seq_len) tf.summary.histogram('num_high_capsules', num_high_capsules) @@ -96,49 +119,56 @@ def build_predict_graph(self): # trainable=True, name='capsule_bn') # high_capsules = high_capsules * 0.1 - tf.summary.scalar('high_capsules_norm', tf.reduce_mean(tf.norm(high_capsules, axis=-1))) - tf.summary.scalar('num_high_capsules', tf.reduce_mean(tf.to_float(num_high_capsules))) + tf.summary.scalar('high_capsules_norm', + tf.reduce_mean(tf.norm(high_capsules, axis=-1))) + tf.summary.scalar('num_high_capsules', + tf.reduce_mean(tf.to_float(num_high_capsules))) user_features = tf.layers.batch_normalization( - self._user_features, - training=self._is_training, - trainable=True, - name='user_fea_bn', + self._user_features, + training=self._is_training, + trainable=True, + name='user_fea_bn', ) - user_dnn = dnn.DNN(self.user_dnn, self._l2_reg, 'user_dnn', self._is_training) + user_dnn = dnn.DNN(self.user_dnn, self._l2_reg, 'user_dnn', + self._is_training) user_features = user_dnn(user_features) - tf.summary.scalar('user_features_norm', tf.reduce_mean(tf.norm(self._user_features, axis=-1))) + tf.summary.scalar('user_features_norm', + tf.reduce_mean(tf.norm(self._user_features, axis=-1))) # concatenate with user features - user_features_tile = tf.tile(user_features[:, None, :], [1, tf.shape(high_capsules)[1], 1]) + user_features_tile = tf.tile(user_features[:, None, :], + [1, tf.shape(high_capsules)[1], 1]) user_interests = tf.concat([high_capsules, user_features_tile], axis=2) num_concat_dnn_layer = len(self.concat_dnn.hidden_units) last_hidden = self.concat_dnn.hidden_units.pop() - concat_dnn = dnn.DNN(self.concat_dnn, self._l2_reg, 'concat_dnn', self._is_training) + concat_dnn = dnn.DNN(self.concat_dnn, self._l2_reg, 'concat_dnn', + self._is_training) user_interests = concat_dnn(user_interests) user_interests = tf.layers.dense( - inputs=user_interests, - units=last_hidden, - kernel_regularizer=self._l2_reg, - name='concat_dnn/dnn_%d' % (num_concat_dnn_layer - 1), + inputs=user_interests, + units=last_hidden, + kernel_regularizer=self._l2_reg, + name='concat_dnn/dnn_%d' % (num_concat_dnn_layer - 1), ) num_item_dnn_layer = len(self.item_dnn.hidden_units) last_item_hidden = self.item_dnn.hidden_units.pop() - item_dnn = dnn.DNN(self.item_dnn, self._l2_reg, 'item_dnn', self._is_training) + item_dnn = dnn.DNN(self.item_dnn, self._l2_reg, 'item_dnn', + self._is_training) item_tower_emb = item_dnn(self._item_features) item_tower_emb = tf.layers.dense( - inputs=item_tower_emb, - units=last_item_hidden, - kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), + inputs=item_tower_emb, + units=last_item_hidden, + kernel_regularizer=self._l2_reg, + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), ) assert self._model_config.simi_func in [ - Similarity.COSINE, - Similarity.INNER_PRODUCT, + Similarity.COSINE, + Similarity.INNER_PRODUCT, ] if self._model_config.simi_func == Similarity.COSINE: @@ -150,13 +180,16 @@ def build_predict_graph(self): batch_size = tf.shape(user_interests)[0] pos_item_fea = item_tower_emb[:batch_size] simi = tf.einsum('bhe,be->bh', user_interests, pos_item_fea) - tf.summary.histogram('interest_item_simi/pre_scale', tf.reduce_max(simi, axis=1)) + tf.summary.histogram('interest_item_simi/pre_scale', + tf.reduce_max(simi, axis=1)) # simi = tf.Print(simi, [tf.reduce_max(simi, axis=1), tf.reduce_min(simi, axis=1)], message='simi_max_0') # simi = tf.pow(simi, self._model_config.simi_pow) simi = simi * self._model_config.simi_pow - tf.summary.histogram('interest_item_simi/scaled', tf.reduce_max(simi, axis=1)) + tf.summary.histogram('interest_item_simi/scaled', + tf.reduce_max(simi, axis=1)) # simi = tf.Print(simi, [tf.reduce_max(simi, axis=1), tf.reduce_min(simi, axis=1)], message='simi_max') - simi_mask = tf.sequence_mask(num_high_capsules, self._model_config.capsule_config.max_k) + simi_mask = tf.sequence_mask(num_high_capsules, + self._model_config.capsule_config.max_k) user_interests = user_interests * tf.to_float(simi_mask[:, :, None]) self._prediction_dict['user_interests'] = user_interests @@ -164,13 +197,13 @@ def build_predict_graph(self): max_thresh = (tf.cast(simi_mask, tf.float32) * 2 - 1) * 1e32 simi = tf.minimum(simi, max_thresh) simi = tf.nn.softmax(simi, axis=1) - tf.summary.histogram('interest_item_simi/softmax', tf.reduce_max(simi, axis=1)) + tf.summary.histogram('interest_item_simi/softmax', + tf.reduce_max(simi, axis=1)) if self._model_config.simi_pow >= 100: logging.info( - 'simi_pow=%d, will change to argmax, only use the most similar interests for calculate loss.' - % self._model_config.simi_pow - ) + 'simi_pow=%d, will change to argmax, only use the most similar interests for calculate loss.' + % self._model_config.simi_pow) simi_max_id = tf.argmax(simi, axis=1) simi = tf.one_hot(simi_max_id, tf.shape(simi)[1], dtype=tf.float32) @@ -179,8 +212,16 @@ def build_predict_graph(self): # calculate similarity between user_tower_emb and item_tower_emb user_item_sim = self.sim(user_tower_emb, item_tower_emb) if self._model_config.scale_simi: - sim_w = tf.get_variable('sim_w', dtype=tf.float32, shape=(1), initializer=tf.ones_initializer()) - sim_b = tf.get_variable('sim_b', dtype=tf.float32, shape=(1), initializer=tf.zeros_initializer()) + sim_w = tf.get_variable( + 'sim_w', + dtype=tf.float32, + shape=(1), + initializer=tf.ones_initializer()) + sim_b = tf.get_variable( + 'sim_b', + dtype=tf.float32, + shape=(1), + initializer=tf.zeros_initializer()) y_pred = user_item_sim * tf.abs(sim_w) + sim_b else: y_pred = user_item_sim @@ -203,12 +244,13 @@ def build_predict_graph(self): self._prediction_dict['user_tower_emb'] = user_tower_emb self._prediction_dict['item_tower_emb'] = item_tower_emb self._prediction_dict['user_emb'] = tf.reduce_join( - tf.reduce_join(tf.as_string(user_interests), axis=-1, separator=','), - axis=-1, - separator='|', + tf.reduce_join(tf.as_string(user_interests), axis=-1, separator=','), + axis=-1, + separator='|', ) self._prediction_dict['user_emb_num'] = num_high_capsules - self._prediction_dict['item_emb'] = tf.reduce_join(tf.as_string(item_tower_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join( + tf.as_string(item_tower_emb), axis=-1, separator=',') if self._labels is not None: # for summary purpose @@ -222,13 +264,14 @@ def build_loss_graph(self): loss_dict = super(MIND, self).build_loss_graph() if self._model_config.max_interests_simi < 1.0: loss_dict['reg_interest_simi'] = tf.nn.relu( - self._prediction_dict['interests_simi'] - self._model_config.max_interests_simi - ) + self._prediction_dict['interests_simi'] - + self._model_config.max_interests_simi) return loss_dict def _build_interest_simi(self): user_emb_num = self._prediction_dict['user_emb_num'] - high_capsule_mask = tf.sequence_mask(user_emb_num, self._model_config.capsule_config.max_k) + high_capsule_mask = tf.sequence_mask( + user_emb_num, self._model_config.capsule_config.max_k) user_interests = self._prediction_dict['user_interests'] high_capsule_mask = tf.to_float(high_capsule_mask[:, :, None]) @@ -245,18 +288,21 @@ def _build_interest_simi(self): high_capsule_simi = high_capsule_sum_sqr - high_capsule_sqr_sum # normalize by interest number - interest_div = tf.maximum(tf.to_float(user_emb_num * (user_emb_num - 1)), 1.0) + interest_div = tf.maximum( + tf.to_float(user_emb_num * (user_emb_num - 1)), 1.0) interest_simi = tf.reduce_sum(interest_simi, axis=1) / interest_div high_capsule_simi = tf.reduce_sum(high_capsule_simi, axis=1) / interest_div # normalize by batch_size multi_interest = tf.to_float(user_emb_num > 1) - sum_interest_simi = tf.reduce_sum((interest_simi + 1) * multi_interest) / 2.0 + sum_interest_simi = tf.reduce_sum( + (interest_simi + 1) * multi_interest) / 2.0 sum_div = tf.maximum(tf.reduce_sum(multi_interest), 1.0) avg_interest_simi = sum_interest_simi / sum_div - sum_capsule_simi = tf.reduce_sum((high_capsule_simi + 1) * multi_interest) / 2.0 + sum_capsule_simi = tf.reduce_sum( + (high_capsule_simi + 1) * multi_interest) / 2.0 avg_capsule_simi = sum_capsule_simi / sum_div tf.summary.scalar('interest_similarity', avg_interest_simi) @@ -269,8 +315,8 @@ def build_metric_graph(self, eval_config): # build interest metric interest_simi, capsule_simi = self._build_interest_simi() metric_dict = { - 'interest_similarity': metrics.mean(interest_simi), - 'capsule_similarity': metrics.mean(capsule_simi), + 'interest_similarity': metrics.mean(interest_simi), + 'capsule_similarity': metrics.mean(capsule_simi), } if self._is_point_wise: metric_dict.update(self._build_point_wise_metric_graph(eval_config)) @@ -280,8 +326,8 @@ def build_metric_graph(self, eval_config): for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'recall_at_topk': assert self._loss_type in [ - LossType.CLASSIFICATION, - LossType.SOFTMAX_CROSS_ENTROPY, + LossType.CLASSIFICATION, + LossType.SOFTMAX_CROSS_ENTROPY, ] if metric.recall_at_topk.topk not in recall_at_topks: recall_at_topks.append(metric.recall_at_topk.topk) @@ -299,7 +345,8 @@ def build_metric_graph(self, eval_config): if hard_neg_indices is not None: logging.info('With hard negative examples') noclk_size = tf.shape(hard_neg_indices)[0] - simple_item_emb, hard_neg_item_emb = tf.split(item_tower_emb, [-1, noclk_size], axis=0) + simple_item_emb, hard_neg_item_emb = tf.split( + item_tower_emb, [-1, noclk_size], axis=0) else: simple_item_emb = item_tower_emb hard_neg_item_emb = None @@ -312,115 +359,125 @@ def build_metric_graph(self, eval_config): # labels = tf.zeros_like(logits[:, :1], dtype=tf.int64) pos_indices = tf.range(batch_size) - pos_indices = tf.concat([pos_indices[:, None], pos_indices[:, None]], axis=1) - pos_item_sim = tf.gather_nd(simple_item_sim[:batch_size, :batch_size], pos_indices) + pos_indices = tf.concat([pos_indices[:, None], pos_indices[:, None]], + axis=1) + pos_item_sim = tf.gather_nd(simple_item_sim[:batch_size, :batch_size], + pos_indices) - simple_item_sim_v2 = tf.concat([pos_item_sim[:, None], simple_item_sim[:, batch_size:]], axis=1) + simple_item_sim_v2 = tf.concat( + [pos_item_sim[:, None], simple_item_sim[:, batch_size:]], axis=1) simple_lbls_v2 = tf.zeros_like(simple_item_sim_v2[:, :1], dtype=tf.int64) for topk in recall_at_topks: metric_dict['interests_recall@%d' % topk] = metrics.recall_at_k( - labels=simple_lbls, - predictions=simple_item_sim, - k=topk, - name='interests_recall_at_%d' % topk, + labels=simple_lbls, + predictions=simple_item_sim, + k=topk, + name='interests_recall_at_%d' % topk, ) metric_dict['interests_neg_sam_recall@%d' % topk] = metrics.recall_at_k( - labels=simple_lbls_v2, - predictions=simple_item_sim_v2, - k=topk, - name='interests_recall_neg_sam_at_%d' % topk, + labels=simple_lbls_v2, + predictions=simple_item_sim_v2, + k=topk, + name='interests_recall_neg_sam_at_%d' % topk, ) logits = self._prediction_dict['logits'] - pos_item_logits = tf.gather_nd(logits[:batch_size, :batch_size], pos_indices) - logits_v2 = tf.concat([pos_item_logits[:, None], logits[:, batch_size:]], axis=1) + pos_item_logits = tf.gather_nd(logits[:batch_size, :batch_size], + pos_indices) + logits_v2 = tf.concat([pos_item_logits[:, None], logits[:, batch_size:]], + axis=1) labels_v2 = tf.zeros_like(logits_v2[:, :1], dtype=tf.int64) for topk in recall_at_topks: metric_dict['recall@%d' % topk] = metrics.recall_at_k( - labels=simple_lbls, - predictions=logits, - k=topk, - name='recall_at_%d' % topk, + labels=simple_lbls, + predictions=logits, + k=topk, + name='recall_at_%d' % topk, ) metric_dict['recall_neg_sam@%d' % topk] = metrics.recall_at_k( - labels=labels_v2, - predictions=logits_v2, - k=topk, - name='recall_neg_sam_at_%d' % topk, + labels=labels_v2, + predictions=logits_v2, + k=topk, + name='recall_neg_sam_at_%d' % topk, ) eval_logits = logits[:, :batch_size] eval_logits = tf.cond( - batch_size < topk, - lambda: tf.pad( - eval_logits, - [[0, 0], [0, topk - batch_size]], - mode='CONSTANT', - constant_values=-1e32, - name='pad_eval_logits', - ), - lambda: eval_logits, + batch_size < topk, + lambda: tf.pad( + eval_logits, + [[0, 0], [0, topk - batch_size]], + mode='CONSTANT', + constant_values=-1e32, + name='pad_eval_logits', + ), + lambda: eval_logits, ) metric_dict['recall_in_batch@%d' % topk] = metrics.recall_at_k( - labels=simple_lbls, - predictions=eval_logits, - k=topk, - name='recall_in_batch_at_%d' % topk, + labels=simple_lbls, + predictions=eval_logits, + k=topk, + name='recall_in_batch_at_%d' % topk, ) # batch_size num_interest if hard_neg_indices is not None: hard_neg_user_emb = tf.gather(user_interests, hard_neg_indices[:, 0]) - hard_neg_sim = tf.einsum('nhe,ne->nh', hard_neg_user_emb, hard_neg_item_emb) + hard_neg_sim = tf.einsum('nhe,ne->nh', hard_neg_user_emb, + hard_neg_item_emb) hard_neg_sim = tf.reduce_max(hard_neg_sim, axis=1) max_num_neg = tf.reduce_max(hard_neg_indices[:, 1]) + 1 hard_neg_shape = tf.stack([tf.to_int64(batch_size), max_num_neg]) hard_neg_mask = tf.scatter_nd( - hard_neg_indices, - tf.ones_like(hard_neg_sim, dtype=tf.float32), - shape=hard_neg_shape, + hard_neg_indices, + tf.ones_like(hard_neg_sim, dtype=tf.float32), + shape=hard_neg_shape, ) - hard_neg_sim = tf.scatter_nd(hard_neg_indices, hard_neg_sim, hard_neg_shape) + hard_neg_sim = tf.scatter_nd(hard_neg_indices, hard_neg_sim, + hard_neg_shape) hard_neg_sim = hard_neg_sim - (1 - hard_neg_mask) * 1e32 hard_logits = tf.concat([pos_item_logits[:, None], hard_neg_sim], axis=1) hard_lbls = tf.zeros_like(hard_logits[:, :1], dtype=tf.int64) - metric_dict['hard_neg_acc'] = metrics.accuracy(hard_lbls, tf.argmax(hard_logits, axis=1)) + metric_dict['hard_neg_acc'] = metrics.accuracy( + hard_lbls, tf.argmax(hard_logits, axis=1)) return metric_dict def get_outputs(self): if self._loss_type == LossType.CLASSIFICATION: return [ - 'logits', - 'probs', - 'user_emb', - 'item_emb', - 'user_emb_num', - 'user_interests', - 'item_tower_emb', + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_emb_num', + 'user_interests', + 'item_tower_emb', ] elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: - self._prediction_dict['logits'] = tf.squeeze(self._prediction_dict['logits'], axis=-1) - self._prediction_dict['probs'] = tf.nn.sigmoid(self._prediction_dict['logits']) + self._prediction_dict['logits'] = tf.squeeze( + self._prediction_dict['logits'], axis=-1) + self._prediction_dict['probs'] = tf.nn.sigmoid( + self._prediction_dict['logits']) return [ - 'logits', - 'probs', - 'user_emb', - 'item_emb', - 'user_emb_num', - 'user_interests', - 'item_tower_emb', + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_emb_num', + 'user_interests', + 'item_tower_emb', ] elif self._loss_type == LossType.L2_LOSS: return [ - 'y', - 'user_emb', - 'item_emb', - 'user_emb_num', - 'user_interests', - 'item_tower_emb', + 'y', + 'user_emb', + 'item_emb', + 'user_emb_num', + 'user_interests', + 'item_tower_emb', ] else: raise ValueError('invalid loss type: %s' % str(self._loss_type)) diff --git a/easy_rec/python/model/mmoe.py b/easy_rec/python/model/mmoe.py index 66e8185d6..23a8a6643 100644 --- a/easy_rec/python/model/mmoe.py +++ b/easy_rec/python/model/mmoe.py @@ -2,7 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import dnn, mmoe +from easy_rec.python.layers import dnn +from easy_rec.python.layers import mmoe from easy_rec.python.model.multi_task_model import MultiTaskModel from easy_rec.python.protos.mmoe_pb2 import MMoE as MMoEConfig @@ -11,11 +12,19 @@ class MMoE(MultiTaskModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(MMoE, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'mmoe', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model' - ) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(MMoE, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert self._model_config.WhichOneof( + 'model' + ) == 'mmoe', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.mmoe assert isinstance(self._model_config, MMoEConfig) @@ -28,17 +37,17 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai def build_predict_graph(self): if self._model_config.HasField('expert_dnn'): mmoe_layer = mmoe.MMOE( - self._model_config.expert_dnn, - l2_reg=self._l2_reg, - num_task=self._task_num, - num_expert=self._model_config.num_expert, + self._model_config.expert_dnn, + l2_reg=self._l2_reg, + num_task=self._task_num, + num_expert=self._model_config.num_expert, ) else: # For backward compatibility with original mmoe layer config mmoe_layer = mmoe.MMOE( - [x.dnn for x in self._model_config.experts], - l2_reg=self._l2_reg, - num_task=self._task_num, + [x.dnn for x in self._model_config.experts], + l2_reg=self._l2_reg, + num_task=self._task_num, ) task_input_list = mmoe_layer(self._features) @@ -48,19 +57,19 @@ def build_predict_graph(self): if task_tower_cfg.HasField('dnn'): tower_dnn = dnn.DNN( - task_tower_cfg.dnn, - self._l2_reg, - name=tower_name, - is_training=self._is_training, + task_tower_cfg.dnn, + self._l2_reg, + name=tower_name, + is_training=self._is_training, ) tower_output = tower_dnn(task_input_list[i]) else: tower_output = task_input_list[i] tower_output = tf.layers.dense( - inputs=tower_output, - units=task_tower_cfg.num_class, - kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i, + inputs=tower_output, + units=task_tower_cfg.num_class, + kernel_regularizer=self._l2_reg, + name='dnn_output_%d' % i, ) tower_outputs[tower_name] = tower_output diff --git a/easy_rec/python/model/multi_task_model.py b/easy_rec/python/model/multi_task_model.py index e09ab9cd1..9fa67e10b 100644 --- a/easy_rec/python/model/multi_task_model.py +++ b/easy_rec/python/model/multi_task_model.py @@ -17,15 +17,24 @@ class MultiTaskModel(RankModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(MultiTaskModel, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(MultiTaskModel, self).__init__(model_config, feature_configs, + features, labels, is_training) self._task_towers = [] self._task_num = None self._label_name_dict = {} def build_predict_graph(self): if not self.has_backbone: - raise NotImplementedError('method `build_predict_graph` must be implemented when backbone network do not exists') + raise NotImplementedError( + 'method `build_predict_graph` must be implemented when backbone network do not exists' + ) model = self._model_config.WhichOneof('model') assert model == 'model_params', '`model_params` must be configured' config = self._model_config.model_params @@ -37,7 +46,8 @@ def build_predict_graph(self): backbone = self.backbone if type(backbone) in (list, tuple): if len(backbone) != len(config.task_towers): - raise ValueError('The number of backbone outputs and task towers must be equal') + raise ValueError( + 'The number of backbone outputs and task towers must be equal') task_input_list = backbone else: task_input_list = [backbone] * len(config.task_towers) @@ -48,10 +58,10 @@ def build_predict_graph(self): with tf.name_scope(tower_name): if task_tower_cfg.HasField('dnn'): tower_dnn = DNN( - task_tower_cfg.dnn, - self._l2_reg, - name=tower_name, - is_training=self._is_training, + task_tower_cfg.dnn, + self._l2_reg, + name=tower_name, + is_training=self._is_training, ) tower_output = tower_dnn(task_input_list[i]) else: @@ -66,25 +76,26 @@ def build_predict_graph(self): with tf.name_scope(tower_name): if task_tower_cfg.HasField('relation_dnn'): relation_dnn = DNN( - task_tower_cfg.relation_dnn, - self._l2_reg, - name=tower_name + '/relation_dnn', - is_training=self._is_training, + task_tower_cfg.relation_dnn, + self._l2_reg, + name=tower_name + '/relation_dnn', + is_training=self._is_training, ) tower_inputs = [tower_features[tower_name]] for relation_tower_name in task_tower_cfg.relation_tower_names: tower_inputs.append(relation_features[relation_tower_name]) - relation_input = tf.concat(tower_inputs, axis=-1, name=tower_name + '/relation_input') + relation_input = tf.concat( + tower_inputs, axis=-1, name=tower_name + '/relation_input') relation_fea = relation_dnn(relation_input) relation_features[tower_name] = relation_fea else: relation_fea = tower_features[tower_name] output_logits = tf.layers.dense( - relation_fea, - task_tower_cfg.num_class, - kernel_regularizer=self._l2_reg, - name=tower_name + '/output', + relation_fea, + task_tower_cfg.num_class, + kernel_regularizer=self._l2_reg, + name=tower_name + '/output', ) tower_outputs[tower_name] = output_logits @@ -97,7 +108,7 @@ def _init_towers(self, task_tower_configs): self._task_num = len(task_tower_configs) for i, task_tower_config in enumerate(task_tower_configs): assert isinstance(task_tower_config, tower_pb2.TaskTower) or isinstance( - task_tower_config, tower_pb2.BayesTaskTower + task_tower_config, tower_pb2.BayesTaskTower ), 'task_tower_config must be a instance of tower_pb2.TaskTower or tower_pb2.BayesTaskTower' tower_name = task_tower_config.tower_name @@ -108,7 +119,8 @@ def _init_towers(self, task_tower_configs): else: # If label name is not specified, task_tower and label will be matched by order label_name = list(self._labels.keys())[i] - logging.info('Task Tower [%s] use label [%s]' % (tower_name, label_name)) + logging.info('Task Tower [%s] use label [%s]' % + (tower_name, label_name)) assert label_name in self._labels, 'label [%s] must exists in labels' % label_name self._label_name_dict[tower_name] = label_name @@ -117,23 +129,21 @@ def _add_to_prediction_dict(self, output): tower_name = task_tower_cfg.tower_name if len(task_tower_cfg.losses) == 0: self._prediction_dict.update( - self._output_to_prediction_impl( - output[tower_name], - loss_type=task_tower_cfg.loss_type, - num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name, - ) - ) + self._output_to_prediction_impl( + output[tower_name], + loss_type=task_tower_cfg.loss_type, + num_class=task_tower_cfg.num_class, + suffix='_%s' % tower_name, + )) else: for loss in task_tower_cfg.losses: self._prediction_dict.update( - self._output_to_prediction_impl( - output[tower_name], - loss_type=loss.loss_type, - num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name, - ) - ) + self._output_to_prediction_impl( + output[tower_name], + loss_type=loss.loss_type, + num_class=task_tower_cfg.num_class, + suffix='_%s' % tower_name, + )) def build_metric_graph(self, eval_config): """Build metric graph for multi task model.""" @@ -144,14 +154,13 @@ def build_metric_graph(self, eval_config): if len(task_tower_cfg.losses) > 0: loss_types = {loss.loss_type for loss in task_tower_cfg.losses} self._metric_dict.update( - self._build_metric_impl( - metric, - loss_type=loss_types, - label_name=self._label_name_dict[tower_name], - num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name, - ) - ) + self._build_metric_impl( + metric, + loss_type=loss_types, + label_name=self._label_name_dict[tower_name], + num_class=task_tower_cfg.num_class, + suffix='_%s' % tower_name, + )) return self._metric_dict def build_loss_weight(self): @@ -162,7 +171,9 @@ def build_loss_weight(self): losses = task_tower_cfg.losses n = len(losses) if n > 0: - loss_weights[tower_name] = [loss.weight * task_tower_cfg.weight for loss in losses] + loss_weights[tower_name] = [ + loss.weight * task_tower_cfg.weight for loss in losses + ] num_loss += n else: loss_weights[tower_name] = [task_tower_cfg.weight] @@ -175,14 +186,15 @@ def build_loss_weight(self): i = 0 for k, v in loss_weights.items(): n = len(v) - loss_weights[k] = weights[i : i + n] + loss_weights[k] = weights[i:i + n] i += n return loss_weights def get_learnt_loss(self, loss_type, name, value): strategy = self._base_model_config.loss_weight_strategy if strategy == self._base_model_config.Uncertainty: - uncertainty = tf.Variable(0, name='%s_loss_weight' % name, dtype=tf.float32) + uncertainty = tf.Variable( + 0, name='%s_loss_weight' % name, dtype=tf.float32) tf.summary.scalar('loss/%s_uncertainty' % name, uncertainty) if loss_type in {LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS}: return 0.5 * tf.exp(-uncertainty) * value + 0.5 * uncertainty @@ -201,37 +213,36 @@ def build_loss_graph(self): if task_tower_cfg.use_sample_weight: loss_weight *= self._sample_weight - if hasattr(task_tower_cfg, 'task_space_indicator_label') and task_tower_cfg.HasField( - 'task_space_indicator_label' - ): - in_task_space = tf.to_float(self._labels[task_tower_cfg.task_space_indicator_label] > 0) + if hasattr(task_tower_cfg, 'task_space_indicator_label' + ) and task_tower_cfg.HasField('task_space_indicator_label'): + in_task_space = tf.to_float( + self._labels[task_tower_cfg.task_space_indicator_label] > 0) loss_weight = loss_weight * ( - task_tower_cfg.in_task_space_weight * in_task_space - + task_tower_cfg.out_task_space_weight * (1 - in_task_space) - ) + task_tower_cfg.in_task_space_weight * in_task_space + + task_tower_cfg.out_task_space_weight * (1 - in_task_space)) - if task_tower_cfg.HasField('task_space_indicator_name') and task_tower_cfg.HasField('task_space_indicator_value'): + if task_tower_cfg.HasField( + 'task_space_indicator_name') and task_tower_cfg.HasField( + 'task_space_indicator_value'): in_task_space = tf.to_float( - tf.equal( - self._feature_dict[task_tower_cfg.task_space_indicator_name], - task_tower_cfg.task_space_indicator_value, - ) - ) + tf.equal( + self._feature_dict[task_tower_cfg.task_space_indicator_name], + task_tower_cfg.task_space_indicator_value, + )) loss_weight = loss_weight * ( - task_tower_cfg.in_task_space_weight * in_task_space - + task_tower_cfg.out_task_space_weight * (1 - in_task_space) - ) + task_tower_cfg.in_task_space_weight * in_task_space + + task_tower_cfg.out_task_space_weight * (1 - in_task_space)) task_loss_weight = task_loss_weights[tower_name] loss_dict = {} losses = task_tower_cfg.losses if len(losses) == 0: loss_dict = self._build_loss_impl( - task_tower_cfg.loss_type, - label_name=self._label_name_dict[tower_name], - loss_weight=loss_weight, - num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name, + task_tower_cfg.loss_type, + label_name=self._label_name_dict[tower_name], + loss_weight=loss_weight, + num_class=task_tower_cfg.num_class, + suffix='_%s' % tower_name, ) for loss_name in loss_dict.keys(): loss_dict[loss_name] = loss_dict[loss_name] * task_loss_weight[0] @@ -244,24 +255,26 @@ def build_loss_graph(self): y_rt = self._prediction_dict['probs_%s' % relation_tower_name] cali_loss = tf.reduce_mean(tf.nn.relu(y_t - y_rt)) calibrate_loss.append(cali_loss * loss.weight) - logging.info('calibrate loss: %s -> %s' % (relation_tower_name, tower_name)) + logging.info('calibrate loss: %s -> %s' % + (relation_tower_name, tower_name)) continue loss_param = loss.WhichOneof('loss_param') if loss_param is not None: loss_param = getattr(loss, loss_param) loss_ops = self._build_loss_impl( - loss.loss_type, - label_name=self._label_name_dict[tower_name], - loss_weight=loss_weight, - num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name, - loss_name=loss.loss_name, - loss_param=loss_param, + loss.loss_type, + label_name=self._label_name_dict[tower_name], + loss_weight=loss_weight, + num_class=task_tower_cfg.num_class, + suffix='_%s' % tower_name, + loss_name=loss.loss_name, + loss_param=loss_param, ) for i, loss_name in enumerate(loss_ops): loss_value = loss_ops[loss_name] if loss.learn_loss_weight: - loss_dict[loss_name] = self.get_learnt_loss(loss.loss_type, loss_name, loss_value) + loss_dict[loss_name] = self.get_learnt_loss( + loss.loss_type, loss_name, loss_value) else: loss_dict[loss_name] = loss_value * task_loss_weight[i] if calibrate_loss: @@ -270,7 +283,8 @@ def build_loss_graph(self): tf.summary.scalar('loss/order_calibrate_loss', cali_loss) self._loss_dict.update(loss_dict) - kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, self._labels, self._feature_dict) + kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, + self._labels, self._feature_dict) self._loss_dict.update(kd_loss_dict) return self._loss_dict @@ -283,21 +297,19 @@ def get_outputs(self): tower_name = task_tower_cfg.tower_name if len(task_tower_cfg.losses) == 0: outputs.extend( - self._get_outputs_impl( - task_tower_cfg.loss_type, - task_tower_cfg.num_class, - suffix='_%s' % tower_name, - ) - ) + self._get_outputs_impl( + task_tower_cfg.loss_type, + task_tower_cfg.num_class, + suffix='_%s' % tower_name, + )) else: for loss in task_tower_cfg.losses: if loss.loss_type == LossType.ORDER_CALIBRATE_LOSS: continue outputs.extend( - self._get_outputs_impl( - loss.loss_type, - task_tower_cfg.num_class, - suffix='_%s' % tower_name, - ) - ) + self._get_outputs_impl( + loss.loss_type, + task_tower_cfg.num_class, + suffix='_%s' % tower_name, + )) return list(set(outputs)) diff --git a/easy_rec/python/model/multi_tower.py b/easy_rec/python/model/multi_tower.py index 648ed0774..0663602e8 100644 --- a/easy_rec/python/model/multi_tower.py +++ b/easy_rec/python/model/multi_tower.py @@ -5,6 +5,7 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': @@ -12,11 +13,17 @@ class MultiTower(RankModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(MultiTower, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(MultiTower, self).__init__(model_config, feature_configs, features, + labels, is_training) assert self._model_config.WhichOneof('model') == 'multi_tower', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.multi_tower assert isinstance(self._model_config, MultiTowerConfig) @@ -34,18 +41,20 @@ def build_predict_graph(self): tower = self._model_config.towers[tower_id] tower_name = tower.input tower_fea = tf.layers.batch_normalization( - tower_fea, - training=self._is_training, - trainable=True, - name='%s_fea_bn' % tower_name, + tower_fea, + training=self._is_training, + trainable=True, + name='%s_fea_bn' % tower_name, ) - tower_dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training) + tower_dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, + self._is_training) tower_fea = tower_dnn_layer(tower_fea) tower_fea_arr.append(tower_fea) all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, + 'final_dnn', self._is_training) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/multi_tower_bst.py b/easy_rec/python/model/multi_tower_bst.py index 99dc612cf..17bea71a6 100644 --- a/easy_rec/python/model/multi_tower_bst.py +++ b/easy_rec/python/model/multi_tower_bst.py @@ -6,8 +6,11 @@ import tensorflow as tf from easy_rec.python.compat import regularizers -from easy_rec.python.layers import dnn, layer_norm, seq_input_layer +from easy_rec.python.layers import dnn +from easy_rec.python.layers import layer_norm +from easy_rec.python.layers import seq_input_layer from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': @@ -15,17 +18,23 @@ class MultiTowerBST(RankModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(MultiTowerBST, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(MultiTowerBST, self).__init__(model_config, feature_configs, features, + labels, is_training) self._seq_input_layer = seq_input_layer.SeqInputLayer( - feature_configs, - model_config.seq_att_groups, - embedding_regularizer=self._emb_reg, - ev_params=self._global_ev_params, + feature_configs, + model_config.seq_att_groups, + embedding_regularizer=self._emb_reg, + ev_params=self._global_ev_params, ) assert self._model_config.WhichOneof('model') == 'multi_tower', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.multi_tower assert isinstance(self._model_config, MultiTowerConfig) @@ -39,14 +48,17 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai self._bst_tower_features = [] self._bst_tower_num = len(self._model_config.bst_towers) - logging.info('all tower num: {0}'.format(self._tower_num + self._bst_tower_num)) + logging.info('all tower num: {0}'.format(self._tower_num + + self._bst_tower_num)) logging.info('bst tower num: {0}'.format(self._bst_tower_num)) for tower_id in range(self._bst_tower_num): tower = self._model_config.bst_towers[tower_id] tower_feature = self._seq_input_layer(self._feature_dict, tower.input) - regularizers.apply_regularization(self._emb_reg, weights_list=[tower_feature['key']]) - regularizers.apply_regularization(self._emb_reg, weights_list=[tower_feature['hist_seq_emb']]) + regularizers.apply_regularization( + self._emb_reg, weights_list=[tower_feature['key']]) + regularizers.apply_regularization( + self._emb_reg, weights_list=[tower_feature['hist_seq_emb']]) self._bst_tower_features.append(tower_feature) def dnn_net(self, net, dnn_units, name): @@ -55,25 +67,30 @@ def dnn_net(self, net, dnn_units, name): for idx, units in enumerate(dnn_units): if idx + 1 < dnn_units_len: net = tf.layers.dense( - net, - units=units, - activation=tf.nn.relu, - name='%s_%d' % (name, idx), + net, + units=units, + activation=tf.nn.relu, + name='%s_%d' % (name, idx), ) else: - net = tf.layers.dense(net, units=units, activation=None, name='%s_%d' % (name, idx)) + net = tf.layers.dense( + net, units=units, activation=None, name='%s_%d' % (name, idx)) return net def attention_net(self, net, dim, cur_seq_len, seq_size, name): query_net = self.dnn_net(net, [dim], name + '_query') # B, seq_len，dim key_net = self.dnn_net(net, [dim], name + '_key') value_net = self.dnn_net(net, [dim], name + '_value') - scores = tf.matmul(query_net, key_net, transpose_b=True) # [B, seq_size, seq_size] + scores = tf.matmul( + query_net, key_net, transpose_b=True) # [B, seq_size, seq_size] - hist_mask = tf.sequence_mask(cur_seq_len, maxlen=seq_size - 1) # [B, seq_size-1] - cur_id_mask = tf.ones(tf.stack([tf.shape(hist_mask)[0], 1]), dtype=tf.bool) # [B, 1] + hist_mask = tf.sequence_mask( + cur_seq_len, maxlen=seq_size - 1) # [B, seq_size-1] + cur_id_mask = tf.ones( + tf.stack([tf.shape(hist_mask)[0], 1]), dtype=tf.bool) # [B, 1] mask = tf.concat([hist_mask, cur_id_mask], axis=1) # [B, seq_size] - masks = tf.reshape(tf.tile(mask, [1, seq_size]), (-1, seq_size, seq_size)) # [B, seq_size, seq_size] + masks = tf.reshape(tf.tile(mask, [1, seq_size]), + (-1, seq_size, seq_size)) # [B, seq_size, seq_size] padding = tf.ones_like(scores) * (-(2**32) + 1) scores = tf.where(masks, scores, padding) # [B, seq_size, seq_size] @@ -88,17 +105,19 @@ def multi_head_att_net(self, id_cols, head_count, emb_dim, seq_len, seq_size): for start_idx in range(0, emb_dim, part_cols_emd_dim): if start_idx + part_cols_emd_dim > emb_dim: part_cols_emd_dim = emb_dim - start_idx - part_id_col = tf.slice(id_cols, [0, 0, start_idx], [-1, -1, part_cols_emd_dim]) + part_id_col = tf.slice(id_cols, [0, 0, start_idx], + [-1, -1, part_cols_emd_dim]) part_attention_net = self.attention_net( - part_id_col, - part_cols_emd_dim, - seq_len, - seq_size, - name='multi_head_%d' % start_idx, + part_id_col, + part_cols_emd_dim, + seq_len, + seq_size, + name='multi_head_%d' % start_idx, ) multi_head_attention_res.append(part_attention_net) multi_head_attention_res_net = tf.concat(multi_head_attention_res, axis=2) - multi_head_attention_res_net = self.dnn_net(multi_head_attention_res_net, [emb_dim], name='multi_head_attention') + multi_head_attention_res_net = self.dnn_net( + multi_head_attention_res_net, [emb_dim], name='multi_head_attention') return multi_head_attention_res_net def add_and_norm(self, net_1, net_2, emb_dim, name): @@ -110,30 +129,34 @@ def add_and_norm(self, net_1, net_2, emb_dim, name): def bst(self, bst_fea, seq_size, head_count, name): cur_id, hist_id_col, seq_len = ( - bst_fea['key'], - bst_fea['hist_seq_emb'], - bst_fea['hist_seq_len'], + bst_fea['key'], + bst_fea['hist_seq_emb'], + bst_fea['hist_seq_len'], ) cur_batch_max_seq_len = tf.shape(hist_id_col)[1] hist_id_col = tf.cond( - tf.constant(seq_size) > cur_batch_max_seq_len, - lambda: tf.pad( - hist_id_col, - [[0, 0], [0, seq_size - cur_batch_max_seq_len - 1], [0, 0]], - 'CONSTANT', - ), - lambda: tf.slice(hist_id_col, [0, 0, 0], [-1, seq_size - 1, -1]), + tf.constant(seq_size) > cur_batch_max_seq_len, + lambda: tf.pad( + hist_id_col, + [[0, 0], [0, seq_size - cur_batch_max_seq_len - 1], [0, 0]], + 'CONSTANT', + ), + lambda: tf.slice(hist_id_col, [0, 0, 0], [-1, seq_size - 1, -1]), ) - all_ids = tf.concat([hist_id_col, tf.expand_dims(cur_id, 1)], axis=1) # b, seq_size, emb_dim + all_ids = tf.concat([hist_id_col, tf.expand_dims(cur_id, 1)], + axis=1) # b, seq_size, emb_dim emb_dim = int(all_ids.shape[2]) - attention_net = self.multi_head_att_net(all_ids, head_count, emb_dim, seq_len, seq_size) + attention_net = self.multi_head_att_net(all_ids, head_count, emb_dim, + seq_len, seq_size) - tmp_net = self.add_and_norm(all_ids, attention_net, emb_dim, name='add_and_norm_1') + tmp_net = self.add_and_norm( + all_ids, attention_net, emb_dim, name='add_and_norm_1') feed_forward_net = self.dnn_net(tmp_net, [emb_dim], 'feed_forward_net') - net = self.add_and_norm(tmp_net, feed_forward_net, emb_dim, name='add_and_norm_2') + net = self.add_and_norm( + tmp_net, feed_forward_net, emb_dim, name='add_and_norm_2') bst_output = tf.reshape(net, [-1, seq_size * emb_dim]) return bst_output @@ -144,12 +167,13 @@ def build_predict_graph(self): tower = self._model_config.towers[tower_id] tower_name = tower.input tower_fea = tf.layers.batch_normalization( - tower_fea, - training=self._is_training, - trainable=True, - name='%s_fea_bn' % tower_name, + tower_fea, + training=self._is_training, + trainable=True, + name='%s_fea_bn' % tower_name, ) - tower_dnn = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training) + tower_dnn = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, + self._is_training) tower_fea = tower_dnn(tower_fea) tower_fea_arr.append(tower_fea) @@ -160,15 +184,16 @@ def build_predict_graph(self): tower_seq_len = tower.seq_len tower_multi_head_size = tower.multi_head_size tower_fea = self.bst( - tower_fea, - seq_size=tower_seq_len, - head_count=tower_multi_head_size, - name=tower_name, + tower_fea, + seq_size=tower_seq_len, + head_count=tower_multi_head_size, + name=tower_name, ) tower_fea_arr.append(tower_fea) all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) + final_dnn = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', + self._is_training) all_fea = final_dnn(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/multi_tower_din.py b/easy_rec/python/model/multi_tower_din.py index 1a92f1d94..04f34c753 100644 --- a/easy_rec/python/model/multi_tower_din.py +++ b/easy_rec/python/model/multi_tower_din.py @@ -5,8 +5,10 @@ import tensorflow as tf from easy_rec.python.compat import regularizers -from easy_rec.python.layers import dnn, seq_input_layer +from easy_rec.python.layers import dnn +from easy_rec.python.layers import seq_input_layer from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.multi_tower_pb2 import MultiTower as MultiTowerConfig # NOQA if tf.__version__ >= '2.0': @@ -14,17 +16,23 @@ class MultiTowerDIN(RankModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(MultiTowerDIN, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(MultiTowerDIN, self).__init__(model_config, feature_configs, features, + labels, is_training) self._seq_input_layer = seq_input_layer.SeqInputLayer( - feature_configs, - model_config.seq_att_groups, - embedding_regularizer=self._emb_reg, - ev_params=self._global_ev_params, + feature_configs, + model_config.seq_att_groups, + embedding_regularizer=self._emb_reg, + ev_params=self._global_ev_params, ) assert self._model_config.WhichOneof('model') == 'multi_tower', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.multi_tower assert isinstance(self._model_config, MultiTowerConfig) @@ -38,7 +46,8 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai self._din_tower_features = [] self._din_tower_num = len(self._model_config.din_towers) - logging.info('all tower num: {0}'.format(self._tower_num + self._din_tower_num)) + logging.info('all tower num: {0}'.format(self._tower_num + + self._din_tower_num)) logging.info('din tower num: {0}'.format(self._din_tower_num)) for tower_id in range(self._din_tower_num): @@ -47,34 +56,36 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai # apply regularization for sequence feature key in seq_input_layer. - regularizers.apply_regularization(self._emb_reg, weights_list=[tower_feature['hist_seq_emb']]) + regularizers.apply_regularization( + self._emb_reg, weights_list=[tower_feature['hist_seq_emb']]) self._din_tower_features.append(tower_feature) def din(self, dnn_config, deep_fea, name): cur_id, hist_id_col, seq_len = ( - deep_fea['key'], - deep_fea['hist_seq_emb'], - deep_fea['hist_seq_len'], + deep_fea['key'], + deep_fea['hist_seq_emb'], + deep_fea['hist_seq_len'], ) seq_max_len = tf.shape(hist_id_col)[1] emb_dim = hist_id_col.shape[2] cur_ids = tf.tile(cur_id, [1, seq_max_len]) - cur_ids = tf.reshape(cur_ids, tf.shape(hist_id_col)) # (B, seq_max_len, emb_dim) + cur_ids = tf.reshape(cur_ids, + tf.shape(hist_id_col)) # (B, seq_max_len, emb_dim) din_net = tf.concat( - [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1, + [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], + axis=-1, ) # (B, seq_max_len, emb_dim*4) din_layer = dnn.DNN( - dnn_config, - self._l2_reg, - name, - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True, + dnn_config, + self._l2_reg, + name, + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True, ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) @@ -98,12 +109,13 @@ def build_predict_graph(self): tower = self._model_config.towers[tower_id] tower_name = tower.input tower_fea = tf.layers.batch_normalization( - tower_fea, - training=self._is_training, - trainable=True, - name='%s_fea_bn' % tower_name, + tower_fea, + training=self._is_training, + trainable=True, + name='%s_fea_bn' % tower_name, ) - dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training) + dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, + self._is_training) tower_fea = dnn_layer(tower_fea) tower_fea_arr.append(tower_fea) @@ -115,7 +127,8 @@ def build_predict_graph(self): tower_fea_arr.append(tower_fea) all_fea = tf.concat(tower_fea_arr, axis=1) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, + 'final_dnn', self._is_training) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/multi_tower_recall.py b/easy_rec/python/model/multi_tower_recall.py index 733b689ee..bf42a2cb8 100644 --- a/easy_rec/python/model/multi_tower_recall.py +++ b/easy_rec/python/model/multi_tower_recall.py @@ -5,6 +5,7 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.multi_tower_recall_pb2 import MultiTowerRecall as MultiTowerRecallConfig # NOQA if tf.__version__ >= '2.0': @@ -12,11 +13,17 @@ class MultiTowerRecall(RankModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(MultiTowerRecall, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(MultiTowerRecall, self).__init__(model_config, feature_configs, + features, labels, is_training) assert self._model_config.WhichOneof('model') == 'multi_tower_recall', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.multi_tower_recall assert isinstance(self._model_config, MultiTowerRecallConfig) @@ -29,26 +36,28 @@ def build_predict_graph(self): pos_item_feature = self.item_tower_feature[:batch_size] neg_item_feature = self.item_tower_feature[batch_size:] item_tower_feature = tf.concat( - [ - pos_item_feature[:, tf.newaxis, :], - tf.tile(neg_item_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1]), - ], - axis=1, + [ + pos_item_feature[:, tf.newaxis, :], + tf.tile( + neg_item_feature[tf.newaxis, :, :], + multiples=[batch_size, 1, 1]), + ], + axis=1, ) # noqa: E126 user_dnn = dnn.DNN( - self._model_config.user_tower.dnn, - self._l2_reg, - 'user_dnn', - self._is_training, + self._model_config.user_tower.dnn, + self._l2_reg, + 'user_dnn', + self._is_training, ) user_tower_emb = user_dnn(user_tower_feature) item_dnn = dnn.DNN( - self._model_config.item_tower.dnn, - self._l2_reg, - 'item_dnn', - self._is_training, + self._model_config.item_tower.dnn, + self._l2_reg, + 'item_dnn', + self._is_training, ) item_tower_emb = item_dnn(item_tower_feature) item_tower_emb = tf.reshape(item_tower_emb, tf.shape(user_tower_emb)) @@ -58,7 +67,8 @@ def build_predict_graph(self): tower_fea_arr.append(item_tower_emb) all_fea = tf.concat(tower_fea_arr, axis=-1) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, + 'final_dnn', self._is_training) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense(all_fea, 1, name='output') output = output[:, 0] diff --git a/easy_rec/python/model/pdn.py b/easy_rec/python/model/pdn.py index 279a840fc..195bbba1f 100644 --- a/easy_rec/python/model/pdn.py +++ b/easy_rec/python/model/pdn.py @@ -14,11 +14,19 @@ class PDN(MatchModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(PDN, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'pdn', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model' - ) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(PDN, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert self._model_config.WhichOneof( + 'model' + ) == 'pdn', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.pdn self._user_features, _ = self._input_layer(self._feature_dict, 'user') @@ -38,7 +46,9 @@ def build_predict_graph(self): sim_out = self._build_similarity_net() logits = tf.multiply(sim_out, trigger_out) - seq_mask = tf.to_float(tf.sequence_mask(self._seq_len, tf.shape(sim_out)[1])) + seq_mask = tf.to_float( + tf.sequence_mask(self._seq_len, + tf.shape(sim_out)[1])) logits = tf.reduce_sum(logits * seq_mask[:, :, None], axis=1) direct_logits = self._build_direct_net() @@ -53,7 +63,8 @@ def build_predict_graph(self): probs = 1 - tf.exp(-logits) # map [0, inf) to [0, 1) self._prediction_dict['probs'] = probs - self._prediction_dict['logits'] = tf.log(tf.clip_by_value(probs, 1e-8, 1 - 1e-8)) + self._prediction_dict['logits'] = tf.log( + tf.clip_by_value(probs, 1e-8, 1 - 1e-8)) return self._prediction_dict def _get_seq_features(self, name): @@ -63,21 +74,23 @@ def _get_seq_features(self, name): return seq, seq_len def _build_trigger_net(self): - user_dnn_layer = dnn.DNN(self._model_config.user_dnn, self._l2_reg, 'user_dnn', self._is_training) + user_dnn_layer = dnn.DNN(self._model_config.user_dnn, self._l2_reg, + 'user_dnn', self._is_training) user_fea = user_dnn_layer(self._user_features) trigger_seq = tf.concat([self._u2i_seq, self._i_seq], axis=2) - u2i_dnn_layer = dnn.DNN(self._model_config.u2i_dnn, self._l2_reg, 'u2i_dnn', self._is_training) + u2i_dnn_layer = dnn.DNN(self._model_config.u2i_dnn, self._l2_reg, 'u2i_dnn', + self._is_training) trigger_seq_fea = u2i_dnn_layer(trigger_seq) trigger_merge_fea = trigger_seq_fea + user_fea[:, None, :] trigger_dnn_layer = dnn.DNN( - self._model_config.trigger_dnn, - self._l2_reg, - 'trigger_dnn', - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True, + self._model_config.trigger_dnn, + self._l2_reg, + 'trigger_dnn', + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True, ) # output: N x seq_len x d, d is usually set to 1 @@ -86,35 +99,39 @@ def _build_trigger_net(self): trigger_out = tf.exp(trigger_out) self._prediction_dict['trigger_out'] = tf.reduce_join( - tf.reduce_join( - tf.as_string(trigger_out, precision=4, shortest=True), - axis=2, - separator=',', - ), - axis=1, - separator=';', + tf.reduce_join( + tf.as_string(trigger_out, precision=4, shortest=True), + axis=2, + separator=',', + ), + axis=1, + separator=';', ) return trigger_out def _build_similarity_net(self): - item_dnn_layer = dnn.DNN(self._model_config.item_dnn, self._l2_reg, 'item_dnn', self._is_training) + item_dnn_layer = dnn.DNN(self._model_config.item_dnn, self._l2_reg, + 'item_dnn', self._is_training) item_fea = item_dnn_layer(self._item_features) - sim_side_dnn_layer = dnn.DNN(self._model_config.i2i_dnn, self._l2_reg, 'i2i_dnn', self._is_training) + sim_side_dnn_layer = dnn.DNN(self._model_config.i2i_dnn, self._l2_reg, + 'i2i_dnn', self._is_training) sim_seq_fea = sim_side_dnn_layer(self._i_seq) sim_seq_cross = sim_seq_fea * item_fea[:, None, :] - item_fea_tile = tf.tile(item_fea[:, None, :], [1, tf.shape(sim_seq_fea)[1], 1]) + item_fea_tile = tf.tile(item_fea[:, None, :], + [1, tf.shape(sim_seq_fea)[1], 1]) - sim_seq_concat = tf.concat([sim_seq_cross, sim_seq_cross, self._i2i_seq, item_fea_tile], axis=2) + sim_seq_concat = tf.concat( + [sim_seq_cross, sim_seq_cross, self._i2i_seq, item_fea_tile], axis=2) sim_dnn_layer = dnn.DNN( - self._model_config.sim_dnn, - self._l2_reg, - 'sim_dnn', - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True, + self._model_config.sim_dnn, + self._l2_reg, + 'sim_dnn', + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True, ) # output: N x seq_len x 1 sim_out = sim_dnn_layer(sim_seq_concat) @@ -122,28 +139,32 @@ def _build_similarity_net(self): sim_out = tf.exp(sim_out) self._prediction_dict['sim_out'] = tf.reduce_join( - tf.reduce_join(tf.as_string(sim_out, precision=4, shortest=True), axis=2, separator=','), - axis=1, - separator=';', + tf.reduce_join( + tf.as_string(sim_out, precision=4, shortest=True), + axis=2, + separator=','), + axis=1, + separator=';', ) return sim_out def _build_direct_net(self): - if self._model_config.HasField('direct_user_dnn') and self._model_config.HasField('direct_item_dnn'): + if self._model_config.HasField( + 'direct_user_dnn') and self._model_config.HasField('direct_item_dnn'): direct_user_layer = dnn.DNN( - self._model_config.direct_user_dnn, - 'direct_user_dnn', - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True, + self._model_config.direct_user_dnn, + 'direct_user_dnn', + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True, ) direct_user_out = direct_user_layer(self._user_features) direct_item_layer = dnn.DNN( - self._model_config.direct_item_dnn, - 'direct_item_dnn', - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True, + self._model_config.direct_item_dnn, + 'direct_item_dnn', + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True, ) direct_item_out = direct_item_layer(self._item_features) @@ -157,16 +178,16 @@ def _build_direct_net(self): if self._model_config.scale_simi: sim_w = tf.get_variable( - 'direct_net/sim_w', - dtype=tf.float32, - shape=(1), - initializer=tf.ones_initializer(), + 'direct_net/sim_w', + dtype=tf.float32, + shape=(1), + initializer=tf.ones_initializer(), ) sim_b = tf.get_variable( - 'direct_net/sim_b', - dtype=tf.float32, - shape=(1), - initializer=tf.zeros_initializer(), + 'direct_net/sim_b', + dtype=tf.float32, + shape=(1), + initializer=tf.zeros_initializer(), ) direct_logits = direct_logits * tf.abs(sim_w) + sim_b @@ -178,12 +199,12 @@ def _build_bias_net(self): if self._model_config.HasField('bias_dnn'): assert self._bias_features is not None, 'bias group must be defined' bias_dnn_layer = dnn.DNN( - self._model_config.bias_dnn, - self._l2_reg, - 'bias_dnn', - self._is_training, - last_layer_no_activation=True, - last_layer_no_batch_norm=True, + self._model_config.bias_dnn, + self._l2_reg, + 'bias_dnn', + self._is_training, + last_layer_no_activation=True, + last_layer_no_batch_norm=True, ) bias_logits = bias_dnn_layer(self._bias_features) return tf.nn.softplus(bias_logits) diff --git a/easy_rec/python/model/ple.py b/easy_rec/python/model/ple.py index f04d9a463..6159c8272 100644 --- a/easy_rec/python/model/ple.py +++ b/easy_rec/python/model/ple.py @@ -11,11 +11,19 @@ class PLE(MultiTaskModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(PLE, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'ple', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model' - ) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(PLE, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert self._model_config.WhichOneof( + 'model' + ) == 'ple', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.ple assert isinstance(self._model_config, PLEConfig) @@ -30,11 +38,11 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai def gate(self, selector_fea, vec_feas, name): vec = tf.stack(vec_feas, axis=1) gate = tf.layers.dense( - inputs=selector_fea, - units=len(vec_feas), - kernel_regularizer=self._l2_reg, - activation=None, - name=name + '_gate/dnn', + inputs=selector_fea, + units=len(vec_feas), + kernel_regularizer=self._l2_reg, + activation=None, + name=name + '_gate/dnn', ) gate = tf.nn.softmax(gate, axis=1) gate = tf.expand_dims(gate, -1) @@ -46,28 +54,28 @@ def experts_layer(self, deep_fea, expert_num, experts_cfg, name): tower_outputs = [] for expert_id in range(expert_num): tower_dnn = dnn.DNN( - experts_cfg, - self._l2_reg, - name=name + '_expert_%d/dnn' % expert_id, - is_training=self._is_training, + experts_cfg, + self._l2_reg, + name=name + '_expert_%d/dnn' % expert_id, + is_training=self._is_training, ) tower_output = tower_dnn(deep_fea) tower_outputs.append(tower_output) return tower_outputs def CGC_layer( - self, - extraction_networks_cfg, - extraction_network_fea, - shared_expert_fea, - final_flag, + self, + extraction_networks_cfg, + extraction_network_fea, + shared_expert_fea, + final_flag, ): layer_name = extraction_networks_cfg.network_name expert_shared_out = self.experts_layer( - shared_expert_fea, - extraction_networks_cfg.share_num, - extraction_networks_cfg.share_expert_net, - layer_name + '_share/dnn', + shared_expert_fea, + extraction_networks_cfg.share_num, + extraction_networks_cfg.share_expert_net, + layer_name + '_share/dnn', ) experts_outs = [] @@ -75,12 +83,13 @@ def CGC_layer( for task_idx in range(self._task_nums): name = layer_name + '_task_%d' % task_idx experts_out = self.experts_layer( - extraction_network_fea[task_idx], - extraction_networks_cfg.expert_num_per_task, - extraction_networks_cfg.task_expert_net, - name, + extraction_network_fea[task_idx], + extraction_networks_cfg.expert_num_per_task, + extraction_networks_cfg.task_expert_net, + name, ) - cgc_layer_out = self.gate(extraction_network_fea[task_idx], experts_out + expert_shared_out, name) + cgc_layer_out = self.gate(extraction_network_fea[task_idx], + experts_out + expert_shared_out, name) experts_outs.extend(experts_out) cgc_layer_outs.append(cgc_layer_out) @@ -88,9 +97,9 @@ def CGC_layer( shared_layer_out = None else: shared_layer_out = self.gate( - shared_expert_fea, - experts_outs + expert_shared_out, - layer_name + '_share', + shared_expert_fea, + experts_outs + expert_shared_out, + layer_name + '_share', ) return cgc_layer_outs, shared_layer_out @@ -103,28 +112,28 @@ def build_predict_graph(self): if idx == len(self._model_config.extraction_networks) - 1: final_flag = True extraction_network_fea, shared_expert_fea = self.CGC_layer( - extraction_network, - extraction_network_fea, - shared_expert_fea, - final_flag, + extraction_network, + extraction_network_fea, + shared_expert_fea, + final_flag, ) tower_outputs = {} for i, task_tower_cfg in enumerate(self._model_config.task_towers): tower_name = task_tower_cfg.tower_name tower_dnn = dnn.DNN( - task_tower_cfg.dnn, - self._l2_reg, - name=tower_name, - is_training=self._is_training, + task_tower_cfg.dnn, + self._l2_reg, + name=tower_name, + is_training=self._is_training, ) tower_output = tower_dnn(extraction_network_fea[i]) tower_output = tf.layers.dense( - inputs=tower_output, - units=task_tower_cfg.num_class, - kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i, + inputs=tower_output, + units=task_tower_cfg.num_class, + kernel_regularizer=self._l2_reg, + name='dnn_output_%d' % i, ) tower_outputs[tower_name] = tower_output diff --git a/easy_rec/python/model/rank_model.py b/easy_rec/python/model/rank_model.py index bab2b9ec9..e08a80e3c 100644 --- a/easy_rec/python/model/rank_model.py +++ b/easy_rec/python/model/rank_model.py @@ -6,19 +6,26 @@ from tensorflow.python.ops import math_ops from easy_rec.python.builders import loss_builder -from easy_rec.python.loss.zero_inflated_lognormal import ( # NOQA - zero_inflated_lognormal_pred, -) from easy_rec.python.model.easy_rec_model import EasyRecModel from easy_rec.python.protos.loss_pb2 import LossType +from easy_rec.python.loss.zero_inflated_lognormal import ( # NOQA + zero_inflated_lognormal_pred,) + if tf.__version__ >= '2.0': tf = tf.compat.v1 class RankModel(EasyRecModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(RankModel, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(RankModel, self).__init__(model_config, feature_configs, features, + labels, is_training) self._loss_type = self._model_config.loss_type self._num_class = self._model_config.num_class self._losses = self._model_config.losses @@ -31,7 +38,9 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai def build_predict_graph(self): if not self.has_backbone: - raise NotImplementedError('method `build_predict_graph` must be implemented when backbone network do not exits') + raise NotImplementedError( + 'method `build_predict_graph` must be implemented when backbone network do not exits' + ) model = self._model_config.WhichOneof('model') assert model == 'model_params', '`model_params` must be configured' config = self._model_config.model_params @@ -45,18 +54,23 @@ def build_predict_graph(self): self._add_to_prediction_dict(output) return self._prediction_dict - def _output_to_prediction_impl(self, output, loss_type, num_class=1, suffix='', **kwargs): + def _output_to_prediction_impl(self, + output, + loss_type, + num_class=1, + suffix='', + **kwargs): prediction_dict = {} binary_loss_type = { - LossType.F1_REWEIGHTED_LOSS, - LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, - LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, - LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, - LossType.BINARY_CROSS_ENTROPY_LOSS, - LossType.LISTWISE_DISTILL_LOSS, + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.BINARY_CROSS_ENTROPY_LOSS, + LossType.LISTWISE_DISTILL_LOSS, } if loss_type in binary_loss_type: assert num_class == 1, 'num_class must be 1 when loss type is %s' % loss_type.name @@ -78,10 +92,10 @@ def _output_to_prediction_impl(self, output, loss_type, num_class=1, suffix='', return_log = kwargs.get('return_log_pred_value', False) max_sigma = kwargs.get('max_sigma', 5.0) probs, preds = zero_inflated_lognormal_pred( - output, - max_sigma=max_sigma, - max_log_clip=max_log_clip_val, - return_log=return_log, + output, + max_sigma=max_sigma, + max_log_clip=max_log_clip_val, + return_log=return_log, ) tf.summary.scalar('prediction/probs', tf.reduce_mean(probs)) tf.summary.scalar('prediction/y', tf.reduce_mean(preds)) @@ -101,8 +115,10 @@ def _output_to_prediction_impl(self, output, loss_type, num_class=1, suffix='', prediction_dict['logits' + suffix + '_1'] = output[:, 1] prediction_dict['probs' + suffix] = probs prediction_dict['probs' + suffix + '_1'] = probs[:, 1] - prediction_dict['logits' + suffix + '_y'] = math_ops.reduce_max(output, axis=1) - prediction_dict['probs' + suffix + '_y'] = math_ops.reduce_max(probs, axis=1) + prediction_dict['logits' + suffix + '_y'] = math_ops.reduce_max( + output, axis=1) + prediction_dict['probs' + suffix + '_y'] = math_ops.reduce_max( + probs, axis=1) prediction_dict['y' + suffix] = tf.argmax(output, axis=1) elif loss_type == LossType.L2_LOSS: output = tf.squeeze(output, axis=1) @@ -114,7 +130,8 @@ def _output_to_prediction_impl(self, output, loss_type, num_class=1, suffix='', def _add_to_prediction_dict(self, output): if len(self._losses) == 0: - prediction_dict = self._output_to_prediction_impl(output, loss_type=self._loss_type, num_class=self._num_class) + prediction_dict = self._output_to_prediction_impl( + output, loss_type=self._loss_type, num_class=self._num_class) self._prediction_dict.update(prediction_dict) else: for loss in self._losses: @@ -127,10 +144,10 @@ def _add_to_prediction_dict(self, output): kwargs['return_log_pred_value'] = loss_param.return_log_pred_value kwargs['max_sigma'] = loss_param.max_sigma prediction_dict = self._output_to_prediction_impl( - output, - loss_type=loss.loss_type, - num_class=self._num_class, - **kwargs, + output, + loss_type=loss.loss_type, + num_class=self._num_class, + **kwargs, ) self._prediction_dict.update(prediction_dict) @@ -143,10 +160,9 @@ def build_rtp_output_dict(self): op = tf.get_default_graph().get_operation_by_name('rank_predict') if len(op.outputs) != 1: raise ValueError( - ('failed to build RTP rank_predict output: op {}[{}] has output ' + 'size {}, however 1 is expected.').format( - op.name, op.type, len(op.outputs) - ) - ) + ('failed to build RTP rank_predict output: op {}[{}] has output ' + + 'size {}, however 1 is expected.').format(op.name, op.type, + len(op.outputs))) rank_predict = op.outputs[0] except KeyError: forwarded = None @@ -154,40 +170,41 @@ def build_rtp_output_dict(self): if len(self._losses) > 0: loss_types = {loss.loss_type for loss in self._losses} binary_loss_set = { - LossType.CLASSIFICATION, - LossType.F1_REWEIGHTED_LOSS, - LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, - LossType.PAIRWISE_FOCAL_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, - LossType.JRC_LOSS, - LossType.LISTWISE_DISTILL_LOSS, - LossType.LISTWISE_RANK_LOSS, + LossType.CLASSIFICATION, + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.JRC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, + LossType.LISTWISE_RANK_LOSS, } if loss_types & binary_loss_set: if 'probs' in self._prediction_dict: forwarded = self._prediction_dict['probs'] else: raise ValueError( - 'failed to build RTP rank_predict output: classification model ' - + "expect 'probs' prediction, which is not found. Please check if" - + ' build_predict_graph() is called.' - ) + 'failed to build RTP rank_predict output: classification model ' + + "expect 'probs' prediction, which is not found. Please check if" + + ' build_predict_graph() is called.') elif loss_types & { - LossType.L2_LOSS, - LossType.SIGMOID_L2_LOSS, - LossType.ZILN_LOSS, + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: if 'y' in self._prediction_dict: forwarded = self._prediction_dict['y'] else: raise ValueError( - 'failed to build RTP rank_predict output: regression model expect' - + "'y' prediction, which is not found. Please check if build_predic" - + 't_graph() is called.' - ) + 'failed to build RTP rank_predict output: regression model expect' + + + "'y' prediction, which is not found. Please check if build_predic" + + 't_graph() is called.') else: - logging.warning('failed to build RTP rank_predict: unsupported loss type {}'.format(loss_types)) + logging.warning( + 'failed to build RTP rank_predict: unsupported loss type {}'.format( + loss_types)) if forwarded is not None: rank_predict = tf.identity(forwarded, name='rank_predict') if rank_predict is not None: @@ -195,29 +212,31 @@ def build_rtp_output_dict(self): return outputs def _build_loss_impl( - self, - loss_type, - label_name, - loss_weight=1.0, - num_class=1, - suffix='', - loss_name='', - loss_param=None, + self, + loss_type, + label_name, + loss_weight=1.0, + num_class=1, + suffix='', + loss_name='', + loss_param=None, ): loss_dict = {} binary_loss_type = { - LossType.F1_REWEIGHTED_LOSS, - LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, - LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, - LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, - LossType.JRC_LOSS, - LossType.LISTWISE_DISTILL_LOSS, - LossType.ZILN_LOSS, + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.JRC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, + LossType.ZILN_LOSS, } - if loss_type in {LossType.CLASSIFICATION, LossType.BINARY_CROSS_ENTROPY_LOSS}: + if loss_type in { + LossType.CLASSIFICATION, LossType.BINARY_CROSS_ENTROPY_LOSS + }: loss_name = loss_name if loss_name else 'cross_entropy_loss' + suffix pred = self._prediction_dict['logits' + suffix] elif loss_type in binary_loss_type: @@ -233,21 +252,21 @@ def _build_loss_impl( raise ValueError('invalid loss type: %s' % LossType.Name(loss_type)) tf.summary.scalar( - 'labels/%s' % label_name, - tf.reduce_mean(tf.to_float(self._labels[label_name])), + 'labels/%s' % label_name, + tf.reduce_mean(tf.to_float(self._labels[label_name])), ) kwargs = {'loss_name': loss_name} if loss_param is not None: if hasattr(loss_param, 'session_name'): kwargs['session_ids'] = self._feature_dict[loss_param.session_name] loss_dict[loss_name] = loss_builder.build( - loss_type, - self._labels[label_name], - pred, - loss_weight, - num_class, - loss_param=loss_param, - **kwargs, + loss_type, + self._labels[label_name], + pred, + loss_weight, + num_class, + loss_param=loss_param, + **kwargs, ) return loss_dict @@ -256,10 +275,10 @@ def build_loss_graph(self): with tf.name_scope('loss'): if len(self._losses) == 0: loss_dict = self._build_loss_impl( - self._loss_type, - label_name=self._label_name, - loss_weight=self._sample_weight, - num_class=self._num_class, + self._loss_type, + label_name=self._label_name, + loss_weight=self._sample_weight, + num_class=self._num_class, ) else: strategy = self._base_model_config.loss_weight_strategy @@ -272,12 +291,12 @@ def build_loss_graph(self): if loss_param is not None: loss_param = getattr(loss, loss_param) loss_ops = self._build_loss_impl( - loss.loss_type, - label_name=self._label_name, - loss_weight=self._sample_weight, - num_class=self._num_class, - loss_name=loss.loss_name, - loss_param=loss_param, + loss.loss_type, + label_name=self._label_name, + loss_weight=self._sample_weight, + num_class=self._num_class, + loss_name=loss.loss_name, + loss_param=loss_param, ) for loss_name, loss_value in loss_ops.items(): if strategy == self._base_model_config.Fixed: @@ -285,47 +304,57 @@ def build_loss_graph(self): elif strategy == self._base_model_config.Uncertainty: if loss.learn_loss_weight: uncertainty = tf.Variable( - 0, - name='%s_loss_weight' % loss_name, - dtype=tf.float32, + 0, + name='%s_loss_weight' % loss_name, + dtype=tf.float32, ) tf.summary.scalar('%s_uncertainty' % loss_name, uncertainty) if loss.loss_type in { - LossType.L2_LOSS, - LossType.SIGMOID_L2_LOSS, + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, }: - loss_dict[loss_name] = 0.5 * tf.exp(-uncertainty) * loss_value + 0.5 * uncertainty + loss_dict[loss_name] = 0.5 * tf.exp( + -uncertainty) * loss_value + 0.5 * uncertainty else: - loss_dict[loss_name] = tf.exp(-uncertainty) * loss_value + 0.5 * uncertainty + loss_dict[loss_name] = tf.exp( + -uncertainty) * loss_value + 0.5 * uncertainty else: loss_dict[loss_name] = loss_value * loss.weight elif strategy == self._base_model_config.Random: loss_dict[loss_name] = loss_value * loss_weight[i] else: - raise ValueError('Unsupported loss weight strategy: ' + strategy.Name) + raise ValueError('Unsupported loss weight strategy: ' + + strategy.Name) self._loss_dict.update(loss_dict) # build kd loss - kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, self._labels, self._feature_dict) + kd_loss_dict = loss_builder.build_kd_loss(self.kd, self._prediction_dict, + self._labels, + self._feature_dict) self._loss_dict.update(kd_loss_dict) return self._loss_dict - def _build_metric_impl(self, metric, loss_type, label_name, num_class=1, suffix=''): + def _build_metric_impl(self, + metric, + loss_type, + label_name, + num_class=1, + suffix=''): if not isinstance(loss_type, set): loss_type = {loss_type} from easy_rec.python.core import metrics as metrics_lib from easy_rec.python.core.easyrec_metrics import metrics_tf binary_loss_set = { - LossType.CLASSIFICATION, - LossType.F1_REWEIGHTED_LOSS, - LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, - LossType.PAIRWISE_FOCAL_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, - LossType.JRC_LOSS, - LossType.LISTWISE_DISTILL_LOSS, - LossType.LISTWISE_RANK_LOSS, - LossType.ZILN_LOSS, + LossType.CLASSIFICATION, + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.JRC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.ZILN_LOSS, } metric_dict = {} if metric.WhichOneof('metric') == 'auc': @@ -333,16 +362,16 @@ def _build_metric_impl(self, metric, loss_type, label_name, num_class=1, suffix= if num_class == 1 or loss_type & {LossType.JRC_LOSS, LossType.ZILN_LOSS}: label = tf.to_int64(self._labels[label_name]) metric_dict['auc' + suffix] = metrics_tf.auc( - label, - self._prediction_dict['probs' + suffix], - num_thresholds=metric.auc.num_thresholds, + label, + self._prediction_dict['probs' + suffix], + num_thresholds=metric.auc.num_thresholds, ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) metric_dict['auc' + suffix] = metrics_tf.auc( - label, - self._prediction_dict['probs' + suffix][:, 1], - num_thresholds=metric.auc.num_thresholds, + label, + self._prediction_dict['probs' + suffix][:, 1], + num_thresholds=metric.auc.num_thresholds, ) else: raise ValueError('Wrong class number') @@ -352,21 +381,22 @@ def _build_metric_impl(self, metric, loss_type, label_name, num_class=1, suffix= label = tf.to_int64(self._labels[label_name]) uids = self._feature_dict[metric.gauc.uid_field] if isinstance(uids, tf.sparse.SparseTensor): - uids = tf.sparse_to_dense(uids.indices, uids.dense_shape, uids.values, default_value='') + uids = tf.sparse_to_dense( + uids.indices, uids.dense_shape, uids.values, default_value='') uids = tf.reshape(uids, [-1]) metric_dict['gauc' + suffix] = metrics_lib.gauc( - label, - self._prediction_dict['probs' + suffix], - uids=uids, - reduction=metric.gauc.reduction, + label, + self._prediction_dict['probs' + suffix], + uids=uids, + reduction=metric.gauc.reduction, ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) metric_dict['gauc' + suffix] = metrics_lib.gauc( - label, - self._prediction_dict['probs' + suffix][:, 1], - uids=self._feature_dict[metric.gauc.uid_field], - reduction=metric.gauc.reduction, + label, + self._prediction_dict['probs' + suffix][:, 1], + uids=self._feature_dict[metric.gauc.uid_field], + reduction=metric.gauc.reduction, ) else: raise ValueError('Wrong class number') @@ -375,18 +405,18 @@ def _build_metric_impl(self, metric, loss_type, label_name, num_class=1, suffix= if num_class == 1 or loss_type & {LossType.JRC_LOSS, LossType.ZILN_LOSS}: label = tf.to_int64(self._labels[label_name]) metric_dict['session_auc' + suffix] = metrics_lib.session_auc( - label, - self._prediction_dict['probs' + suffix], - session_ids=self._feature_dict[metric.session_auc.session_id_field], - reduction=metric.session_auc.reduction, + label, + self._prediction_dict['probs' + suffix], + session_ids=self._feature_dict[metric.session_auc.session_id_field], + reduction=metric.session_auc.reduction, ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) metric_dict['session_auc' + suffix] = metrics_lib.session_auc( - label, - self._prediction_dict['probs' + suffix][:, 1], - session_ids=self._feature_dict[metric.session_auc.session_id_field], - reduction=metric.session_auc.reduction, + label, + self._prediction_dict['probs' + suffix][:, 1], + session_ids=self._feature_dict[metric.session_auc.session_id_field], + reduction=metric.session_auc.reduction, ) else: raise ValueError('Wrong class number') @@ -394,10 +424,12 @@ def _build_metric_impl(self, metric, loss_type, label_name, num_class=1, suffix= assert loss_type & binary_loss_set if num_class == 1 or loss_type & {LossType.JRC_LOSS, LossType.ZILN_LOSS}: label = tf.to_int64(self._labels[label_name]) - metric_dict['max_f1' + suffix] = metrics_lib.max_f1(label, self._prediction_dict['logits' + suffix]) + metric_dict['max_f1' + suffix] = metrics_lib.max_f1( + label, self._prediction_dict['logits' + suffix]) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) - metric_dict['max_f1' + suffix] = metrics_lib.max_f1(label, self._prediction_dict['logits' + suffix][:, 1]) + metric_dict['max_f1' + suffix] = metrics_lib.max_f1( + label, self._prediction_dict['logits' + suffix][:, 1]) else: raise ValueError('Wrong class number') elif metric.WhichOneof('metric') == 'recall_at_topk': @@ -405,63 +437,64 @@ def _build_metric_impl(self, metric, loss_type, label_name, num_class=1, suffix= assert num_class > 1 label = tf.to_int64(self._labels[label_name]) metric_dict['recall_at_topk' + suffix] = metrics_tf.recall_at_k( - label, - self._prediction_dict['logits' + suffix], - metric.recall_at_topk.topk, + label, + self._prediction_dict['logits' + suffix], + metric.recall_at_topk.topk, ) elif metric.WhichOneof('metric') == 'mean_absolute_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, - LossType.SIGMOID_L2_LOSS, - LossType.ZILN_LOSS, + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: - metric_dict['mean_absolute_error' + suffix] = metrics_tf.mean_absolute_error( - label, self._prediction_dict['y' + suffix] - ) + metric_dict['mean_absolute_error' + + suffix] = metrics_tf.mean_absolute_error( + label, self._prediction_dict['y' + suffix]) elif loss_type & {LossType.CLASSIFICATION} and num_class == 1: - metric_dict['mean_absolute_error' + suffix] = metrics_tf.mean_absolute_error( - label, self._prediction_dict['probs' + suffix] - ) + metric_dict['mean_absolute_error' + + suffix] = metrics_tf.mean_absolute_error( + label, self._prediction_dict['probs' + suffix]) else: assert False, 'mean_absolute_error is not supported for this model' elif metric.WhichOneof('metric') == 'mean_squared_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, - LossType.SIGMOID_L2_LOSS, - LossType.ZILN_LOSS, + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: - metric_dict['mean_squared_error' + suffix] = metrics_tf.mean_squared_error( - label, self._prediction_dict['y' + suffix] - ) + metric_dict['mean_squared_error' + + suffix] = metrics_tf.mean_squared_error( + label, self._prediction_dict['y' + suffix]) elif num_class == 1 and loss_type & binary_loss_set: - metric_dict['mean_squared_error' + suffix] = metrics_tf.mean_squared_error( - label, self._prediction_dict['probs' + suffix] - ) + metric_dict['mean_squared_error' + + suffix] = metrics_tf.mean_squared_error( + label, self._prediction_dict['probs' + suffix]) else: assert False, 'mean_squared_error is not supported for this model' elif metric.WhichOneof('metric') == 'root_mean_squared_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, - LossType.SIGMOID_L2_LOSS, - LossType.ZILN_LOSS, + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: - metric_dict['root_mean_squared_error' + suffix] = metrics_tf.root_mean_squared_error( - label, self._prediction_dict['y' + suffix] - ) + metric_dict['root_mean_squared_error' + + suffix] = metrics_tf.root_mean_squared_error( + label, self._prediction_dict['y' + suffix]) elif loss_type & {LossType.CLASSIFICATION} and num_class == 1: - metric_dict['root_mean_squared_error' + suffix] = metrics_tf.root_mean_squared_error( - label, self._prediction_dict['probs' + suffix] - ) + metric_dict['root_mean_squared_error' + + suffix] = metrics_tf.root_mean_squared_error( + label, self._prediction_dict['probs' + suffix]) else: assert False, 'root_mean_squared_error is not supported for this model' elif metric.WhichOneof('metric') == 'accuracy': assert loss_type & {LossType.CLASSIFICATION} assert num_class > 1 label = tf.to_int64(self._labels[label_name]) - metric_dict['accuracy' + suffix] = metrics_tf.accuracy(label, self._prediction_dict['y' + suffix]) + metric_dict['accuracy' + suffix] = metrics_tf.accuracy( + label, self._prediction_dict['y' + suffix]) return metric_dict def build_metric_graph(self, eval_config): @@ -470,25 +503,24 @@ def build_metric_graph(self, eval_config): loss_types = {loss.loss_type for loss in self._losses} for metric in eval_config.metrics_set: self._metric_dict.update( - self._build_metric_impl( - metric, - loss_type=loss_types, - label_name=self._label_name, - num_class=self._num_class, - ) - ) + self._build_metric_impl( + metric, + loss_type=loss_types, + label_name=self._label_name, + num_class=self._num_class, + )) return self._metric_dict def _get_outputs_impl(self, loss_type, num_class=1, suffix=''): binary_loss_set = { - LossType.F1_REWEIGHTED_LOSS, - LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, - LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, - LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, - LossType.LISTWISE_DISTILL_LOSS, + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, } if loss_type in binary_loss_set: return ['probs' + suffix, 'logits' + suffix] @@ -501,13 +533,13 @@ def _get_outputs_impl(self, loss_type, num_class=1, suffix=''): return ['probs' + suffix, 'logits' + suffix] else: return [ - 'y' + suffix, - 'probs' + suffix, - 'logits' + suffix, - 'probs' + suffix + '_y', - 'logits' + suffix + '_y', - 'probs' + suffix + '_1', - 'logits' + suffix + '_1', + 'y' + suffix, + 'probs' + suffix, + 'logits' + suffix, + 'probs' + suffix + '_y', + 'logits' + suffix + '_y', + 'probs' + suffix + '_1', + 'logits' + suffix + '_1', ] elif loss_type in [LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS]: return ['y' + suffix] diff --git a/easy_rec/python/model/rocket_launching.py b/easy_rec/python/model/rocket_launching.py index f1bd18942..d98b8b854 100755 --- a/easy_rec/python/model/rocket_launching.py +++ b/easy_rec/python/model/rocket_launching.py @@ -6,19 +6,26 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel from easy_rec.python.protos.loss_pb2 import LossType -from easy_rec.python.protos.rocket_launching_pb2 import RocketLaunching as RocketLaunchingConfig # NOQA from easy_rec.python.protos.simi_pb2 import Similarity +from easy_rec.python.protos.rocket_launching_pb2 import RocketLaunching as RocketLaunchingConfig # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 class RocketLaunching(RankModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(RocketLaunching, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(RocketLaunching, self).__init__(model_config, feature_configs, + features, labels, is_training) assert self._model_config.WhichOneof('model') == 'rocket_launching', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.rocket_launching assert isinstance(self._model_config, RocketLaunchingConfig) if self._labels is not None: @@ -27,7 +34,8 @@ def __init__(self, model_config, feature_configs, features, labels=None, is_trai self._features, _ = self._input_layer(self._feature_dict, 'all') def sim(self, feature_emb1, feature_emb2): - emb1_emb2_sim = tf.reduce_sum(tf.multiply(feature_emb1, feature_emb2), axis=1, keepdims=True) + emb1_emb2_sim = tf.reduce_sum( + tf.multiply(feature_emb1, feature_emb2), axis=1, keepdims=True) return emb1_emb2_sim def norm(self, fea): @@ -35,83 +43,94 @@ def norm(self, fea): return fea_norm def feature_based_sim(self, feature_based_distillation, i, j): - booster_feature_no_gradient = tf.stop_gradient(self.booster_feature['hidden_layer' + str(j)]) + booster_feature_no_gradient = tf.stop_gradient( + self.booster_feature['hidden_layer' + str(j)]) if feature_based_distillation == Similarity.COSINE: booster_feature_no_gradient_norm = self.norm(booster_feature_no_gradient) - light_feature_norm = self.norm(self.light_feature['hidden_layer' + str(i)]) - sim_middle_layer = tf.reduce_mean(self.sim(booster_feature_no_gradient_norm, light_feature_norm)) + light_feature_norm = self.norm(self.light_feature['hidden_layer' + + str(i)]) + sim_middle_layer = tf.reduce_mean( + self.sim(booster_feature_no_gradient_norm, light_feature_norm)) return sim_middle_layer else: return tf.sqrt( - tf.reduce_sum(tf.square(booster_feature_no_gradient - self.light_feature['hidden_layer' + str(i)])) - ) + tf.reduce_sum( + tf.square(booster_feature_no_gradient - + self.light_feature['hidden_layer' + str(i)]))) def build_predict_graph(self): self.hidden_layer_feature_output = self._model_config.feature_based_distillation if self._model_config.HasField('share_dnn'): share_dnn_layer = dnn.DNN( - self._model_config.share_dnn, - self._l2_reg, - 'share_dnn', - self._is_training, + self._model_config.share_dnn, + self._l2_reg, + 'share_dnn', + self._is_training, ) share_feature = share_dnn_layer(self._features) booster_dnn_layer = dnn.DNN( - self._model_config.booster_dnn, - self._l2_reg, - 'booster_dnn', - self._is_training, + self._model_config.booster_dnn, + self._l2_reg, + 'booster_dnn', + self._is_training, ) - light_dnn_layer = dnn.DNN(self._model_config.light_dnn, self._l2_reg, 'light_dnn', self._is_training) + light_dnn_layer = dnn.DNN(self._model_config.light_dnn, self._l2_reg, + 'light_dnn', self._is_training) if self._model_config.HasField('share_dnn'): - self.booster_feature = booster_dnn_layer(share_feature, self.hidden_layer_feature_output) + self.booster_feature = booster_dnn_layer(share_feature, + self.hidden_layer_feature_output) input_embedding_stop_gradient = tf.stop_gradient(share_feature) - self.light_feature = light_dnn_layer(input_embedding_stop_gradient, self.hidden_layer_feature_output) + self.light_feature = light_dnn_layer(input_embedding_stop_gradient, + self.hidden_layer_feature_output) else: - self.booster_feature = booster_dnn_layer(self._features, self.hidden_layer_feature_output) + self.booster_feature = booster_dnn_layer(self._features, + self.hidden_layer_feature_output) input_embedding_stop_gradient = tf.stop_gradient(self._features) - self.light_feature = light_dnn_layer(input_embedding_stop_gradient, self.hidden_layer_feature_output) + self.light_feature = light_dnn_layer(input_embedding_stop_gradient, + self.hidden_layer_feature_output) if self._model_config.feature_based_distillation: booster_out = tf.layers.dense( - self.booster_feature['hidden_layer_end'], - self._num_class, - kernel_regularizer=self._l2_reg, - name='booster_output', + self.booster_feature['hidden_layer_end'], + self._num_class, + kernel_regularizer=self._l2_reg, + name='booster_output', ) light_out = tf.layers.dense( - self.light_feature['hidden_layer_end'], - self._num_class, - kernel_regularizer=self._l2_reg, - name='light_output', + self.light_feature['hidden_layer_end'], + self._num_class, + kernel_regularizer=self._l2_reg, + name='light_output', ) else: booster_out = tf.layers.dense( - self.booster_feature, - self._num_class, - kernel_regularizer=self._l2_reg, - name='booster_output', + self.booster_feature, + self._num_class, + kernel_regularizer=self._l2_reg, + name='booster_output', ) light_out = tf.layers.dense( - self.light_feature, - self._num_class, - kernel_regularizer=self._l2_reg, - name='light_output', + self.light_feature, + self._num_class, + kernel_regularizer=self._l2_reg, + name='light_output', ) self._prediction_dict.update( - self._output_to_prediction_impl( - booster_out, - self._loss_type, - num_class=self._num_class, - suffix='_booster', - ) - ) + self._output_to_prediction_impl( + booster_out, + self._loss_type, + num_class=self._num_class, + suffix='_booster', + )) self._prediction_dict.update( - self._output_to_prediction_impl(light_out, self._loss_type, num_class=self._num_class, suffix='_light') - ) + self._output_to_prediction_impl( + light_out, + self._loss_type, + num_class=self._num_class, + suffix='_light')) return self._prediction_dict @@ -129,39 +148,37 @@ def build_loss_graph(self): for i, unit_i in enumerate(light_hidden_units): for j, unit_j in enumerate(booster_hidden_units): if light_hidden_units[i] == booster_hidden_units[j]: - self._prediction_dict['similarity_' + str(count)] = self.feature_based_sim( - self._model_config.feature_based_distillation, i, j - ) + self._prediction_dict[ + 'similarity_' + str(count)] = self.feature_based_sim( + self._model_config.feature_based_distillation, i, j) count += 1 break self._loss_dict.update( - self._build_loss_impl( - LossType.CLASSIFICATION, - label_name=self._label_name, - loss_weight=self._sample_weight, - num_class=self._num_class, - suffix='_booster', - ) - ) + self._build_loss_impl( + LossType.CLASSIFICATION, + label_name=self._label_name, + loss_weight=self._sample_weight, + num_class=self._num_class, + suffix='_booster', + )) self._loss_dict.update( - self._build_loss_impl( - LossType.CLASSIFICATION, - label_name=self._label_name, - loss_weight=self._sample_weight, - num_class=self._num_class, - suffix='_light', - ) - ) + self._build_loss_impl( + LossType.CLASSIFICATION, + label_name=self._label_name, + loss_weight=self._sample_weight, + num_class=self._num_class, + suffix='_light', + )) booster_logits_no_grad = tf.stop_gradient(logits_booster) self._loss_dict['hint_loss'] = loss_builder.build( - LossType.L2_LOSS, - label=booster_logits_no_grad, - pred=logits_light, - loss_weight=self._sample_weight, + LossType.L2_LOSS, + label=booster_logits_no_grad, + pred=logits_light, + loss_weight=self._sample_weight, ) if self._model_config.feature_based_distillation: @@ -176,27 +193,29 @@ def build_metric_graph(self, eval_config): metric_dict = {} for metric in eval_config.metrics_set: metric_dict.update( - self._build_metric_impl( - metric, - loss_type=LossType.CLASSIFICATION, - label_name=self._label_name, - num_class=self._num_class, - suffix='_light', - ) - ) + self._build_metric_impl( + metric, + loss_type=LossType.CLASSIFICATION, + label_name=self._label_name, + num_class=self._num_class, + suffix='_light', + )) metric_dict.update( - self._build_metric_impl( - metric, - loss_type=LossType.CLASSIFICATION, - label_name=self._label_name, - num_class=self._num_class, - suffix='_booster', - ) - ) + self._build_metric_impl( + metric, + loss_type=LossType.CLASSIFICATION, + label_name=self._label_name, + num_class=self._num_class, + suffix='_booster', + )) return metric_dict def get_outputs(self): outputs = [] - outputs.extend(self._get_outputs_impl(self._loss_type, self._num_class, suffix='_light')) - outputs.extend(self._get_outputs_impl(self._loss_type, self._num_class, suffix='_booster')) + outputs.extend( + self._get_outputs_impl( + self._loss_type, self._num_class, suffix='_light')) + outputs.extend( + self._get_outputs_impl( + self._loss_type, self._num_class, suffix='_booster')) return outputs diff --git a/easy_rec/python/model/simple_multi_task.py b/easy_rec/python/model/simple_multi_task.py index 00527b47a..8215a7b4c 100644 --- a/easy_rec/python/model/simple_multi_task.py +++ b/easy_rec/python/model/simple_multi_task.py @@ -4,6 +4,7 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.multi_task_model import MultiTaskModel + from easy_rec.python.protos.simple_multi_task_pb2 import SimpleMultiTask as SimpleMultiTaskConfig # NOQA if tf.__version__ >= '2.0': @@ -11,12 +12,18 @@ class SimpleMultiTask(MultiTaskModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(SimpleMultiTask, self).__init__(model_config, feature_configs, features, labels, is_training) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(SimpleMultiTask, self).__init__(model_config, feature_configs, + features, labels, is_training) assert self._model_config.WhichOneof('model') == 'simple_multi_task', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.simple_multi_task assert isinstance(self._model_config, SimpleMultiTaskConfig) @@ -31,17 +38,17 @@ def build_predict_graph(self): for i, task_tower_cfg in enumerate(self._task_towers): tower_name = task_tower_cfg.tower_name task_dnn = dnn.DNN( - task_tower_cfg.dnn, - self._l2_reg, - name=tower_name, - is_training=self._is_training, + task_tower_cfg.dnn, + self._l2_reg, + name=tower_name, + is_training=self._is_training, ) task_fea = task_dnn(self._features) task_output = tf.layers.dense( - inputs=task_fea, - units=task_tower_cfg.num_class, - kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i, + inputs=task_fea, + units=task_tower_cfg.num_class, + kernel_regularizer=self._l2_reg, + name='dnn_output_%d' % i, ) tower_outputs[tower_name] = task_output diff --git a/easy_rec/python/model/uniter.py b/easy_rec/python/model/uniter.py index 3b8173979..2fd8c9c8b 100644 --- a/easy_rec/python/model/uniter.py +++ b/easy_rec/python/model/uniter.py @@ -2,8 +2,10 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.layers import dnn, uniter +from easy_rec.python.layers import dnn +from easy_rec.python.layers import uniter from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.uniter_pb2 import Uniter as UNITERConfig # NOQA if tf.__version__ >= '2.0': @@ -17,24 +19,30 @@ class Uniter(RankModel): https://arxiv.org/abs/1909.11740 """ - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(Uniter, self).__init__(model_config, feature_configs, features, labels, is_training) + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(Uniter, self).__init__(model_config, feature_configs, features, + labels, is_training) assert self._model_config.WhichOneof('model') == 'uniter', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model') - ) + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._uniter_layer = uniter.Uniter( - model_config, - feature_configs, - features, - self._model_config.uniter.config, - self._input_layer, + model_config, + feature_configs, + features, + self._model_config.uniter.config, + self._input_layer, ) self._model_config = self._model_config.uniter def build_predict_graph(self): hidden = self._uniter_layer(self._is_training, l2_reg=self._l2_reg) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, + 'final_dnn', self._is_training) all_fea = final_dnn_layer(hidden) final = tf.layers.dense(all_fea, self._num_class, name='output') diff --git a/easy_rec/python/model/wide_and_deep.py b/easy_rec/python/model/wide_and_deep.py index cc24a60c0..78b538523 100755 --- a/easy_rec/python/model/wide_and_deep.py +++ b/easy_rec/python/model/wide_and_deep.py @@ -6,6 +6,7 @@ from easy_rec.python.layers import dnn from easy_rec.python.model.rank_model import RankModel + from easy_rec.python.protos.wide_and_deep_pb2 import WideAndDeep as WideAndDeepConfig # NOQA if tf.__version__ >= '2.0': @@ -13,11 +14,19 @@ class WideAndDeep(RankModel): - def __init__(self, model_config, feature_configs, features, labels=None, is_training=False): - super(WideAndDeep, self).__init__(model_config, feature_configs, features, labels, is_training) - assert model_config.WhichOneof('model') == 'wide_and_deep', 'invalid model config: %s' % model_config.WhichOneof( - 'model' - ) + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(WideAndDeep, self).__init__(model_config, feature_configs, features, + labels, is_training) + assert model_config.WhichOneof( + 'model' + ) == 'wide_and_deep', 'invalid model config: %s' % model_config.WhichOneof( + 'model') self._model_config = model_config.wide_and_deep assert isinstance(self._model_config, WideAndDeepConfig) assert self._input_layer.has_group('wide') @@ -39,30 +48,37 @@ def build_predict_graph(self): logging.info('wide features dimension: %d' % wide_fea.get_shape()[-1]) self._deep_features = tf.concat(self._deep_features, axis=1) - logging.info('input deep features dimension: %d' % self._deep_features.get_shape()[-1]) + logging.info('input deep features dimension: %d' % + self._deep_features.get_shape()[-1]) - deep_layer = dnn.DNN(self._model_config.dnn, self._l2_reg, 'deep_feature', self._is_training) + deep_layer = dnn.DNN(self._model_config.dnn, self._l2_reg, 'deep_feature', + self._is_training) deep_fea = deep_layer(self._deep_features) - logging.info('output deep features dimension: %d' % deep_fea.get_shape()[-1]) + logging.info('output deep features dimension: %d' % + deep_fea.get_shape()[-1]) has_final = len(self._model_config.final_dnn.hidden_units) > 0 print('wide_deep has_final_dnn layers = %d' % has_final) if has_final: all_fea = tf.concat([wide_fea, deep_fea], axis=1) final_layer = dnn.DNN( - self._model_config.final_dnn, - self._l2_reg, - 'final_dnn', - self._is_training, + self._model_config.final_dnn, + self._l2_reg, + 'final_dnn', + self._is_training, ) all_fea = final_layer(all_fea) - output = tf.layers.dense(all_fea, self._num_class, kernel_regularizer=self._l2_reg, name='output') + output = tf.layers.dense( + all_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='output') else: deep_out = tf.layers.dense( - deep_fea, - self._num_class, - kernel_regularizer=self._l2_reg, - name='deep_out', + deep_fea, + self._num_class, + kernel_regularizer=self._l2_reg, + name='deep_out', ) output = deep_out + wide_fea @@ -82,17 +98,17 @@ def get_grouped_vars(self, opt_num): list of list of variables. """ assert opt_num <= 3, ( - 'could only support 2 or 3 optimizers, ' - + 'if opt_num = 2, one for the wide , and one for the others, ' - + 'if opt_num = 3, one for the wide, second for the deep embeddings, ' - + 'and third for the other layers.' - ) + 'could only support 2 or 3 optimizers, ' + + 'if opt_num = 2, one for the wide , and one for the others, ' + + 'if opt_num = 3, one for the wide, second for the deep embeddings, ' + + 'and third for the other layers.') if opt_num == 2: wide_vars = [] deep_vars = [] for tmp_var in tf.trainable_variables(): - if tmp_var.name.startswith('input_layer') and (not tmp_var.name.startswith('input_layer_1')): + if tmp_var.name.startswith('input_layer') and ( + not tmp_var.name.startswith('input_layer_1')): wide_vars.append(tmp_var) else: deep_vars.append(tmp_var) @@ -102,9 +118,11 @@ def get_grouped_vars(self, opt_num): embedding_vars = [] deep_vars = [] for tmp_var in tf.trainable_variables(): - if tmp_var.name.startswith('input_layer') and (not tmp_var.name.startswith('input_layer_1')): + if tmp_var.name.startswith('input_layer') and ( + not tmp_var.name.startswith('input_layer_1')): wide_vars.append(tmp_var) - elif tmp_var.name.startswith('input_layer') or '/embedding_weights' in tmp_var.name: + elif tmp_var.name.startswith( + 'input_layer') or '/embedding_weights' in tmp_var.name: embedding_vars.append(tmp_var) else: deep_vars.append(tmp_var) diff --git a/easy_rec/python/ops/gen_kafka_ops.py b/easy_rec/python/ops/gen_kafka_ops.py index a6bb716fd..4948d377e 100644 --- a/easy_rec/python/ops/gen_kafka_ops.py +++ b/easy_rec/python/ops/gen_kafka_ops.py @@ -14,7 +14,6 @@ from tensorflow.python.eager import context as _context from tensorflow.python.eager import core as _core from tensorflow.python.eager import execute as _execute - # Needed to trigger the call to _set_call_cpp_shape_fn. from tensorflow.python.framework import dtypes as _dtypes from tensorflow.python.framework import ops as _ops @@ -29,21 +28,22 @@ try: kafka_module = tf.load_op_library(kafka_ops_path) except Exception: - logging.warning('load %s failed: %s' % (kafka_ops_path, traceback.format_exc())) + logging.warning('load %s failed: %s' % + (kafka_ops_path, traceback.format_exc())) @tf_export('io_kafka_dataset_v2') def io_kafka_dataset_v2( - topics, - servers, - group, - eof, - timeout, - config_global, - config_topic, - message_key, - message_offset, - name=None, + topics, + servers, + group, + eof, + timeout, + config_global, + config_topic, + message_key, + message_offset, + name=None, ): """Creates a dataset that emits the messages of one or more Kafka topics. @@ -75,31 +75,31 @@ def io_kafka_dataset_v2( A `Tensor` of type `variant`. """ return kafka_module.io_kafka_dataset_v2( - topics=topics, - servers=servers, - group=group, - eof=eof, - timeout=timeout, - config_global=config_global, - config_topic=config_topic, - message_key=message_key, - message_offset=message_offset, - name=name, + topics=topics, + servers=servers, + group=group, + eof=eof, + timeout=timeout, + config_global=config_global, + config_topic=config_topic, + message_key=message_key, + message_offset=message_offset, + name=name, ) def io_kafka_dataset_eager_fallback( - topics, - servers, - group, - eof, - timeout, - config_global, - config_topic, - message_key, - message_offset, - name=None, - ctx=None, + topics, + servers, + group, + eof, + timeout, + config_global, + config_topic, + message_key, + message_offset, + name=None, + ctx=None, ): """This is the slowpath function for Eager mode. @@ -116,19 +116,26 @@ def io_kafka_dataset_eager_fallback( message_key = _ops.convert_to_tensor(message_key, _dtypes.bool) message_offset = _ops.convert_to_tensor(message_offset, _dtypes.bool) _inputs_flat = [ - topics, - servers, - group, - eof, - timeout, - config_global, - config_topic, - message_key, - message_offset, + topics, + servers, + group, + eof, + timeout, + config_global, + config_topic, + message_key, + message_offset, ] _attrs = None - _result = _execute.execute(b'IOKafkaDataset', 1, inputs=_inputs_flat, attrs=_attrs, ctx=_ctx, name=name) - _execute.record_gradient('IOKafkaDataset', _inputs_flat, _attrs, _result, name) + _result = _execute.execute( + b'IOKafkaDataset', + 1, + inputs=_inputs_flat, + attrs=_attrs, + ctx=_ctx, + name=name) + _execute.record_gradient('IOKafkaDataset', _inputs_flat, _attrs, _result, + name) (_result,) = _result return _result @@ -148,29 +155,32 @@ def io_write_kafka_v2(message, topic, servers, name=None): """ _ctx = _context._context if _ctx is None or not _ctx._eager_context.is_eager: - _op = kafka_module.io_write_kafka_v2(message=message, topic=topic, servers=servers, name=name) + _op = kafka_module.io_write_kafka_v2( + message=message, topic=topic, servers=servers, name=name) _result = _op.outputs[:] _inputs_flat = _op.inputs _attrs = None - _execute.record_gradient('IOWriteKafka', _inputs_flat, _attrs, _result, name) + _execute.record_gradient('IOWriteKafka', _inputs_flat, _attrs, _result, + name) (_result,) = _result return _result else: try: _result = _pywrap_tensorflow.TFE_Py_FastPathExecute( - _ctx._context_handle, - _ctx._eager_context.device_name, - 'IOWriteKafka', - name, - _ctx._post_execution_callbacks, - message, - topic, - servers, + _ctx._context_handle, + _ctx._eager_context.device_name, + 'IOWriteKafka', + name, + _ctx._post_execution_callbacks, + message, + topic, + servers, ) return _result except _core._FallbackException: - return io_write_kafka_eager_fallback(message, topic, servers, name=name, ctx=_ctx) + return io_write_kafka_eager_fallback( + message, topic, servers, name=name, ctx=_ctx) except _core._NotOkStatusException as e: if name is not None: message = e.message + ' name: ' + name @@ -190,7 +200,13 @@ def io_write_kafka_eager_fallback(message, topic, servers, name=None, ctx=None): servers = _ops.convert_to_tensor(servers, _dtypes.string) _inputs_flat = [message, topic, servers] _attrs = None - _result = _execute.execute(b'IOWriteKafka', 1, inputs=_inputs_flat, attrs=_attrs, ctx=_ctx, name=name) + _result = _execute.execute( + b'IOWriteKafka', + 1, + inputs=_inputs_flat, + attrs=_attrs, + ctx=_ctx, + name=name) _execute.record_gradient('IOWriteKafka', _inputs_flat, _attrs, _result, name) (_result,) = _result return _result diff --git a/easy_rec/python/ops/gen_str_avx_op.py b/easy_rec/python/ops/gen_str_avx_op.py index f46d8df81..ba57d41f1 100644 --- a/easy_rec/python/ops/gen_str_avx_op.py +++ b/easy_rec/python/ops/gen_str_avx_op.py @@ -21,8 +21,8 @@ def str_split_by_chr(input_str, sep, skip_empty): if constant.has_avx_str_split() and str_avx_op is not None: assert len(sep) == 1, 'invalid data_config.separator(%s) len(%d) != 1' % ( - sep, - len(sep), + sep, + len(sep), ) return str_avx_op.avx512_string_split(input_str, sep, skip_empty=skip_empty) else: diff --git a/easy_rec/python/ops/incr_record.py b/easy_rec/python/ops/incr_record.py index 18688df86..b4bad11e4 100644 --- a/easy_rec/python/ops/incr_record.py +++ b/easy_rec/python/ops/incr_record.py @@ -20,9 +20,11 @@ get_sparse_indices = None set_sparse_indices = None kv_resource_incr_gather = None - logging.warning('failed to import gen_io_ops.collect_sparse_indices: %s' % str(ex)) + logging.warning('failed to import gen_io_ops.collect_sparse_indices: %s' % + str(ex)) except Exception as ex: get_sparse_indices = None set_sparse_indices = None kv_resource_incr_gather = None - logging.warning('failed to import gen_io_ops.collect_sparse_indices: %s' % str(ex)) + logging.warning('failed to import gen_io_ops.collect_sparse_indices: %s' % + str(ex)) diff --git a/easy_rec/python/predict.py b/easy_rec/python/predict.py index 34ec7435b..3362ba7b4 100644 --- a/easy_rec/python/predict.py +++ b/easy_rec/python/predict.py @@ -9,51 +9,58 @@ from tensorflow.python.lib.io import file_io from easy_rec.python.inference.csv_predictor import CSVPredictor -from easy_rec.python.inference.hive_parquet_predictor import ( # NOQA - HiveParquetPredictor, -) from easy_rec.python.inference.hive_predictor import HivePredictor from easy_rec.python.inference.parquet_predictor import ParquetPredictor from easy_rec.python.inference.parquet_predictor_v2 import ParquetPredictorV2 from easy_rec.python.main import predict from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.utils import config_util, numpy_utils +from easy_rec.python.utils import config_util +from easy_rec.python.utils import numpy_utils from easy_rec.python.utils.hive_utils import HiveUtils +from easy_rec.python.inference.hive_parquet_predictor import ( # NOQA + HiveParquetPredictor,) + if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) tf.app.flags.DEFINE_string('input_path', None, 'predict data path') tf.app.flags.DEFINE_string('output_path', None, 'path to save predict result') tf.app.flags.DEFINE_integer('batch_size', 1024, help='batch size') -tf.app.flags.DEFINE_bool('with_header', False, 'whether the input csv file has header') +tf.app.flags.DEFINE_bool('with_header', False, + 'whether the input csv file has header') # predict by checkpoint -tf.app.flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') +tf.app.flags.DEFINE_string('pipeline_config_path', None, + 'Path to pipeline config ' + 'file.') tf.app.flags.DEFINE_string( - 'checkpoint_path', - None, - 'checkpoint to be evaled ' ' if not specified, use the latest checkpoint in ' 'train_config.model_dir', + 'checkpoint_path', + None, + 'checkpoint to be evaled ' + ' if not specified, use the latest checkpoint in ' + 'train_config.model_dir', ) tf.app.flags.DEFINE_string('model_dir', None, help='will update the model_dir') # predict by saved_model tf.app.flags.DEFINE_string('saved_model_dir', None, help='save model dir') tf.app.flags.DEFINE_string( - 'reserved_cols', - 'ALL_COLUMNS', - 'columns to keep from input table, they are separated with ,', + 'reserved_cols', + 'ALL_COLUMNS', + 'columns to keep from input table, they are separated with ,', ) tf.app.flags.DEFINE_string( - 'output_cols', - 'ALL_COLUMNS', - 'output columns, such as: score float. multiple columns are separated by ,', + 'output_cols', + 'ALL_COLUMNS', + 'output columns, such as: score float. multiple columns are separated by ,', ) -tf.app.flags.DEFINE_string('output_sep', chr(1), 'separator of predict result file') +tf.app.flags.DEFINE_string('output_sep', chr(1), + 'separator of predict result file') tf.app.flags.DEFINE_string('selected_cols', None, '') tf.app.flags.DEFINE_string('fg_json_path', '', '') tf.app.flags.DEFINE_string('ds_vector_recall', '', '') @@ -76,62 +83,65 @@ def main(argv): if FLAGS.pipeline_config_path: pipeline_config_path = FLAGS.pipeline_config_path else: - pipeline_config_path = config_util.search_pipeline_config(FLAGS.saved_model_dir) - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) + pipeline_config_path = config_util.search_pipeline_config( + FLAGS.saved_model_dir) + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path, False) data_config = pipeline_config.data_config input_type = get_input_type(FLAGS.input_type, data_config) if input_type in [data_config.HiveParquetInput, data_config.HiveInput]: all_cols, all_col_types = HiveUtils( - data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input, + data_config=pipeline_config.data_config, + hive_config=pipeline_config.hive_train_input, ).get_all_cols(FLAGS.input_path) if input_type == DatasetConfig.HiveParquetInput: predictor = HiveParquetPredictor( - FLAGS.saved_model_dir, - pipeline_config.data_config, - fg_json_path=FLAGS.fg_json_path, - hive_config=pipeline_config.hive_train_input, - output_sep=FLAGS.output_sep, - all_cols=all_cols, - all_col_types=all_col_types, + FLAGS.saved_model_dir, + pipeline_config.data_config, + fg_json_path=FLAGS.fg_json_path, + hive_config=pipeline_config.hive_train_input, + output_sep=FLAGS.output_sep, + all_cols=all_cols, + all_col_types=all_col_types, ) else: predictor = HivePredictor( - FLAGS.saved_model_dir, - pipeline_config.data_config, - fg_json_path=FLAGS.fg_json_path, - hive_config=pipeline_config.hive_train_input, - output_sep=FLAGS.output_sep, - all_cols=all_cols, - all_col_types=all_col_types, + FLAGS.saved_model_dir, + pipeline_config.data_config, + fg_json_path=FLAGS.fg_json_path, + hive_config=pipeline_config.hive_train_input, + output_sep=FLAGS.output_sep, + all_cols=all_cols, + all_col_types=all_col_types, ) elif input_type in [data_config.ParquetInput, data_config.ParquetInputV2]: predictor_cls = ParquetPredictor if input_type == data_config.ParquetInputV2: predictor_cls = ParquetPredictorV2 predictor = predictor_cls( - FLAGS.saved_model_dir, - pipeline_config.data_config, - ds_vector_recall=FLAGS.ds_vector_recall, - fg_json_path=FLAGS.fg_json_path, - selected_cols=FLAGS.selected_cols, - output_sep=FLAGS.output_sep, - pipeline_config=pipeline_config, + FLAGS.saved_model_dir, + pipeline_config.data_config, + ds_vector_recall=FLAGS.ds_vector_recall, + fg_json_path=FLAGS.fg_json_path, + selected_cols=FLAGS.selected_cols, + output_sep=FLAGS.output_sep, + pipeline_config=pipeline_config, ) elif input_type == data_config.CSVInput: predictor = CSVPredictor( - FLAGS.saved_model_dir, - pipeline_config.data_config, - FLAGS.with_header, - ds_vector_recall=FLAGS.ds_vector_recall, - fg_json_path=FLAGS.fg_json_path, - selected_cols=FLAGS.selected_cols, - output_sep=FLAGS.output_sep, + FLAGS.saved_model_dir, + pipeline_config.data_config, + FLAGS.with_header, + ds_vector_recall=FLAGS.ds_vector_recall, + fg_json_path=FLAGS.fg_json_path, + selected_cols=FLAGS.selected_cols, + output_sep=FLAGS.output_sep, ) else: assert False, 'invalid input type: %s' % input_class_map_r[input_type] - logging.info('input_path = %s, output_path = %s' % (FLAGS.input_path, FLAGS.output_path)) + logging.info('input_path = %s, output_path = %s' % + (FLAGS.input_path, FLAGS.output_path)) if 'TF_CONFIG' in os.environ: tf_config = json.loads(os.environ['TF_CONFIG']) worker_num = len(tf_config['cluster']['worker']) @@ -140,13 +150,13 @@ def main(argv): worker_num = 1 task_index = 0 predictor.predict_impl( - FLAGS.input_path, - FLAGS.output_path, - reserved_cols=FLAGS.reserved_cols, - output_cols=FLAGS.output_cols, - batch_size=FLAGS.batch_size, - slice_id=task_index, - slice_num=worker_num, + FLAGS.input_path, + FLAGS.output_path, + reserved_cols=FLAGS.reserved_cols, + output_cols=FLAGS.output_cols, + batch_size=FLAGS.batch_size, + slice_id=task_index, + slice_num=worker_num, ) else: logging.info('Predict by checkpoint_path.') @@ -160,7 +170,8 @@ def main(argv): else: pipeline_config_path = FLAGS.pipeline_config_path - pred_result = predict(pipeline_config_path, FLAGS.checkpoint_path, FLAGS.input_path) + pred_result = predict(pipeline_config_path, FLAGS.checkpoint_path, + FLAGS.input_path) if FLAGS.output_path is not None: logging.info('will save predict result to %s' % FLAGS.output_path) with tf.gfile.GFile(FLAGS.output_path, 'wb') as fout: diff --git a/easy_rec/python/test/csv_input_test.py b/easy_rec/python/test/csv_input_test.py index 90f615ac0..0b2e41c64 100644 --- a/easy_rec/python/test/csv_input_test.py +++ b/easy_rec/python/test/csv_input_test.py @@ -12,7 +12,8 @@ from easy_rec.python.input.csv_input_ex import CSVInputEx from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.protos.feature_config_pb2 import FeatureConfig -from easy_rec.python.utils import config_util, constant +from easy_rec.python.utils import config_util +from easy_rec.python.utils import constant from easy_rec.python.utils.test_utils import RunAsSubprocess if tf.__version__ >= '2.0': @@ -23,6 +24,7 @@ class CSVInputTest(tf.test.TestCase): + def __init__(self, methodName='CSVInputTest'): super(CSVInputTest, self).__init__(methodName=methodName) self._input_path = 'data/test/test.csv' @@ -70,7 +72,8 @@ def test_csv_data(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput(dataset_config, feature_configs, self._input_path).create_input() + train_input_fn = CSVInput(dataset_config, feature_configs, + self._input_path).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -78,9 +81,9 @@ def test_csv_data(self): init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=False, + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=False, ) with self.test_session(config=session_config) as sess: sess.run(init_op) @@ -133,7 +136,8 @@ def test_csv_data_flt_to_str_exception(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput(dataset_config, feature_configs, self._input_path).create_input() + train_input_fn = CSVInput(dataset_config, feature_configs, + self._input_path).create_input() try: dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) # noqa: F841 passed = True @@ -189,7 +193,8 @@ def test_csv_data_flt_to_str(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput(dataset_config, feature_configs, self._input_path).create_input() + train_input_fn = CSVInput(dataset_config, feature_configs, + self._input_path).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) @@ -199,9 +204,9 @@ def test_csv_data_flt_to_str(self): init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=False, + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=False, ) with self.test_session(config=session_config) as sess: sess.run(init_op) @@ -249,7 +254,8 @@ def test_csv_input_ex(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInputEx(dataset_config, feature_configs, self._input_path_with_quote).create_input() + train_input_fn = CSVInputEx(dataset_config, feature_configs, + self._input_path_with_quote).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -257,17 +263,17 @@ def test_csv_input_ex(self): init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=False, + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=False, ) with self.test_session(config=session_config) as sess: sess.run(init_op) feature_dict, label_dict = sess.run([features, labels]) @unittest.skipIf( - 'AVX_TEST' not in os.environ, - 'Only execute when avx512 instructions are supported', + 'AVX_TEST' not in os.environ, + 'Only execute when avx512 instructions are supported', ) @RunAsSubprocess def test_csv_input_ex_avx(self): @@ -318,7 +324,8 @@ def test_csv_data_ignore_error(self): tmp_config.CopyFrom(empty_config) tmp_config.input_names.append(tmp_name) feature_configs.append(tmp_config) - train_input_fn = CSVInput(dataset_config, feature_configs, self._input_path_with_quote).create_input() + train_input_fn = CSVInput(dataset_config, feature_configs, + self._input_path_with_quote).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -326,9 +333,9 @@ def test_csv_data_ignore_error(self): init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=False, + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=False, ) with self.test_session(config=session_config) as sess: sess.run(init_op) diff --git a/easy_rec/python/test/custom_early_stop_func.py b/easy_rec/python/test/custom_early_stop_func.py index 68de6a4e3..fbc2f3cba 100644 --- a/easy_rec/python/test/custom_early_stop_func.py +++ b/easy_rec/python/test/custom_early_stop_func.py @@ -13,8 +13,7 @@ def custom_early_stop_func(eval_results, func_param): val = metrics[metric_name] if val > tmp_thre: logging.info( - 'At step %s, metric "%s" has value %s, which is larger than %s, causing early stop.' - % (step, metric_name, val, tmp_thre) - ) + 'At step %s, metric "%s" has value %s, which is larger than %s, causing early stop.' + % (step, metric_name, val, tmp_thre)) return True return False diff --git a/easy_rec/python/test/dh_local_run.py b/easy_rec/python/test/dh_local_run.py index 497b34c24..f243c7d84 100644 --- a/easy_rec/python/test/dh_local_run.py +++ b/easy_rec/python/test/dh_local_run.py @@ -8,14 +8,14 @@ from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare +from easy_rec.python.utils import test_utils + from easy_rec.python.test.odps_test_util import ( # NOQA - OdpsOSSConfig, - delete_oss_path, - get_oss_bucket, + OdpsOSSConfig, delete_oss_path, get_oss_bucket, ) -from easy_rec.python.utils import test_utils -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') odps_oss_config = OdpsOSSConfig(script_path='./samples/dh_script') @@ -38,9 +38,9 @@ def test_datahub_train_eval(self): odps_cmd = OdpsCommand(odps_oss_config) self._success = test_utils.test_datahub_train_eval( - '%s/configs/deepfm.config' % odps_oss_config.temp_dir, - odps_oss_config, - self._test_dir, + '%s/configs/deepfm.config' % odps_oss_config.temp_dir, + odps_oss_config, + self._test_dir, ) odps_cmd.run_list(end) self.assertTrue(self._success) @@ -48,14 +48,24 @@ def test_datahub_train_eval(self): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--odps_config', type=str, default=None, help='odps config path') - parser.add_argument('--oss_config', type=str, default=None, help='ossutilconfig path') - parser.add_argument('--bucket_name', type=str, default=None, help='test oss bucket name') + parser.add_argument( + '--odps_config', type=str, default=None, help='odps config path') + parser.add_argument( + '--oss_config', type=str, default=None, help='ossutilconfig path') + parser.add_argument( + '--bucket_name', type=str, default=None, help='test oss bucket name') parser.add_argument('--arn', type=str, default=None, help='oss rolearn') - parser.add_argument('--odpscmd', type=str, default='odpscmd', help='odpscmd path') - parser.add_argument('--algo_project', type=str, default=None, help='algo project name') - parser.add_argument('--algo_res_project', type=str, default=None, help='algo resource project name') - parser.add_argument('--algo_version', type=str, default=None, help='algo version') + parser.add_argument( + '--odpscmd', type=str, default='odpscmd', help='odpscmd path') + parser.add_argument( + '--algo_project', type=str, default=None, help='algo project name') + parser.add_argument( + '--algo_res_project', + type=str, + default=None, + help='algo resource project name') + parser.add_argument( + '--algo_version', type=str, default=None, help='algo version') args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] @@ -81,8 +91,8 @@ def test_datahub_train_eval(self): odps_oss_config.bucket_name = args.bucket_name prepare(odps_oss_config) start = [ - 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql', + 'deep_fm/create_external_deepfm_table.sql', + 'deep_fm/create_inner_deepfm_table.sql', ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) @@ -91,10 +101,10 @@ def test_datahub_train_eval(self): tf.test.main() # delete oss path bucket = get_oss_bucket( - odps_oss_config.oss_key, - odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name, + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp diff --git a/easy_rec/python/test/embed_test.py b/easy_rec/python/test/embed_test.py index 8ef378d67..07e1bdede 100644 --- a/easy_rec/python/test/embed_test.py +++ b/easy_rec/python/test/embed_test.py @@ -11,13 +11,15 @@ from easy_rec.python.feature_column.feature_column import FeatureColumnParser from easy_rec.python.input.dummy_input import DummyInput from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.protos.feature_config_pb2 import FeatureConfig, WideOrDeep +from easy_rec.python.protos.feature_config_pb2 import FeatureConfig +from easy_rec.python.protos.feature_config_pb2 import WideOrDeep if tf.__version__ >= '2.0': tf = tf.compat.v1 class EmbedTest(tf.test.TestCase): + def test_raw_embed(self): # embedding variable is: # [[1, 2 ], @@ -61,7 +63,8 @@ def test_raw_embed(self): feature_configs = [feature_config] features = {'field1': tf.constant(['0.1,0.2,0.3,0.4,0.5'])} - dummy_input = DummyInput(data_config, feature_configs, '', input_vals=features) + dummy_input = DummyInput( + data_config, feature_configs, '', input_vals=features) field_dict, _ = dummy_input._build(tf.estimator.ModeKeys.TRAIN, {}) wide_and_deep_dict = {'field1': WideOrDeep.WIDE_AND_DEEP} @@ -126,13 +129,15 @@ def test_seq_multi_embed(self): feature_configs = [feature_config] features = {'field1': tf.constant(['0112', '132430'])} - dummy_input = DummyInput(data_config, feature_configs, '', input_vals=features) + dummy_input = DummyInput( + data_config, feature_configs, '', input_vals=features) field_dict, _ = dummy_input._build(tf.estimator.ModeKeys.TRAIN, {}) wide_and_deep_dict = {'field1': WideOrDeep.DEEP} fc_parser = FeatureColumnParser(feature_configs, wide_and_deep_dict) builder = feature_column._LazyBuilder(field_dict) - hist_embedding, hist_seq_len = fc_parser.sequence_columns['field1']._get_sequence_dense_tensor(builder) + hist_embedding, hist_seq_len = fc_parser.sequence_columns[ + 'field1']._get_sequence_dense_tensor(builder) init = tf.initialize_all_variables() with tf.Session() as sess: diff --git a/easy_rec/python/test/emr_run.py b/easy_rec/python/test/emr_run.py index 725a83b4e..0a514ccaf 100644 --- a/easy_rec/python/test/emr_run.py +++ b/easy_rec/python/test/emr_run.py @@ -11,14 +11,14 @@ from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare +from easy_rec.python.utils import test_utils + from easy_rec.python.test.odps_test_util import ( # NOQA - OdpsOSSConfig, - delete_oss_path, - get_oss_bucket, + OdpsOSSConfig, delete_oss_path, get_oss_bucket, ) -from easy_rec.python.utils import test_utils -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') odps_oss_config = OdpsOSSConfig(script_path='./samples/emr_script') @@ -28,7 +28,8 @@ class TestPipelineOnEmr(tf.test.TestCase): def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) - self._test_hdfs_dir = test_utils.get_hdfs_tmp_dir('hdfs://emr-header-1:9000/user/easy_rec/emr_test') + self._test_hdfs_dir = test_utils.get_hdfs_tmp_dir( + 'hdfs://emr-header-1:9000/user/easy_rec/emr_test') self._success = True logging.info('test hdfs dir: %s' % self._test_hdfs_dir) @@ -39,30 +40,30 @@ def tearDown(self): def test_deepfm_train_eval_export(self): start = [ - 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql', + 'deep_fm/create_external_deepfm_table.sql', + 'deep_fm/create_inner_deepfm_table.sql', ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) odps_cmd.run_list(start) self._success = test_utils.test_hdfs_train_eval( - '%s/configs/deepfm.config' % odps_oss_config.temp_dir, - '%s/yaml_config/train.paitf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir, + '%s/configs/deepfm.config' % odps_oss_config.temp_dir, + '%s/yaml_config/train.paitf.yaml' % odps_oss_config.temp_dir, + self._test_hdfs_dir, ) self.assertTrue(self._success) self._success = test_utils.test_hdfs_eval( - '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, - '%s/yaml_config/eval.tf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir, + '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, + '%s/yaml_config/eval.tf.yaml' % odps_oss_config.temp_dir, + self._test_hdfs_dir, ) self.assertTrue(self._success) self._success = test_utils.test_hdfs_export( - '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, - '%s/yaml_config/export.tf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir, + '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, + '%s/yaml_config/export.tf.yaml' % odps_oss_config.temp_dir, + self._test_hdfs_dir, ) self.assertTrue(self._success) @@ -71,14 +72,24 @@ def test_deepfm_train_eval_export(self): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--odps_config', type=str, default=None, help='odps config path') - parser.add_argument('--oss_config', type=str, default=None, help='ossutilconfig path') - parser.add_argument('--bucket_name', type=str, default=None, help='test oss bucket name') + parser.add_argument( + '--odps_config', type=str, default=None, help='odps config path') + parser.add_argument( + '--oss_config', type=str, default=None, help='ossutilconfig path') + parser.add_argument( + '--bucket_name', type=str, default=None, help='test oss bucket name') parser.add_argument('--arn', type=str, default=None, help='oss rolearn') - parser.add_argument('--odpscmd', type=str, default='odpscmd', help='odpscmd path') - parser.add_argument('--algo_project', type=str, default=None, help='algo project name') - parser.add_argument('--algo_res_project', type=str, default=None, help='algo resource project name') - parser.add_argument('--algo_version', type=str, default=None, help='algo version') + parser.add_argument( + '--odpscmd', type=str, default='odpscmd', help='odpscmd path') + parser.add_argument( + '--algo_project', type=str, default=None, help='algo project name') + parser.add_argument( + '--algo_res_project', + type=str, + default=None, + help='algo resource project name') + parser.add_argument( + '--algo_version', type=str, default=None, help='algo version') args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] for unk_arg in unknown_args: @@ -106,10 +117,10 @@ def test_deepfm_train_eval_export(self): tf.test.main() # delete oss path bucket = get_oss_bucket( - odps_oss_config.oss_key, - odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name, + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp diff --git a/easy_rec/python/test/eval_metric_test.py b/easy_rec/python/test/eval_metric_test.py index c7814a3d0..186cd8ced 100644 --- a/easy_rec/python/test/eval_metric_test.py +++ b/easy_rec/python/test/eval_metric_test.py @@ -14,6 +14,7 @@ class MetricsTest(tf.test.TestCase, parameterized.TestCase): + def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) @@ -44,13 +45,11 @@ def test_gauc_all_negative_label(self): score = sess.run(value_op) self.assertAlmostEqual(score, 0.0) - @parameterized.named_parameters( - [ + @parameterized.named_parameters([ ['_reduction_mean', 'mean', 0.5833333], ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6], - ] - ) + ]) @RunAsSubprocess def test_gauc(self, reduction, expected): from easy_rec.python.core.metrics import gauc @@ -61,31 +60,29 @@ def test_gauc(self, reduction, expected): value_op, update_op = gauc(labels, probs, uids, reduction=reduction) with tf.Session() as sess: sess.run( - update_op, - feed_dict={ - labels: [1, 0, 1, 1, 0], - probs: [0.9, 0.8, 0.7, 0.6, 0.5], - uids: [1, 1, 1, 1, 1], - }, + update_op, + feed_dict={ + labels: [1, 0, 1, 1, 0], + probs: [0.9, 0.8, 0.7, 0.6, 0.5], + uids: [1, 1, 1, 1, 1], + }, ) sess.run( - update_op, - feed_dict={ - labels: [1, 0, 0, 1], - probs: [0.9, 0.8, 0.7, 0.6], - uids: [2, 2, 2, 2], - }, + update_op, + feed_dict={ + labels: [1, 0, 0, 1], + probs: [0.9, 0.8, 0.7, 0.6], + uids: [2, 2, 2, 2], + }, ) score = sess.run(value_op) self.assertAlmostEqual(score, expected) - @parameterized.named_parameters( - [ + @parameterized.named_parameters([ ['_reduction_mean', 'mean', 0.5833333], ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6], - ] - ) + ]) @RunAsSubprocess def test_session_auc(self, reduction, expected): from easy_rec.python.core.metrics import session_auc @@ -93,23 +90,24 @@ def test_session_auc(self, reduction, expected): labels = tf.placeholder(dtype=tf.int32, shape=(None,)) probs = tf.placeholder(dtype=tf.float32, shape=(None,)) session_ids = tf.placeholder(dtype=tf.int32, shape=(None,)) - value_op, update_op = session_auc(labels, probs, session_ids, reduction=reduction) + value_op, update_op = session_auc( + labels, probs, session_ids, reduction=reduction) with tf.Session() as sess: sess.run( - update_op, - feed_dict={ - labels: [1, 0, 1, 1, 0], - probs: [0.9, 0.8, 0.7, 0.6, 0.5], - session_ids: [1, 1, 1, 1, 1], - }, + update_op, + feed_dict={ + labels: [1, 0, 1, 1, 0], + probs: [0.9, 0.8, 0.7, 0.6, 0.5], + session_ids: [1, 1, 1, 1, 1], + }, ) sess.run( - update_op, - feed_dict={ - labels: [1, 0, 0, 1], - probs: [0.9, 0.8, 0.7, 0.6], - session_ids: [2, 2, 2, 2], - }, + update_op, + feed_dict={ + labels: [1, 0, 0, 1], + probs: [0.9, 0.8, 0.7, 0.6], + session_ids: [2, 2, 2, 2], + }, ) score = sess.run(value_op) self.assertAlmostEqual(score, expected) diff --git a/easy_rec/python/test/excel_convert_test.py b/easy_rec/python/test/excel_convert_test.py index 479202383..3e83d18b5 100644 --- a/easy_rec/python/test/excel_convert_test.py +++ b/easy_rec/python/test/excel_convert_test.py @@ -10,6 +10,7 @@ class ExcelConvertTest(tf.test.TestCase): + def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) @@ -20,42 +21,42 @@ def test_deepfm_convert(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'avazu_deepfm_excel.config') - convert_cmd = ( - """ + convert_cmd = (""" python -m easy_rec.python.tools.create_config_from_excel --excel_path samples/excel_config/dwd_avazu_ctr_deepfm.xls --model_type deepfm --output_path %s --train_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv --eval_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv - """ - % pipeline_config_path - ) - proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) + """ % pipeline_config_path) + proc = test_utils.run_cmd(convert_cmd, + '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) + self.assertTrue( + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) def test_multi_tower_convert(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'avazu_deepfm_excel.config') - convert_cmd = ( - """ + convert_cmd = (""" python -m easy_rec.python.tools.create_config_from_excel --excel_path samples/excel_config/dwd_avazu_ctr_deepfm.xls --model_type multi_tower --output_path %s --train_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv --eval_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv - """ - % pipeline_config_path - ) - proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) + """ % pipeline_config_path) + proc = test_utils.run_cmd(convert_cmd, + '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) + self.assertTrue( + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) diff --git a/easy_rec/python/test/export_test.py b/easy_rec/python/test/export_test.py index 5f65c8abd..1761df360 100644 --- a/easy_rec/python/test/export_test.py +++ b/easy_rec/python/test/export_test.py @@ -14,11 +14,13 @@ import easy_rec from easy_rec.python.inference.predictor import Predictor -from easy_rec.python.utils import config_util, test_utils +from easy_rec.python.utils import config_util +from easy_rec.python.utils import test_utils from easy_rec.python.utils.test_utils import RunAsSubprocess class ExportTest(tf.test.TestCase): + def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) @@ -27,13 +29,13 @@ def tearDown(self): @RunAsSubprocess def _predict_and_check( - self, - data_path, - saved_model_dir, - cmp_result, - keys=['probs'], - separator=',', - tol=1e-4, + self, + data_path, + saved_model_dir, + cmp_result, + keys=['probs'], + separator=',', + tol=1e-4, ): predictor = Predictor(saved_model_dir) with open(data_path, 'r') as fin: @@ -52,9 +54,9 @@ def _predict_and_check( val1 = cmp_result[i][key] diff = np.max(np.abs(val0 - val1)) assert diff < tol, 'too much difference: %.6f for %s, tol=%.6f' % ( - diff, - key, - tol, + diff, + key, + tol, ) def _extract_data(self, input_path, output_path, offset=1, separator=','): @@ -76,38 +78,42 @@ def _extract_rtp_data(self, input_path, output_path, separator=';'): fout.write('%s\n' % line_toks[-1]) def test_multi_tower(self): - self._export_test('samples/model_config/multi_tower_export.config', self._extract_data) + self._export_test('samples/model_config/multi_tower_export.config', + self._extract_data) def test_filter_input(self): - self._export_test('samples/model_config/export_filter_input.config', self._extract_data) + self._export_test('samples/model_config/export_filter_input.config', + self._extract_data) def test_mmoe(self): self._export_test( - 'samples/model_config/mmoe_on_taobao.config', - functools.partial(self._extract_data, offset=2), - keys=['probs_ctr', 'probs_cvr'], + 'samples/model_config/mmoe_on_taobao.config', + functools.partial(self._extract_data, offset=2), + keys=['probs_ctr', 'probs_cvr'], ) def test_fg(self): self._export_test( - 'samples/model_config/taobao_fg.config', - self._extract_rtp_data, - separator='', + 'samples/model_config/taobao_fg.config', + self._extract_rtp_data, + separator='', ) def test_fg_export(self): self._export_test( - 'samples/model_config/taobao_fg_export.config', - self._extract_rtp_data, - separator='', - test_multi=False, + 'samples/model_config/taobao_fg_export.config', + self._extract_rtp_data, + separator='', + test_multi=False, ) def test_export_with_asset(self): pipeline_config_path = 'samples/model_config/taobao_fg.config' test_dir = test_utils.get_tmp_dir() # prepare model - self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) + self.assertTrue( + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir)) test_utils.set_gpu_id(None) config_path = os.path.join(test_dir, 'pipeline.config') export_dir = os.path.join(test_dir, 'export/') @@ -118,10 +124,11 @@ def test_export_with_asset(self): --asset_files fg.json:samples/model_config/taobao_fg.json --export_done_file ExportDone """ % ( - config_path, - export_dir, + config_path, + export_dir, ) - proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) + proc = test_utils.run_cmd(export_cmd, + '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() self.assertTrue(proc.returncode == 0) files = gfile.Glob(export_dir + '*') @@ -145,56 +152,60 @@ def _post_check_func(pipeline_config): --checkpoint_path %s --export_dir %s """ % ( - pipeline_config_path, - ckpt_path, - export_dir, + pipeline_config_path, + ckpt_path, + export_dir, ) - proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) + proc = test_utils.run_cmd(export_cmd, + '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() return proc.returncode == 0 # prepare model self.assertTrue( - test_utils.test_single_train_eval( - pipeline_config_path, - test_dir=test_dir, - post_check_func=_post_check_func, - ) - ) + test_utils.test_single_train_eval( + pipeline_config_path, + test_dir=test_dir, + post_check_func=_post_check_func, + )) def test_multi_class_predict(self): self._export_test( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - extract_data_func=self._extract_data, - keys=['probs', 'logits', 'probs_y', 'logits_y', 'y'], + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + extract_data_func=self._extract_data, + keys=['probs', 'logits', 'probs_y', 'logits_y', 'y'], ) def _export_test( - self, - pipeline_config_path, - extract_data_func=None, - separator=',', - keys=['probs'], - test_multi=True, + self, + pipeline_config_path, + extract_data_func=None, + separator=',', + keys=['probs'], + test_multi=True, ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) # prepare model - self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) + self.assertTrue( + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir)) test_utils.set_gpu_id(None) # prepare two version config config_path_single = os.path.join(test_dir, 'pipeline.config') config_path_multi = os.path.join(test_dir, 'pipeline_v2.config') - pipeline_config = config_util.get_configs_from_pipeline_file(config_path_single) + pipeline_config = config_util.get_configs_from_pipeline_file( + config_path_single) if pipeline_config.export_config.multi_placeholder: config_path_single, config_path_multi = ( - config_path_multi, - config_path_single, + config_path_multi, + config_path_single, ) pipeline_config.export_config.multi_placeholder = not pipeline_config.export_config.multi_placeholder - config_util.save_pipeline_config(pipeline_config, test_dir, 'pipeline_v2.config') + config_util.save_pipeline_config(pipeline_config, test_dir, + 'pipeline_v2.config') # prepare two version export dir export_dir_single = os.path.join(test_dir, 'train/export/final') @@ -204,10 +215,11 @@ def _export_test( --pipeline_config_path %s --export_dir %s """ % ( - config_path_multi, - export_dir_multi, + config_path_multi, + export_dir_multi, ) - proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) + proc = test_utils.run_cmd(export_cmd, + '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() self.assertTrue(proc.returncode == 0) @@ -218,10 +230,11 @@ def _export_test( --pipeline_config_path %s --output_path %s """ % ( - config_path_single, - result_path, + config_path_single, + result_path, ) - proc = test_utils.run_cmd(predict_cmd % (), '%s/log_%s.txt' % (test_dir, 'predict')) + proc = test_utils.run_cmd(predict_cmd % (), + '%s/log_%s.txt' % (test_dir, 'predict')) proc.wait() self.assertTrue(proc.returncode == 0) with open(result_path, 'r') as fin: @@ -236,28 +249,28 @@ def _export_test( extract_data_func(test_data_path, tmp_data_path) test_data_path = tmp_data_path self._predict_and_check( - test_data_path, - export_dir_single, - cmp_result, - keys=keys, - separator=separator, - ) - if test_multi: - self._predict_and_check( test_data_path, - export_dir_multi, + export_dir_single, cmp_result, keys=keys, separator=separator, + ) + if test_multi: + self._predict_and_check( + test_data_path, + export_dir_multi, + cmp_result, + keys=keys, + separator=separator, ) test_utils.clean_up(test_dir) def _test_big_model_export( - self, - pipeline_config_path, - test_data_path, - extract_data_func=None, - total_steps=50, + self, + pipeline_config_path, + test_data_path, + extract_data_func=None, + total_steps=50, ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) @@ -266,7 +279,9 @@ def _test_big_model_export( tf.load_op_library(lookup_op_path) # prepare model - self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir, total_steps=total_steps)) + self.assertTrue( + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir, total_steps=total_steps)) test_utils.set_gpu_id(None) # the pipeline.config is produced by the prepare model cmd @@ -283,13 +298,14 @@ def _test_big_model_export( --redis_write_kv 1 --verbose 1 """ % ( - config_path, - export_dir, - test_data_path, - os.environ['redis_url'], - os.environ['redis_passwd'], + config_path, + export_dir, + test_data_path, + os.environ['redis_url'], + os.environ['redis_passwd'], ) - proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) + proc = test_utils.run_cmd(export_cmd, + '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() self.assertTrue(proc.returncode == 0) @@ -305,11 +321,12 @@ def _test_big_model_export( --input_path %s --output_path %s """ % ( - config_path, - test_data_path, - result_path, + config_path, + test_data_path, + result_path, ) - proc = test_utils.run_cmd(predict_cmd % (), '%s/log_%s.txt' % (test_dir, 'predict')) + proc = test_utils.run_cmd(predict_cmd % (), + '%s/log_%s.txt' % (test_dir, 'predict')) proc.wait() self.assertTrue(proc.returncode == 0) with open(result_path, 'r') as fin: @@ -325,105 +342,109 @@ def _test_big_model_export( self._predict_and_check(test_data_path, export_dir, cmp_result) @unittest.skipIf( - 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd', + 'redis_url' not in os.environ, + 'Only execute when redis is available: redis_url, redis_passwd', ) def test_big_model_export(self): pipeline_config_path = 'samples/model_config/multi_tower_export.config' test_data_path = 'data/test/export/data.csv' - self._test_big_model_export(pipeline_config_path, test_data_path, extract_data_func=self._extract_data) + self._test_big_model_export( + pipeline_config_path, + test_data_path, + extract_data_func=self._extract_data) @unittest.skipIf( - 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd', + 'redis_url' not in os.environ, + 'Only execute when redis is available: redis_url, redis_passwd', ) def test_big_model_deepfm_export(self): pipeline_config_path = 'samples/model_config/deepfm_combo_on_avazu_ctr.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' - self._test_big_model_export(pipeline_config_path, test_data_path, extract_data_func=self._extract_data) + self._test_big_model_export( + pipeline_config_path, + test_data_path, + extract_data_func=self._extract_data) @unittest.skipIf( - 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd', + 'redis_url' not in os.environ, + 'Only execute when redis is available: redis_url, redis_passwd', ) def test_big_model_din_export(self): pipeline_config_path = 'samples/model_config/din_on_taobao.config' test_data_path = 'data/test/tb_data/taobao_test_data' self._test_big_model_export( - pipeline_config_path, - test_data_path, - extract_data_func=functools.partial(self._extract_data, offset=2), + pipeline_config_path, + test_data_path, + extract_data_func=functools.partial(self._extract_data, offset=2), ) @unittest.skipIf( - 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd', + 'redis_url' not in os.environ, + 'Only execute when redis is available: redis_url, redis_passwd', ) def test_big_model_wide_and_deep_export(self): pipeline_config_path = 'samples/model_config/wide_and_deep_two_opti.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' self._test_big_model_export( - pipeline_config_path, - test_data_path, - extract_data_func=functools.partial(self._extract_data), + pipeline_config_path, + test_data_path, + extract_data_func=functools.partial(self._extract_data), ) @unittest.skipIf( - 'redis_url' not in os.environ or '-PAI' not in tf.__version__, - 'Only execute when pai-tf and redis is available: redis_url, redis_passwd', + 'redis_url' not in os.environ or '-PAI' not in tf.__version__, + 'Only execute when pai-tf and redis is available: redis_url, redis_passwd', ) def test_big_model_embedding_variable_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev.config' test_data_path = 'data/test/rtp/taobao_valid_feature.txt' self._test_big_model_export( - pipeline_config_path, - test_data_path, - self._extract_rtp_data, - total_steps=1000, + pipeline_config_path, + test_data_path, + self._extract_rtp_data, + total_steps=1000, ) @unittest.skipIf( - 'oss_endpoint' not in os.environ - or 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ - or 'oss_path' not in os.environ - or '-PAI' not in tf.__version__, - 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' 'and pai-tf is available.', + 'oss_endpoint' not in os.environ or 'oss_ak' not in os.environ or + 'oss_sk' not in os.environ or 'oss_path' not in os.environ or + '-PAI' not in tf.__version__, + 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' + 'and pai-tf is available.', ) def test_big_model_embedding_variable_oss_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev.config' test_data_path = 'data/test/rtp/taobao_valid_feature.txt' self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - self._extract_rtp_data, - total_steps=100, + pipeline_config_path, + test_data_path, + self._extract_rtp_data, + total_steps=100, ) @unittest.skipIf( - 'oss_endpoint' not in os.environ - or 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ - or 'oss_path' not in os.environ - or '-PAI' not in tf.__version__, - 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' 'and pai-tf is available.', + 'oss_endpoint' not in os.environ or 'oss_ak' not in os.environ or + 'oss_sk' not in os.environ or 'oss_path' not in os.environ or + '-PAI' not in tf.__version__, + 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' + 'and pai-tf is available.', ) def test_big_model_embedding_variable_v2_oss_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev_v2.config' test_data_path = 'data/test/rtp/taobao_valid_feature.txt' self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - self._extract_rtp_data, - total_steps=100, + pipeline_config_path, + test_data_path, + self._extract_rtp_data, + total_steps=100, ) def _test_big_model_export_to_oss( - self, - pipeline_config_path, - test_data_path, - extract_data_func=None, - total_steps=50, + self, + pipeline_config_path, + test_data_path, + extract_data_func=None, + total_steps=50, ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) @@ -432,7 +453,9 @@ def _test_big_model_export_to_oss( tf.load_op_library(lookup_op_path) # prepare model - self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir, total_steps=total_steps)) + self.assertTrue( + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir, total_steps=total_steps)) test_utils.set_gpu_id(None) # the pipeline.config is produced by the prepare model cmd @@ -451,15 +474,16 @@ def _test_big_model_export_to_oss( --oss_write_kv 1 --verbose 1 """ % ( - config_path, - export_dir, - test_data_path, - os.environ['oss_path'], - os.environ['oss_endpoint'], - os.environ['oss_ak'], - os.environ['oss_sk'], + config_path, + export_dir, + test_data_path, + os.environ['oss_path'], + os.environ['oss_endpoint'], + os.environ['oss_ak'], + os.environ['oss_sk'], ) - proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) + proc = test_utils.run_cmd(export_cmd, + '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() self.assertTrue(proc.returncode == 0) @@ -475,11 +499,12 @@ def _test_big_model_export_to_oss( --input_path %s --output_path %s """ % ( - config_path, - test_data_path, - result_path, + config_path, + test_data_path, + result_path, ) - proc = test_utils.run_cmd(predict_cmd, '%s/log_%s.txt' % (test_dir, 'predict')) + proc = test_utils.run_cmd(predict_cmd, + '%s/log_%s.txt' % (test_dir, 'predict')) proc.wait() self.assertTrue(proc.returncode == 0) with open(result_path, 'r') as fin: @@ -495,47 +520,53 @@ def _test_big_model_export_to_oss( self._predict_and_check(test_data_path, export_dir, cmp_result) @unittest.skipIf( - 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', + 'oss_path' not in os.environ, + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_export_to_oss(self): pipeline_config_path = 'samples/model_config/multi_tower_export.config' test_data_path = 'data/test/export/data.csv' - self._test_big_model_export_to_oss(pipeline_config_path, test_data_path, extract_data_func=self._extract_data) + self._test_big_model_export_to_oss( + pipeline_config_path, + test_data_path, + extract_data_func=self._extract_data) @unittest.skipIf( - 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', + 'oss_path' not in os.environ, + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_deepfm_export_to_oss(self): pipeline_config_path = 'samples/model_config/deepfm_combo_on_avazu_ctr.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' - self._test_big_model_export_to_oss(pipeline_config_path, test_data_path, extract_data_func=self._extract_data) + self._test_big_model_export_to_oss( + pipeline_config_path, + test_data_path, + extract_data_func=self._extract_data) @unittest.skipIf( - 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', + 'oss_path' not in os.environ, + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_din_export_to_oss(self): pipeline_config_path = 'samples/model_config/din_on_taobao.config' test_data_path = 'data/test/tb_data/taobao_test_data' self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - extract_data_func=functools.partial(self._extract_data, offset=2), + pipeline_config_path, + test_data_path, + extract_data_func=functools.partial(self._extract_data, offset=2), ) @unittest.skipIf( - 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', + 'oss_path' not in os.environ, + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_wide_and_deep_export_to_oss(self): pipeline_config_path = 'samples/model_config/wide_and_deep_two_opti.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' self._test_big_model_export_to_oss( - pipeline_config_path, - test_data_path, - extract_data_func=functools.partial(self._extract_data), + pipeline_config_path, + test_data_path, + extract_data_func=functools.partial(self._extract_data), ) diff --git a/easy_rec/python/test/fg_test.py b/easy_rec/python/test/fg_test.py index 4d3ead271..efbce46ea 100644 --- a/easy_rec/python/test/fg_test.py +++ b/easy_rec/python/test/fg_test.py @@ -6,13 +6,16 @@ import tensorflow as tf from google.protobuf import text_format -from easy_rec.python.utils import config_util, fg_util, test_utils +from easy_rec.python.utils import config_util +from easy_rec.python.utils import fg_util +from easy_rec.python.utils import test_utils if tf.__version__ >= '2.0': tf = tf.compat.v1 class FGTest(tf.test.TestCase): + def __init__(self, methodName='FGTest'): super(FGTest, self).__init__(methodName=methodName) @@ -32,28 +35,34 @@ def test_fg_json_to_config(self): final_pipeline_config_path = 'samples/rtp_fg/fg_test_extensions_final.config' fg_path = 'samples/rtp_fg/fg_test_extensions.json' - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path) pipeline_config.fg_json_path = fg_path fg_util.load_fg_json_to_config(pipeline_config) - pipeline_config_str = text_format.MessageToString(pipeline_config, as_utf8=True) + pipeline_config_str = text_format.MessageToString( + pipeline_config, as_utf8=True) - final_pipeline_config = config_util.get_configs_from_pipeline_file(final_pipeline_config_path) - final_pipeline_config_str = text_format.MessageToString(final_pipeline_config, as_utf8=True) + final_pipeline_config = config_util.get_configs_from_pipeline_file( + final_pipeline_config_path) + final_pipeline_config_str = text_format.MessageToString( + final_pipeline_config, as_utf8=True) self.assertEqual(pipeline_config_str, final_pipeline_config_str) def test_fg_dtype(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg_test_dtype.config', self._test_dir - ) + 'samples/model_config/taobao_fg_test_dtype.config', self._test_dir) self.assertTrue(self._success) def test_fg_train(self): - self._success = test_utils.test_single_train_eval('samples/model_config/fg_train.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/fg_train.config', self._test_dir) self.assertTrue(self._success) - @unittest.skipIf('-PAI' not in tf.__version__, 'Only test when pai-tf is used.') + @unittest.skipIf('-PAI' not in tf.__version__, + 'Only test when pai-tf is used.') def test_fg_train_ev(self): - self._success = test_utils.test_single_train_eval('samples/model_config/fg_train_ev.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/fg_train_ev.config', self._test_dir) self.assertTrue(self._success) diff --git a/easy_rec/python/test/hive_input_test.py b/easy_rec/python/test/hive_input_test.py index cb320dcb7..bce5f8cd8 100644 --- a/easy_rec/python/test/hive_input_test.py +++ b/easy_rec/python/test/hive_input_test.py @@ -14,7 +14,8 @@ from easy_rec.python.protos.feature_config_pb2 import FeatureConfig from easy_rec.python.protos.hive_config_pb2 import HiveConfig from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig -from easy_rec.python.utils import config_util, test_utils +from easy_rec.python.utils import config_util +from easy_rec.python.utils import test_utils if tf.__version__ >= '2.0': import tensorflow.compat.v1 as tf @@ -29,6 +30,7 @@ class HiveInputTest(tf.test.TestCase): + def _init_config(self): hive_host = os.environ['hive_host'] hive_username = os.environ['hive_username'] @@ -59,11 +61,10 @@ def __init__(self, methodName='HiveInputTest'): super(HiveInputTest, self).__init__(methodName=methodName) @unittest.skipIf( - 'hive_host' not in os.environ - or 'hive_username' not in os.environ - or 'hive_table_name' not in os.environ - or 'hive_hash_fields' not in os.environ, - """Only execute hive_config var are specified,hive_host、 + 'hive_host' not in os.environ or 'hive_username' not in os.environ or + 'hive_table_name' not in os.environ or + 'hive_hash_fields' not in os.environ, + """Only execute hive_config var are specified,hive_host、 hive_username、hive_table_name、hive_hash_fields is available.""", ) def test_hive_input(self): @@ -232,7 +233,8 @@ def test_hive_input(self): empty_config.input_names.pop() while len(empty_config.shared_names) > 0: empty_config.shared_names.pop() - train_input_fn = HiveInput(dataset_config, feature_configs, self.hive_train_input_config).create_input() + train_input_fn = HiveInput(dataset_config, feature_configs, + self.hive_train_input_config).create_input() dataset = train_input_fn(mode=tf.estimator.ModeKeys.TRAIN) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) @@ -240,9 +242,9 @@ def test_hive_input(self): init_op = tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS) gpu_options = tf.GPUOptions(allow_growth=True) session_config = tf.ConfigProto( - gpu_options=gpu_options, - allow_soft_placement=True, - log_device_placement=False, + gpu_options=gpu_options, + allow_soft_placement=True, + log_device_placement=False, ) with self.test_session(config=session_config) as sess: sess.run(init_op) @@ -255,11 +257,10 @@ def test_hive_input(self): return 0 @unittest.skipIf( - 'hive_host' not in os.environ - or 'hive_username' not in os.environ - or 'hive_table_name' not in os.environ - or 'hive_hash_fields' not in os.environ, - """Only execute hive_config var are specified,hive_host、 + 'hive_host' not in os.environ or 'hive_username' not in os.environ or + 'hive_table_name' not in os.environ or + 'hive_hash_fields' not in os.environ, + """Only execute hive_config var are specified,hive_host、 hive_username、hive_table_name、hive_hash_fields is available.""", ) def test_mmoe(self): @@ -278,7 +279,8 @@ def test_mmoe(self): if isinstance(pipeline_config_path, EasyRecConfig): pipeline_config = pipeline_config_path else: - pipeline_config = test_utils._load_config_for_test(pipeline_config_path, self._test_dir) + pipeline_config = test_utils._load_config_for_test( + pipeline_config_path, self._test_dir) pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -288,10 +290,11 @@ def test_mmoe(self): test_pipeline_config_path = os.path.join(self._test_dir, 'pipeline.config') hyperparam_str = '' train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path %s %s' % ( - test_pipeline_config_path, - hyperparam_str, + test_pipeline_config_path, + hyperparam_str, ) - proc = test_utils.run_cmd(train_cmd, '%s/log_%s.txt' % (self._test_dir, 'master')) + proc = test_utils.run_cmd(train_cmd, + '%s/log_%s.txt' % (self._test_dir, 'master')) proc.wait() if proc.returncode != 0: logging.error('train %s failed' % test_pipeline_config_path) diff --git a/easy_rec/python/test/hpo_test.py b/easy_rec/python/test/hpo_test.py index 7594a5b8e..d6ed01fcf 100644 --- a/easy_rec/python/test/hpo_test.py +++ b/easy_rec/python/test/hpo_test.py @@ -10,7 +10,9 @@ import tensorflow as tf from easy_rec.python.protos.feature_config_pb2 import FeatureConfig -from easy_rec.python.utils import config_util, hpo_util, test_utils +from easy_rec.python.utils import config_util +from easy_rec.python.utils import hpo_util +from easy_rec.python.utils import test_utils if tf.__version__ >= '2.0': import tensorflow.io.gfile as gfile @@ -26,6 +28,7 @@ class HPOTest(tf.test.TestCase): + def __init__(self, methodName='HPOTest'): super(HPOTest, self).__init__(methodName=methodName) self._metric_data_path = 'data/test/hpo_test/eval_val/*.tfevents.*' @@ -41,7 +44,8 @@ def load_config(self, config_path): def test_save_eval_metrics(self): test_dir = test_utils.get_tmp_dir() - tmp_file = os.path.join(test_dir, 'easy_rec_hpo_test_%d.metric' % time.time()) + tmp_file = os.path.join(test_dir, + 'easy_rec_hpo_test_%d.metric' % time.time()) hpo_util.save_eval_metrics('data/test/hpo_test/', tmp_file, False) test_utils.clean_up(test_dir) @@ -113,8 +117,8 @@ def test_edit_config_v6(self): for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] >= 'site': assert tmp_fea.embedding_dim == 32, 'input_name = %s %d' % ( - tmp_fea.input_names[0], - tmp_fea.embedding_dim, + tmp_fea.input_names[0], + tmp_fea.embedding_dim, ) else: assert tmp_fea.embedding_dim == 16 @@ -126,7 +130,8 @@ def test_edit_config_v7(self): tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 + assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - + 10.0) < 1e-5 def test_edit_config_v71(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' @@ -135,7 +140,8 @@ def test_edit_config_v71(self): tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 + assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - + 10.0) < 1e-5 def test_edit_config_v8(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' @@ -144,7 +150,8 @@ def test_edit_config_v8(self): tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - 4.0) < 1e-5 + assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - + 4.0) < 1e-5 assert tmp_fea.embedding_dim == 32 def test_edit_config_v81(self): @@ -170,7 +177,8 @@ def test_edit_config_v10(self): tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - 4.0) < 1e-5 + assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - + 4.0) < 1e-5 assert tmp_fea.embedding_dim == 32 def test_edit_config_v11(self): @@ -180,7 +188,8 @@ def test_edit_config_v11(self): tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] == 'c21': - assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - 10.0) < 1e-5 + assert len(tmp_fea.boundaries) == 4 and np.abs(tmp_fea.boundaries[0] - + 10.0) < 1e-5 def test_edit_config_v12(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' @@ -218,7 +227,8 @@ def test_save_eval_metrics_with_env(self): } """ test_dir = test_utils.get_tmp_dir() - tmp_file = os.path.join(test_dir, 'easy_rec_hpo_test_%d.metric' % time.time()) + tmp_file = os.path.join(test_dir, + 'easy_rec_hpo_test_%d.metric' % time.time()) hpo_util.save_eval_metrics('data/test/hpo_test/', tmp_file, False) test_utils.clean_up(test_dir) diff --git a/easy_rec/python/test/kafka_test.py b/easy_rec/python/test/kafka_test.py index 87b2c4a9f..aa94fbff0 100644 --- a/easy_rec/python/test/kafka_test.py +++ b/easy_rec/python/test/kafka_test.py @@ -17,7 +17,8 @@ from easy_rec.python.inference.predictor import Predictor from easy_rec.python.input.kafka_dataset import KafkaDataset -from easy_rec.python.utils import numpy_utils, test_utils +from easy_rec.python.utils import numpy_utils +from easy_rec.python.utils import test_utils try: import kafka @@ -28,6 +29,7 @@ class KafkaTest(tf.test.TestCase): + def setUp(self): self._success = True self._test_dir = test_utils.get_tmp_dir() @@ -36,7 +38,8 @@ def setUp(self): self._zookeeper_proc = None return - logging.info('Testing %s.%s, test_dir=%s' % (type(self).__name__, self._testMethodName, self._test_dir)) + logging.info('Testing %s.%s, test_dir=%s' % + (type(self).__name__, self._testMethodName, self._test_dir)) self._log_dir = os.path.join(self._test_dir, 'logs') if not gfile.IsDirectory(self._log_dir): gfile.MakeDirs(self._log_dir) @@ -57,8 +60,8 @@ def setUp(self): else: fout.write(line_str) cmd = 'bash %s/bin/zookeeper-server-start.sh %s' % ( - kafka_install_dir, - zookeeper_config, + kafka_install_dir, + zookeeper_config, ) log_file = os.path.join(self._log_dir, 'zookeeper.log') self._zookeeper_proc = test_utils.run_cmd(cmd, log_file) @@ -73,15 +76,16 @@ def setUp(self): else: fout.write(line_str) cmd = 'bash %s/bin/kafka-server-start.sh %s' % ( - kafka_install_dir, - kafka_config, + kafka_install_dir, + kafka_config, ) log_file = os.path.join(self._log_dir, 'kafka_server.log') self._kafka_server_proc = test_utils.run_cmd(cmd, log_file) started = False while not started: - if self._kafka_server_proc.poll() and self._kafka_server_proc.returncode: + if self._kafka_server_proc.poll( + ) and self._kafka_server_proc.returncode: logging.warning('start kafka server failed, will retry.') os.system('cat %s' % log_file) self._kafka_server_proc = test_utils.run_cmd(cmd, log_file) @@ -106,11 +110,11 @@ def _create_topic(self, num_partitions=2): logging.info('create topic: %s' % self._test_topic) topic_list = [ - NewTopic( - name=self._test_topic, - num_partitions=num_partitions, - replication_factor=1, - ) + NewTopic( + name=self._test_topic, + num_partitions=num_partitions, + replication_factor=1, + ) ] admin_clt.create_topics(new_topics=topic_list, validate_only=False) @@ -147,13 +151,15 @@ def tearDown(self): if self._success: test_utils.clean_up(self._test_dir) - @unittest.skipIf('kafka_install_dir' not in os.environ, 'Only execute when kafka is available') + @unittest.skipIf('kafka_install_dir' not in os.environ, + 'Only execute when kafka is available') def test_kafka_ops(self): try: test_utils.set_gpu_id(None) def _generate(): - producer = KafkaProducer(bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1)) + producer = KafkaProducer( + bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1)) i = 0 while not self._should_stop: msg = 'user_id_%d' % i @@ -164,19 +170,20 @@ def _generate(): group = 'dataset_consumer' k = KafkaDataset( - servers=self._kafka_servers[0], - topics=[self._test_topic + ':0', self._test_topic + ':1'], - group=group, - eof=True, - # control the maximal read of each partition - config_global=['max.partition.fetch.bytes=1048576'], - message_key=True, - message_offset=True, + servers=self._kafka_servers[0], + topics=[self._test_topic + ':0', self._test_topic + ':1'], + group=group, + eof=True, + # control the maximal read of each partition + config_global=['max.partition.fetch.bytes=1048576'], + message_key=True, + message_offset=True, ) batch_dataset = k.batch(5) - iterator = iterator_ops.Iterator.from_structure(batch_dataset.output_types) + iterator = iterator_ops.Iterator.from_structure( + batch_dataset.output_types) init_batch_op = iterator.make_initializer(batch_dataset) get_next = iterator.get_next() @@ -205,7 +212,8 @@ def _generate(): self._success = False raise ex - @unittest.skipIf('kafka_install_dir' not in os.environ, 'Only execute when kafka is available') + @unittest.skipIf('kafka_install_dir' not in os.environ, + 'Only execute when kafka is available') def test_kafka_train(self): try: # start produce thread @@ -214,15 +222,16 @@ def test_kafka_train(self): test_utils.set_gpu_id(None) self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_avazu_kafka.config', self._test_dir - ) + 'samples/model_config/deepfm_combo_avazu_kafka.config', + self._test_dir) self.assertTrue(self._success) except Exception as ex: self._success = False raise ex def _generate(self): - producer = KafkaProducer(bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1)) + producer = KafkaProducer( + bootstrap_servers=self._kafka_servers, api_version=(0, 10, 1)) while not self._should_stop: with open('data/test/dwd_avazu_ctr_deepmodel_10w.csv', 'r') as fin: for line_str in fin: @@ -235,7 +244,8 @@ def _generate(self): producer.close() logging.info('data generation thread done.') - @unittest.skipIf('kafka_install_dir' not in os.environ, 'Only execute when kafka is available') + @unittest.skipIf('kafka_install_dir' not in os.environ, + 'Only execute when kafka is available') def test_kafka_train_chief_redundant(self): try: # start produce thread @@ -244,16 +254,17 @@ def test_kafka_train_chief_redundant(self): test_utils.set_gpu_id(None) self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_combo_avazu_kafka_chief_redundant.config', - self._test_dir, - num_evaluator=1, + 'samples/model_config/deepfm_combo_avazu_kafka_chief_redundant.config', + self._test_dir, + num_evaluator=1, ) self.assertTrue(self._success) except Exception as ex: self._success = False raise ex - @unittest.skipIf('kafka_install_dir' not in os.environ, 'Only execute when kafka is available') + @unittest.skipIf('kafka_install_dir' not in os.environ, + 'Only execute when kafka is available') def test_kafka_train_v2(self): try: # start produce thread @@ -262,8 +273,8 @@ def test_kafka_train_v2(self): test_utils.set_gpu_id(None) self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_avazu_kafka_time_offset.config', - self._test_dir, + 'samples/model_config/deepfm_combo_avazu_kafka_time_offset.config', + self._test_dir, ) self.assertTrue(self._success) @@ -272,30 +283,29 @@ def test_kafka_train_v2(self): raise ex @unittest.skipIf( - 'kafka_install_dir' not in os.environ - or 'oss_path' not in os.environ - or 'oss_endpoint' not in os.environ - and 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ, - 'Only execute when kafka is available', + 'kafka_install_dir' not in os.environ or 'oss_path' not in os.environ or + 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or + 'oss_sk' not in os.environ, + 'Only execute when kafka is available', ) def test_kafka_processor(self): - self._test_kafka_processor('samples/model_config/taobao_fg_incr_save.config') + self._test_kafka_processor( + 'samples/model_config/taobao_fg_incr_save.config') @unittest.skipIf( - 'kafka_install_dir' not in os.environ - or 'oss_path' not in os.environ - or 'oss_endpoint' not in os.environ - and 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ, - 'Only execute when kafka is available', + 'kafka_install_dir' not in os.environ or 'oss_path' not in os.environ or + 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or + 'oss_sk' not in os.environ, + 'Only execute when kafka is available', ) def test_kafka_processor_ev(self): - self._test_kafka_processor('samples/model_config/taobao_fg_incr_save_ev.config') + self._test_kafka_processor( + 'samples/model_config/taobao_fg_incr_save_ev.config') def _test_kafka_processor(self, config_path): self._success = False - success = test_utils.test_distributed_train_eval(config_path, self._test_dir, total_steps=500) + success = test_utils.test_distributed_train_eval( + config_path, self._test_dir, total_steps=500) self.assertTrue(success) export_cmd = """ python -m easy_rec.python.export --pipeline_config_path %s/pipeline.config @@ -303,15 +313,16 @@ def _test_kafka_processor(self, config_path): --asset_files ./samples/rtp_fg/fg.json --checkpoint_path %s/train/model.ckpt-0 """ % ( - self._test_dir, - self._test_dir, - os.environ['oss_path'], - os.environ['oss_ak'], - os.environ['oss_sk'], - os.environ['oss_endpoint'], - self._test_dir, + self._test_dir, + self._test_dir, + os.environ['oss_path'], + os.environ['oss_ak'], + os.environ['oss_sk'], + os.environ['oss_endpoint'], + self._test_dir, ) - proc = test_utils.run_cmd(export_cmd, '%s/log_export_sep.txt' % self._test_dir) + proc = test_utils.run_cmd(export_cmd, + '%s/log_export_sep.txt' % self._test_dir) proc.wait() self.assertTrue(proc.returncode == 0) files = gfile.Glob(os.path.join(self._test_dir, 'export/sep/[1-9][0-9]*')) @@ -322,13 +333,14 @@ def _test_kafka_processor(self, config_path): --input_path data/test/rtp/taobao_test_feature.txt --output_path %s/processor.out --test_dir %s """ % ( - export_sep_dir, - self._test_dir, - self._test_dir, + export_sep_dir, + self._test_dir, + self._test_dir, ) envs = dict(os.environ) envs['PROCESSOR_TEST'] = '1' - proc = test_utils.run_cmd(predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs) + proc = test_utils.run_cmd( + predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs) proc.wait() self.assertTrue(proc.returncode == 0) @@ -350,7 +362,8 @@ def _test_kafka_processor(self, config_path): with open('%s/predictor.out' % self._test_dir, 'w') as fout: for i in range(len(output_res)): - fout.write(json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n') + fout.write( + json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n') for i in range(len(output_res)): val0 = output_res[i]['probs'] @@ -359,7 +372,8 @@ def _test_kafka_processor(self, config_path): assert diff < 1e-4, 'too much difference[%.6f] >= 1e-4' % diff self._success = True - @unittest.skipIf('kafka_install_dir' not in os.environ, 'Only execute when kafka is available') + @unittest.skipIf('kafka_install_dir' not in os.environ, + 'Only execute when kafka is available') def test_kafka_train_v3(self): try: # start produce thread @@ -368,8 +382,8 @@ def test_kafka_train_v3(self): test_utils.set_gpu_id(None) self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_avazu_kafka_time_offset2.config', - self._test_dir, + 'samples/model_config/deepfm_combo_avazu_kafka_time_offset2.config', + self._test_dir, ) self.assertTrue(self._success) diff --git a/easy_rec/python/test/local_incr_test.py b/easy_rec/python/test/local_incr_test.py index 26623d5d3..5b9e33bd9 100644 --- a/easy_rec/python/test/local_incr_test.py +++ b/easy_rec/python/test/local_incr_test.py @@ -11,58 +11,62 @@ from tensorflow.python.platform import gfile from easy_rec.python.inference.predictor import Predictor -from easy_rec.python.utils import numpy_utils, test_utils +from easy_rec.python.utils import numpy_utils +from easy_rec.python.utils import test_utils class LocalIncrTest(tf.test.TestCase): + def setUp(self): self._success = True self._test_dir = test_utils.get_tmp_dir() - logging.info('Testing %s.%s, test_dir=%s' % (type(self).__name__, self._testMethodName, self._test_dir)) + logging.info('Testing %s.%s, test_dir=%s' % + (type(self).__name__, self._testMethodName, self._test_dir)) self._log_dir = os.path.join(self._test_dir, 'logs') if not gfile.IsDirectory(self._log_dir): gfile.MakeDirs(self._log_dir) @unittest.skipIf( - 'oss_path' not in os.environ - or 'oss_endpoint' not in os.environ - and 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ, - 'Only execute when kafka is available', + 'oss_path' not in os.environ or + 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or + 'oss_sk' not in os.environ, + 'Only execute when kafka is available', ) def test_incr_save(self): - self._test_incr_save('samples/model_config/taobao_fg_incr_save_local.config') + self._test_incr_save( + 'samples/model_config/taobao_fg_incr_save_local.config') @unittest.skipIf( - 'oss_path' not in os.environ - or 'oss_endpoint' not in os.environ - and 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ, - 'Only execute when kafka is available', + 'oss_path' not in os.environ or + 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or + 'oss_sk' not in os.environ, + 'Only execute when kafka is available', ) def test_incr_save_ev(self): - self._test_incr_save('samples/model_config/taobao_fg_incr_save_ev_local.config') + self._test_incr_save( + 'samples/model_config/taobao_fg_incr_save_ev_local.config') @unittest.skipIf( - 'oss_path' not in os.environ - or 'oss_endpoint' not in os.environ - and 'oss_ak' not in os.environ - or 'oss_sk' not in os.environ, - 'Only execute when kafka is available', + 'oss_path' not in os.environ or + 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or + 'oss_sk' not in os.environ, + 'Only execute when kafka is available', ) def test_incr_save_share_ev(self): - self._test_incr_save('samples/model_config/taobao_fg_incr_save_share_ev_local.config') + self._test_incr_save( + 'samples/model_config/taobao_fg_incr_save_share_ev_local.config') def _test_incr_save(self, config_path): self._success = False success = test_utils.test_distributed_train_eval( - config_path, - self._test_dir, - total_steps=100, - edit_config_json={ - 'train_config.incr_save_config.fs.mount_path': os.path.join(self._test_dir, 'train/incr_save/') - }, + config_path, + self._test_dir, + total_steps=100, + edit_config_json={ + 'train_config.incr_save_config.fs.mount_path': + os.path.join(self._test_dir, 'train/incr_save/') + }, ) self.assertTrue(success) export_cmd = """ @@ -71,15 +75,16 @@ def _test_incr_save(self, config_path): --asset_files ./samples/rtp_fg/fg.json --checkpoint_path %s/train/model.ckpt-0 """ % ( - self._test_dir, - self._test_dir, - os.environ['oss_path'], - os.environ['oss_ak'], - os.environ['oss_sk'], - os.environ['oss_endpoint'], - self._test_dir, + self._test_dir, + self._test_dir, + os.environ['oss_path'], + os.environ['oss_ak'], + os.environ['oss_sk'], + os.environ['oss_endpoint'], + self._test_dir, ) - proc = test_utils.run_cmd(export_cmd, '%s/log_export_sep.txt' % self._test_dir) + proc = test_utils.run_cmd(export_cmd, + '%s/log_export_sep.txt' % self._test_dir) proc.wait() self.assertTrue(proc.returncode == 0) files = gfile.Glob(os.path.join(self._test_dir, 'export/sep/[1-9][0-9]*')) @@ -90,13 +95,14 @@ def _test_incr_save(self, config_path): --input_path data/test/rtp/taobao_test_feature.txt --output_path %s/processor.out --test_dir %s """ % ( - export_sep_dir, - self._test_dir, - self._test_dir, + export_sep_dir, + self._test_dir, + self._test_dir, ) envs = dict(os.environ) envs['PROCESSOR_TEST'] = '1' - proc = test_utils.run_cmd(predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs) + proc = test_utils.run_cmd( + predict_cmd, '%s/log_processor.txt' % self._test_dir, env=envs) proc.wait() self.assertTrue(proc.returncode == 0) @@ -118,7 +124,8 @@ def _test_incr_save(self, config_path): with open('%s/predictor.out' % self._test_dir, 'w') as fout: for i in range(len(output_res)): - fout.write(json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n') + fout.write( + json.dumps(output_res[i], cls=numpy_utils.NumpyEncoder) + '\n') for i in range(len(output_res)): val0 = output_res[i]['probs'] diff --git a/easy_rec/python/test/loss_test.py b/easy_rec/python/test/loss_test.py index aa7184df4..bf5d9a265 100644 --- a/easy_rec/python/test/loss_test.py +++ b/easy_rec/python/test/loss_test.py @@ -2,55 +2,51 @@ import tensorflow as tf from easy_rec.python.loss.circle_loss import ( # NOQA - circle_loss, - get_anchor_positive_triplet_mask, + circle_loss, get_anchor_positive_triplet_mask, ) from easy_rec.python.loss.f1_reweight_loss import ( # NOQA - f1_reweight_sigmoid_cross_entropy, -) + f1_reweight_sigmoid_cross_entropy,) from easy_rec.python.loss.softmax_loss_with_negative_mining import ( # NOQA - softmax_loss_with_negative_mining, -) + softmax_loss_with_negative_mining,) if tf.__version__ >= '2.0': tf = tf.compat.v1 class LossTest(tf.test.TestCase): + def test_f1_reweighted_loss(self): print('test_f1_reweighted_loss') logits = tf.constant([0.1, 0.5, 0.3, 0.8, -0.1, 0.3]) labels = tf.constant([1, 1, 0, 0, 1, 1]) - loss = f1_reweight_sigmoid_cross_entropy(labels=labels, logits=logits, beta_square=4) + loss = f1_reweight_sigmoid_cross_entropy( + labels=labels, logits=logits, beta_square=4) with self.test_session() as sess: loss_val = sess.run(loss) self.assertAlmostEqual(loss_val, 0.47844395, delta=1e-5) def test_softmax_loss_with_negative_mining(self): print('test_softmax_loss_with_negative_mining') - user_emb = tf.constant( - [ + user_emb = tf.constant([ [0.1, 0.5, 0.3], [0.8, -0.1, 0.3], [0.28, 0.3, 0.9], [0.37, 0.45, 0.93], [-0.7, 0.15, 0.03], [0.18, 0.9, -0.3], - ] - ) - item_emb = tf.constant( - [ + ]) + item_emb = tf.constant([ [0.1, -0.5, 0.3], [0.8, -0.31, 0.3], [0.7, -0.45, 0.15], [0.08, -0.31, -0.9], [-0.7, 0.85, 0.03], [0.18, 0.89, -0.3], - ] - ) + ]) label = tf.constant([1, 1, 0, 0, 1, 1]) - loss = softmax_loss_with_negative_mining(user_emb, item_emb, label, num_negative_samples=2, seed=1) + loss = softmax_loss_with_negative_mining( + user_emb, item_emb, label, num_negative_samples=2, seed=1) with self.test_session() as sess: loss_val = sess.run(loss) self.assertAlmostEqual(loss_val, 0.48577175, delta=1e-5) @@ -58,15 +54,15 @@ def test_softmax_loss_with_negative_mining(self): def test_circle_loss(self): print('test_circle_loss') emb = tf.constant( - [ - [0.1, 0.2, 0.15, 0.1], - [0.3, 0.6, 0.45, 0.3], - [0.13, 0.6, 0.45, 0.3], - [0.3, 0.26, 0.45, 0.3], - [0.3, 0.6, 0.5, 0.13], - [0.08, 0.43, 0.21, 0.6], - ], - dtype=tf.float32, + [ + [0.1, 0.2, 0.15, 0.1], + [0.3, 0.6, 0.45, 0.3], + [0.13, 0.6, 0.45, 0.3], + [0.3, 0.26, 0.45, 0.3], + [0.3, 0.6, 0.5, 0.13], + [0.08, 0.43, 0.21, 0.6], + ], + dtype=tf.float32, ) label = tf.constant([1, 1, 2, 2, 3, 3]) loss = circle_loss(emb, label, label, margin=0.25, gamma=64) @@ -78,30 +74,30 @@ def test_triplet_mask(self): print('test_triplet_mask') label = tf.constant([1, 1, 2, 2, 3, 3, 4, 5]) positive_mask = tf.constant( - [ - [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], - [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], - ], - dtype=tf.float32, + [ + [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + dtype=tf.float32, ) negative_mask = tf.constant( - [ - [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], - [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], - [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], - [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], - [1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0], - [1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0], - [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0], - [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], - ], - dtype=tf.float32, + [ + [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + ], + dtype=tf.float32, ) with self.test_session(): pos_mask = get_anchor_positive_triplet_mask(label, label) @@ -127,14 +123,16 @@ def _get_anchor_negative_triplet_mask(labels, sessions): """ # Check if sessions[i] != sessions[k] # Uses broadcasting where the 1st argument has shape (1, batch_size) and the 2nd (batch_size, 1) - session_not_equal = tf.not_equal(tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1)) + session_not_equal = tf.not_equal( + tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1)) if labels is sessions: return tf.cast(session_not_equal, tf.float32) # Check if labels[i] != labels[k] # Uses broadcasting where the 1st argument has shape (1, batch_size) and the 2nd (batch_size, 1) - label_not_equal = tf.not_equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1)) + label_not_equal = tf.not_equal( + tf.expand_dims(labels, 0), tf.expand_dims(labels, 1)) mask = tf.logical_or(session_not_equal, label_not_equal) return tf.cast(mask, tf.float32) diff --git a/easy_rec/python/test/odps_command.py b/easy_rec/python/test/odps_command.py index c6ace1bd1..0943887a7 100644 --- a/easy_rec/python/test/odps_command.py +++ b/easy_rec/python/test/odps_command.py @@ -9,6 +9,7 @@ class OdpsCommand: + def __init__(self, odps_oss_config): """Wrapper for running odps command. @@ -16,10 +17,10 @@ def __init__(self, odps_oss_config): odps_oss_config: instance of easy_rec.python.utils.odps_test_util.OdpsOSSConfig """ self.bucket = get_oss_bucket( - odps_oss_config.oss_key, - odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name, + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) self.bucket_name = odps_oss_config.bucket_name self.temp_dir = odps_oss_config.temp_dir @@ -44,16 +45,16 @@ def run_odps_cmd(self, script_file): if self.odps_config_path is None: cmd = 'nohup %s -f %s > %s.log 2>&1' % ( - self.odpscmd, - exec_file_path, - log_file, + self.odpscmd, + exec_file_path, + log_file, ) else: cmd = 'nohup %s --config=%s -f %s > %s.log 2>&1' % ( - self.odpscmd, - self.odps_config_path, - exec_file_path, - log_file, + self.odpscmd, + self.odps_config_path, + exec_file_path, + log_file, ) logging.info('will run cmd: %s' % (cmd)) proc = subprocess.Popen(cmd, shell=True) @@ -61,7 +62,8 @@ def run_odps_cmd(self, script_file): if proc.returncode == 0: logging.info('%s run succeed' % script_file) else: - raise ValueError('%s run FAILED: please check log file:%s.log' % (exec_file_path, log_file)) + raise ValueError('%s run FAILED: please check log file:%s.log' % + (exec_file_path, log_file)) def run_list(self, files): for f in files: diff --git a/easy_rec/python/test/odps_local_run.py b/easy_rec/python/test/odps_local_run.py index 3e83e1b27..a6cc4d121 100644 --- a/easy_rec/python/test/odps_local_run.py +++ b/easy_rec/python/test/odps_local_run.py @@ -11,14 +11,14 @@ from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare +from easy_rec.python.utils import test_utils + from easy_rec.python.test.odps_test_util import ( # NOQA - OdpsOSSConfig, - delete_oss_path, - get_oss_bucket, + OdpsOSSConfig, delete_oss_path, get_oss_bucket, ) -from easy_rec.python.utils import test_utils -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') odps_oss_config = OdpsOSSConfig(script_path='./samples/emr_script') @@ -37,26 +37,29 @@ def tearDown(self): def test_deepfm_local_with_common_io(self): start = [ - 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql', + 'deep_fm/create_external_deepfm_table.sql', + 'deep_fm/create_inner_deepfm_table.sql', ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) odps_cmd.run_list(start) self._success = test_utils.test_single_train_eval( - '%s/configs/deepfm.config' % odps_oss_config.temp_dir, self._test_dir - ) + '%s/configs/deepfm.config' % odps_oss_config.temp_dir, self._test_dir) odps_cmd.run_list(end) self.assertTrue(self._success) if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--odps_config', type=str, default=None, help='odps config path') - parser.add_argument('--oss_config', type=str, default=None, help='ossutilconfig path') - parser.add_argument('--bucket_name', type=str, default=None, help='test oss bucket name') + parser.add_argument( + '--odps_config', type=str, default=None, help='odps config path') + parser.add_argument( + '--oss_config', type=str, default=None, help='ossutilconfig path') + parser.add_argument( + '--bucket_name', type=str, default=None, help='test oss bucket name') parser.add_argument('--arn', type=str, default=None, help='oss rolearn') - parser.add_argument('--odpscmd', type=str, default='odpscmd', help='odpscmd path') + parser.add_argument( + '--odpscmd', type=str, default='odpscmd', help='odpscmd path') args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] for unk_arg in unknown_args: @@ -78,10 +81,10 @@ def test_deepfm_local_with_common_io(self): tf.test.main() # delete oss path bucket = get_oss_bucket( - odps_oss_config.oss_key, - odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name, + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp diff --git a/easy_rec/python/test/odps_run.py b/easy_rec/python/test/odps_run.py index 3b4f8b99b..8943ab509 100644 --- a/easy_rec/python/test/odps_run.py +++ b/easy_rec/python/test/odps_run.py @@ -12,14 +12,14 @@ from easy_rec.python.test.odps_test_cls import OdpsTest from easy_rec.python.test.odps_test_prepare import prepare +from easy_rec.python.utils import config_util + from easy_rec.python.test.odps_test_util import ( # NOQA - OdpsOSSConfig, - delete_oss_path, - get_oss_bucket, + OdpsOSSConfig, delete_oss_path, get_oss_bucket, ) -from easy_rec.python.utils import config_util -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') odps_oss_config = OdpsOSSConfig() @@ -30,11 +30,11 @@ class TestPipelineOnOdps(tf.test.TestCase): def test_deepfm(self): start_files = ['deep_fm/create_inner_deepfm_table.sql'] test_files = [ - 'deep_fm/train_deepfm_model.sql', - 'deep_fm/eval_deepfm.sql', - 'deep_fm/export_deepfm.sql', - 'deep_fm/predict_deepfm.sql', - 'deep_fm/export_rtp_ckpt.sql', + 'deep_fm/train_deepfm_model.sql', + 'deep_fm/eval_deepfm.sql', + 'deep_fm/export_deepfm.sql', + 'deep_fm/predict_deepfm.sql', + 'deep_fm/export_rtp_ckpt.sql', ] end_file = ['deep_fm/drop_table.sql'] @@ -45,10 +45,10 @@ def test_deepfm(self): def test_mmoe(self): start_files = ['mmoe/create_inner_mmoe_table.sql'] test_files = [ - 'mmoe/train_mmoe_model.sql', - 'mmoe/eval_mmoe.sql', - 'mmoe/export_mmoe.sql', - 'mmoe/predict_mmoe.sql', + 'mmoe/train_mmoe_model.sql', + 'mmoe/eval_mmoe.sql', + 'mmoe/export_mmoe.sql', + 'mmoe/predict_mmoe.sql', ] end_file = ['mmoe/drop_mmoe_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -57,16 +57,16 @@ def test_mmoe(self): def test_dssm(self): start_files = [ - 'dssm/create_inner_dssm_table.sql', + 'dssm/create_inner_dssm_table.sql', ] test_files = [ - 'dssm/train_dssm_model.sql', - 'dssm/eval_dssm.sql', - 'dssm/export_dssm.sql', - 'dssm/predict_dssm.sql', + 'dssm/train_dssm_model.sql', + 'dssm/eval_dssm.sql', + 'dssm/export_dssm.sql', + 'dssm/predict_dssm.sql', ] end_file = [ - 'dssm/drop_dssm_table.sql', + 'dssm/drop_dssm_table.sql', ] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) tot.start_test() @@ -75,12 +75,12 @@ def test_dssm(self): def test_multi_tower(self): start_files = ['multi_tower/create_inner_multi_tower_table.sql'] test_files = [ - 'multi_tower/train_multil_tower_din_model.sql', - 'multi_tower/train_multil_tower_bst_model.sql', - 'multi_tower/eval_multil_tower.sql', - 'multi_tower/export_multil_tower.sql', - 'multi_tower/export_again_multi_tower.sql', - 'multi_tower/predict_multil_tower.sql', + 'multi_tower/train_multil_tower_din_model.sql', + 'multi_tower/train_multil_tower_bst_model.sql', + 'multi_tower/eval_multil_tower.sql', + 'multi_tower/export_multil_tower.sql', + 'multi_tower/export_again_multi_tower.sql', + 'multi_tower/predict_multil_tower.sql', ] end_file = ['multi_tower/drop_multil_tower_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -90,16 +90,16 @@ def test_multi_tower(self): def test_other(self): start_files = ['deep_fm/create_inner_deepfm_table.sql'] test_files = [ - # 'other_test/test_train_gpuRequired_mirrored', # 线上报错， - # 'other_test/test_train_distribute_strategy_collective', # 线上报错， - 'other_test/test_train_hpo_with_evaluator.sql', - # 'other_test/test_train_version.sql', - # 'other_test/test_train_distribute_strategy_ess.sql', - 'other_test/test_train_before_export.sql', - 'other_test/test_eval_checkpoint_path.sql', - 'other_test/test_export_checkpoint_path.sql', - 'other_test/test_export_update_model_dir.sql', - 'other_test/test_predict_selected_cols.sql', + # 'other_test/test_train_gpuRequired_mirrored', # 线上报错， + # 'other_test/test_train_distribute_strategy_collective', # 线上报错， + 'other_test/test_train_hpo_with_evaluator.sql', + # 'other_test/test_train_version.sql', + # 'other_test/test_train_distribute_strategy_ess.sql', + 'other_test/test_train_before_export.sql', + 'other_test/test_eval_checkpoint_path.sql', + 'other_test/test_export_checkpoint_path.sql', + 'other_test/test_export_update_model_dir.sql', + 'other_test/test_predict_selected_cols.sql', ] end_file = ['other_test/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -109,14 +109,14 @@ def test_other(self): def test_best_exporter(self): start_files = ['deep_fm/create_inner_deepfm_table.sql'] test_files = [ - 'other_test/test_train_best_export.sql', + 'other_test/test_train_best_export.sql', ] end_file = ['other_test/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) tot.start_test() config_path = os.path.join( - odps_oss_config.temp_dir, - 'configs/dwd_avazu_ctr_deepmodel_ext_best_export.config', + odps_oss_config.temp_dir, + 'configs/dwd_avazu_ctr_deepmodel_ext_best_export.config', ) config = config_util.get_configs_from_pipeline_file(config_path) model_dir = config.model_dir @@ -127,31 +127,38 @@ def test_best_exporter(self): logging.info('stripped model_dir = %s' % model_dir) bucket = get_oss_bucket( - odps_oss_config.oss_key, - odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name, + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) best_ckpt_prefix = os.path.join(model_dir, 'best_ckpt/model.ckpt') - best_ckpts = [x.key for x in oss2.ObjectIterator(bucket, prefix=best_ckpt_prefix) if x.key.endswith('.meta')] + best_ckpts = [ + x.key + for x in oss2.ObjectIterator(bucket, prefix=best_ckpt_prefix) + if x.key.endswith('.meta') + ] logging.info('best ckpts: %s' % str(best_ckpts)) assert len(best_ckpts) <= 2, 'too many best ckpts: %s' % str(best_ckpts) best_export_prefix = os.path.join(model_dir, 'export/best/') best_exports = [ - x.key for x in oss2.ObjectIterator(bucket, prefix=best_export_prefix) if x.key.endswith('/saved_model.pb') + x.key + for x in oss2.ObjectIterator(bucket, prefix=best_export_prefix) + if x.key.endswith('/saved_model.pb') ] logging.info('best exports: %s' % str(best_exports)) - assert len(best_exports) <= 2, 'too many best exports: %s' % str(best_exports) + assert len( + best_exports) <= 2, 'too many best exports: %s' % str(best_exports) return True def test_embedding_variable(self): start_files = [ - 'embedding_variable/create_table.sql', + 'embedding_variable/create_table.sql', ] test_files = [ - 'embedding_variable/train.sql', - 'embedding_variable/train_work_que.sql', - 'embedding_variable/export.sql', + 'embedding_variable/train.sql', + 'embedding_variable/train_work_que.sql', + 'embedding_variable/export.sql', ] end_file = ['embedding_variable/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -168,13 +175,13 @@ def test_multi_value_export(self): def test_boundary_test(self): start_files = [ - 'boundary/create_inner_boundary_table.sql', + 'boundary/create_inner_boundary_table.sql', ] test_files = [ - 'boundary/train_multi_tower_model.sql', - 'boundary/finetune_multi_tower_model.sql', - 'boundary/finetune_multi_tower_conti.sql', - 'boundary/train_compat.sql', + 'boundary/train_multi_tower_model.sql', + 'boundary/finetune_multi_tower_model.sql', + 'boundary/finetune_multi_tower_conti.sql', + 'boundary/train_compat.sql', ] end_file = ['boundary/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -192,20 +199,34 @@ def test_vector_retrieve(self): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--odps_config', type=str, default=None, help='odps config path') - parser.add_argument('--oss_config', type=str, default=None, help='ossutilconfig path') - parser.add_argument('--bucket_name', type=str, default=None, help='test oss bucket name') + parser.add_argument( + '--odps_config', type=str, default=None, help='odps config path') + parser.add_argument( + '--oss_config', type=str, default=None, help='ossutilconfig path') + parser.add_argument( + '--bucket_name', type=str, default=None, help='test oss bucket name') parser.add_argument('--arn', type=str, default=None, help='oss rolearn') - parser.add_argument('--odpscmd', type=str, default='odpscmd', help='odpscmd path') - parser.add_argument('--algo_name', type=str, default='easy_rec_ext', help='whether use pai-tf 1.15') - parser.add_argument('--algo_project', type=str, default=None, help='algo project name') - parser.add_argument('--algo_res_project', type=str, default=None, help='algo resource project name') - parser.add_argument('--algo_version', type=str, default=None, help='algo version') parser.add_argument( - '--is_outer', - type=int, - default=1, - help='is outer pai or inner pai, the arguments are differed slightly due to history reasons', + '--odpscmd', type=str, default='odpscmd', help='odpscmd path') + parser.add_argument( + '--algo_name', + type=str, + default='easy_rec_ext', + help='whether use pai-tf 1.15') + parser.add_argument( + '--algo_project', type=str, default=None, help='algo project name') + parser.add_argument( + '--algo_res_project', + type=str, + default=None, + help='algo resource project name') + parser.add_argument( + '--algo_version', type=str, default=None, help='algo version') + parser.add_argument( + '--is_outer', + type=int, + default=1, + help='is outer pai or inner pai, the arguments are differed slightly due to history reasons', ) args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] @@ -225,7 +246,8 @@ def test_vector_retrieve(self): if args.algo_version: odps_oss_config.algo_version = args.algo_version algo_names = ['easy_rec_ext15', 'easy_rec_ext'] - assert args.algo_name in algo_names, 'algo_name must be oneof: %s' % (','.join(algo_names)) + assert args.algo_name in algo_names, 'algo_name must be oneof: %s' % ( + ','.join(algo_names)) odps_oss_config.algo_name = args.algo_name if args.arn: odps_oss_config.arn = args.arn @@ -236,10 +258,10 @@ def test_vector_retrieve(self): prepare(odps_oss_config) tf.test.main() bucket = get_oss_bucket( - odps_oss_config.oss_key, - odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name, + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) shutil.rmtree(odps_oss_config.temp_dir) diff --git a/easy_rec/python/test/odps_test_prepare.py b/easy_rec/python/test/odps_test_prepare.py index c2c0f5e16..d54017acd 100644 --- a/easy_rec/python/test/odps_test_prepare.py +++ b/easy_rec/python/test/odps_test_prepare.py @@ -9,7 +9,8 @@ import oss2 -from easy_rec.python.test.odps_test_util import OdpsOSSConfig, get_oss_bucket +from easy_rec.python.test.odps_test_util import OdpsOSSConfig +from easy_rec.python.test.odps_test_util import get_oss_bucket def download_data(ali_bucket, script_path): @@ -44,7 +45,8 @@ def download_data(ali_bucket, script_path): if not os.path.exists(dst_dir): os.makedirs(dst_dir) ali_bucket.get_object_to_file(obj_key, dst_path) - logging.info('down file oss://%s/%s to %s completed' % (ali_bucket.bucket_name, obj_key, dst_path)) + logging.info('down file oss://%s/%s to %s completed' % + (ali_bucket.bucket_name, obj_key, dst_path)) def merge_files(merge_dir, merge_out): @@ -105,9 +107,9 @@ def change_files(odps_oss_config, file_path): # tmp_e = tmp_e.replace('.aliyuncs.com', '.oss-internal.aliyun-inc.com') if '-Dbuckets=' in line: line = '-Dbuckets=oss://%s/?role_arn=%s&host=%s\n' % ( - odps_oss_config.bucket_name, - odps_oss_config.arn, - tmp_e, + odps_oss_config.bucket_name, + odps_oss_config.arn, + tmp_e, ) elif '-Darn=' in line or '-DossHost' in line: continue @@ -138,28 +140,30 @@ def put_data_to_bucket(odps_oss_config): odps_oss_config: odps oss config obj """ test_bucket = get_oss_bucket( - odps_oss_config.oss_key, - odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name, + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, ) for sub_dir in ['configs']: - for root, dirs, files in os.walk(os.path.join(odps_oss_config.temp_dir, sub_dir)): + for root, dirs, files in os.walk( + os.path.join(odps_oss_config.temp_dir, sub_dir)): for one_file in files: file_path = os.path.join(root, one_file) obj_path = file_path.split(sub_dir + '/')[1] dst_path = os.path.join(odps_oss_config.exp_dir, sub_dir, obj_path) test_bucket.put_object_from_file(dst_path, file_path) - logging.info('put %s to oss://%s/%s' % (file_path, odps_oss_config.bucket_name, dst_path)) + logging.info('put %s to oss://%s/%s' % + (file_path, odps_oss_config.bucket_name, dst_path)) def prepare(odps_oss_config): logging.info('temp_dir = %s' % odps_oss_config.temp_dir) ali_bucket = get_oss_bucket( - odps_oss_config.ali_oss_key, - odps_oss_config.ali_oss_secret, - odps_oss_config.ali_bucket_endpoint, - odps_oss_config.ali_bucket_name, + odps_oss_config.ali_oss_key, + odps_oss_config.ali_oss_secret, + odps_oss_config.ali_bucket_endpoint, + odps_oss_config.ali_bucket_name, ) shutil.copytree(odps_oss_config.script_path, odps_oss_config.temp_dir) logging.info('start down data') @@ -172,7 +176,7 @@ def prepare(odps_oss_config): file_path = os.path.join(root, file) # drop .template if file_path.endswith('.template'): - tmp_path = file_path[: -len('.template')] + tmp_path = file_path[:-len('.template')] os.rename(file_path, tmp_path) file_path = tmp_path if 'data' not in file_path: @@ -189,7 +193,8 @@ def prepare(odps_oss_config): if __name__ == '__main__': if len(sys.argv) < 5: - print('usage: %s ossutilconfig bucket_name rolearn odpsconfig' % sys.argv[0]) + print('usage: %s ossutilconfig bucket_name rolearn odpsconfig' % + sys.argv[0]) sys.exit(1) odps_oss_config = OdpsOSSConfig() diff --git a/easy_rec/python/test/odps_test_util.py b/easy_rec/python/test/odps_test_util.py index d61103048..2cdcd9684 100644 --- a/easy_rec/python/test/odps_test_util.py +++ b/easy_rec/python/test/odps_test_util.py @@ -11,8 +11,7 @@ try: from datahub import DataHub from datahub.exceptions import ( # NOQA - InvalidOperationException, - ResourceExistException, + InvalidOperationException, ResourceExistException, ) # from datahub.exceptions import LimitExceededException @@ -21,7 +20,8 @@ # from datahub.models import CursorType from datahub.models import FieldType, RecordSchema, RecordType, TupleRecord except Exception: - logging.error('DataHub is not installed, please installed it by: pip install pydatahub') + logging.error( + 'DataHub is not installed, please installed it by: pip install pydatahub') DataHub = None try: @@ -33,6 +33,7 @@ class OdpsOSSConfig: + def __init__(self, script_path='./samples/odps_script'): self.time_stamp = int(time.time()) temp_dir = os.environ.get('TMPDIR', '/tmp') @@ -87,11 +88,11 @@ def load_oss_config(self, config_path): line_str = line_str.strip() line_str = line_str.replace(' ', '') if line_str.startswith('accessKeyID='): - self.oss_key = line_str[len('accessKeyID=') :].strip() + self.oss_key = line_str[len('accessKeyID='):].strip() elif line_str.startswith('accessKeySecret='): - self.oss_secret = line_str[len('accessKeySecret=') :].strip() + self.oss_secret = line_str[len('accessKeySecret='):].strip() elif line_str.startswith('endpoint='): - self.endpoint = line_str[len('endpoint=') :].strip() + self.endpoint = line_str[len('endpoint='):].strip() def load_odps_config(self, config_path): self.odps_config_path = config_path @@ -101,16 +102,16 @@ def load_odps_config(self, config_path): line_str = line_str.replace(' ', '') key_str = 'project_name=' if line_str.startswith(key_str): - self.project_name = line_str[len(key_str) :] + self.project_name = line_str[len(key_str):] key_str = 'end_point=' if line_str.startswith(key_str): - self.odps_endpoint = line_str[len(key_str) :] + self.odps_endpoint = line_str[len(key_str):] key_str = 'access_id=' if line_str.startswith(key_str): - self.dh_id = line_str[len(key_str) :] + self.dh_id = line_str[len(key_str):] key_str = 'access_key=' if line_str.startswith(key_str): - self.dh_key = line_str[len(key_str) :] + self.dh_key = line_str[len(key_str):] def clean_topic(self, dh_project): if not dh_project: @@ -131,25 +132,27 @@ def clean_project(self): pass def clean_subscription(self, topic_name): - subscriptions = self.dh.list_subscription(self.dh_project, topic_name, '', 1, 100).subscriptions + subscriptions = self.dh.list_subscription(self.dh_project, topic_name, '', + 1, 100).subscriptions for subscription in subscriptions: self.dh.delete_subscription(self.dh_project, topic_name, subscription) def get_input_type(self, input_type): DhDict = { - 'INT64': FieldType.BIGINT, - 'INT32': FieldType.BIGINT, - 'STRING': FieldType.STRING, - 'BOOLEAN': FieldType.BOOLEAN, - 'FLOAT32': FieldType.DOUBLE, - 'FLOAT64': FieldType.DOUBLE, + 'INT64': FieldType.BIGINT, + 'INT32': FieldType.BIGINT, + 'STRING': FieldType.STRING, + 'BOOLEAN': FieldType.BOOLEAN, + 'FLOAT32': FieldType.DOUBLE, + 'FLOAT64': FieldType.DOUBLE, } return DhDict.get(input_type) def init_dh_and_odps(self): self.dh = DataHub(self.dh_id, self.dh_key, self.dh_endpoint) - self.odps = ODPS(self.dh_id, self.dh_key, self.project_name, self.odps_endpoint) + self.odps = ODPS(self.dh_id, self.dh_key, self.project_name, + self.odps_endpoint) self.odpsTable = 'deepfm_train_%s' % self.time_stamp self.clean_project() read_odps = DataFrame(self.odps.get_table(self.odpsTable)) @@ -166,12 +169,12 @@ def init_dh_and_odps(self): try: # project_name, topic_name, shard_count, life_cycle, record_schema, comment self.dh.create_tuple_topic( - self.dh_project, - self.dh_topic, - 7, - 3, - record_schema, - comment='EasyRecTest', + self.dh_project, + self.dh_topic, + 7, + 3, + record_schema, + comment='EasyRecTest', ) logging.info('create tuple topic %s success!' % self.dh_topic) except ResourceExistException: @@ -181,7 +184,8 @@ def init_dh_and_odps(self): logging.error(traceback.format_exc()) try: self.dh.wait_shards_ready(self.dh_project, self.dh_topic) - logging.info('datahub[%s,%s] shards all ready' % (self.dh_project, self.dh_topic)) + logging.info('datahub[%s,%s] shards all ready' % + (self.dh_project, self.dh_topic)) topic_result = self.dh.get_topic(self.dh_project, self.dh_topic) if topic_result.record_type != RecordType.TUPLE: logging.error('invalid topic type: %s' % str(topic_result.record_type)) diff --git a/easy_rec/python/test/pre_check_test.py b/easy_rec/python/test/pre_check_test.py index 64c894b07..d62befae6 100644 --- a/easy_rec/python/test/pre_check_test.py +++ b/easy_rec/python/test/pre_check_test.py @@ -13,6 +13,7 @@ class CheckTest(tf.test.TestCase): + def setUp(self): self._test_dir = test_utils.get_tmp_dir() self._success = True @@ -26,24 +27,27 @@ def tearDown(self): def test_csv_input_train_with_check(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_taobao.config', - self._test_dir, - check_mode=True, + 'samples/model_config/dbmtl_on_taobao.config', + self._test_dir, + check_mode=True, ) self.assertTrue(self._success) def test_rtp_input_train_with_check(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.config', self._test_dir, check_mode=True - ) + 'samples/model_config/taobao_fg.config', + self._test_dir, + check_mode=True) self.assertTrue(self._success) def test_csv_input_with_pre_check(self): - self._success = test_utils.test_single_pre_check('samples/model_config/dbmtl_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_pre_check( + 'samples/model_config/dbmtl_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_rtp_input_with_pre_check(self): - self._success = test_utils.test_single_pre_check('samples/model_config/dbmtl_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_pre_check( + 'samples/model_config/dbmtl_on_taobao.config', self._test_dir) self.assertTrue(self._success) diff --git a/easy_rec/python/test/predictor_test.py b/easy_rec/python/test/predictor_test.py index 368d222fb..0f58432e9 100644 --- a/easy_rec/python/test/predictor_test.py +++ b/easy_rec/python/test/predictor_test.py @@ -11,11 +11,13 @@ from easy_rec.python.inference.csv_predictor import CSVPredictor from easy_rec.python.inference.predictor import Predictor -from easy_rec.python.utils import config_util, test_utils +from easy_rec.python.utils import config_util +from easy_rec.python.utils import test_utils from easy_rec.python.utils.test_utils import RunAsSubprocess class PredictorTest(tf.test.TestCase): + def setUp(self): self.gpus = test_utils.get_available_gpus() self.assertTrue(len(self.gpus) > 0, 'no available gpu on this machine') @@ -54,24 +56,24 @@ def test_lookup_pred(self): def test_pred_dict(self): predictor = Predictor('data/test/inference/tb_multitower_export/') field_keys = [ - 'pid', - 'adgroup_id', - 'cate_id', - 'campaign_id', - 'customer', - 'brand', - 'user_id', - 'cms_segid', - 'cms_group_id', - 'final_gender_code', - 'age_level', - 'pvalue_level', - 'shopping_level', - 'occupation', - 'new_user_class_level', - 'tag_category_list', - 'tag_brand_list', - 'price', + 'pid', + 'adgroup_id', + 'cate_id', + 'campaign_id', + 'customer', + 'brand', + 'user_id', + 'cms_segid', + 'cms_group_id', + 'final_gender_code', + 'age_level', + 'pvalue_level', + 'shopping_level', + 'occupation', + 'new_user_class_level', + 'tag_category_list', + 'tag_brand_list', + 'price', ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -83,26 +85,27 @@ def test_pred_dict(self): @RunAsSubprocess def test_pred_placeholder_named_by_input(self): - predictor = Predictor('data/test/inference/tb_multitower_placeholder_rename_export/') + predictor = Predictor( + 'data/test/inference/tb_multitower_placeholder_rename_export/') field_keys = [ - 'pid', - 'adgroup_id', - 'cate_id', - 'campaign_id', - 'customer', - 'brand', - 'user_id', - 'cms_segid', - 'cms_group_id', - 'final_gender_code', - 'age_level', - 'pvalue_level', - 'shopping_level', - 'occupation', - 'new_user_class_level', - 'tag_category_list', - 'tag_brand_list', - 'price', + 'pid', + 'adgroup_id', + 'cate_id', + 'campaign_id', + 'customer', + 'brand', + 'user_id', + 'cms_segid', + 'cms_group_id', + 'final_gender_code', + 'age_level', + 'pvalue_level', + 'shopping_level', + 'occupation', + 'new_user_class_level', + 'tag_category_list', + 'tag_brand_list', + 'price', ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -133,24 +136,24 @@ def test_fm_pred_list(self): def test_fm_pred_dict(self): predictor = Predictor('data/test/inference/fm_export/') field_keys = [ - 'pid', - 'adgroup_id', - 'cate_id', - 'campaign_id', - 'customer', - 'brand', - 'user_id', - 'cms_segid', - 'cms_group_id', - 'final_gender_code', - 'age_level', - 'pvalue_level', - 'shopping_level', - 'occupation', - 'new_user_class_level', - 'tag_category_list', - 'tag_brand_list', - 'price', + 'pid', + 'adgroup_id', + 'cate_id', + 'campaign_id', + 'customer', + 'brand', + 'user_id', + 'cms_segid', + 'cms_group_id', + 'final_gender_code', + 'age_level', + 'pvalue_level', + 'shopping_level', + 'occupation', + 'new_user_class_level', + 'tag_category_list', + 'tag_brand_list', + 'price', ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -162,6 +165,7 @@ def test_fm_pred_dict(self): class PredictorTestOnDS(tf.test.TestCase): + def setUp(self): self._test_dir = test_utils.get_tmp_dir() self._test_output_path = None @@ -177,27 +181,29 @@ def test_local_pred(self): test_input_path = 'data/test/inference/taobao_infer_data.txt' self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') saved_model_dir = 'data/test/inference/tb_multitower_export/' - pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) + pipeline_config_path = os.path.join(saved_model_dir, + 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path, False) predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - output_sep=';', - selected_cols='', + saved_model_dir, + pipeline_config.data_config, + output_sep=';', + selected_cols='', ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='ALL_COLUMNS', - output_cols='ALL_COLUMNS', - slice_id=0, - slice_num=1, + test_input_path, + self._test_output_path, + reserved_cols='ALL_COLUMNS', + output_cols='ALL_COLUMNS', + slice_id=0, + slice_num=1, ) header_truth = ( - 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;' - 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' - 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' + 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;' + 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' + 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' ) with open(self._test_output_path + '/part-0.csv', 'r') as f: @@ -210,30 +216,32 @@ def test_local_pred_with_header(self): test_input_path = 'data/test/inference/taobao_infer_data_with_header.txt' self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') saved_model_dir = 'data/test/inference/tb_multitower_export/' - pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) + pipeline_config_path = os.path.join(saved_model_dir, + 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path, False) pipeline_config.data_config.with_header = True predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - with_header=True, - output_sep=';', - selected_cols='', + saved_model_dir, + pipeline_config.data_config, + with_header=True, + output_sep=';', + selected_cols='', ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='ALL_COLUMNS', - output_cols='ALL_COLUMNS', - slice_id=0, - slice_num=1, + test_input_path, + self._test_output_path, + reserved_cols='ALL_COLUMNS', + output_cols='ALL_COLUMNS', + slice_id=0, + slice_num=1, ) header_truth = ( - 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;' - 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' - 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' + 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;' + 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' + 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' ) with open(self._test_output_path + '/part-0.csv', 'r') as f: @@ -246,9 +254,10 @@ def test_local_pred_without_config(self): test_input_path = 'data/test/inference/taobao_infer_data.txt' self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') saved_model_dir = 'data/test/inference/tb_multitower_export/' - self._success = test_utils.test_single_predict( - self._test_dir, test_input_path, self._test_output_path, saved_model_dir - ) + self._success = test_utils.test_single_predict(self._test_dir, + test_input_path, + self._test_output_path, + saved_model_dir) self.assertTrue(self._success) with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() @@ -259,23 +268,25 @@ def test_local_pred_with_part_col(self): test_input_path = 'data/test/inference/taobao_infer_data.txt' self._test_output_path = os.path.join(self._test_dir, 'taobao_infer_result') saved_model_dir = 'data/test/inference/tb_multitower_export/' - pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) + pipeline_config_path = os.path.join(saved_model_dir, + 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path, False) predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - output_sep=';', - selected_cols='', + saved_model_dir, + pipeline_config.data_config, + output_sep=';', + selected_cols='', ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='clk,buy,user_id,adgroup_id', - output_cols='probs', - slice_id=0, - slice_num=1, + test_input_path, + self._test_output_path, + reserved_cols='clk,buy,user_id,adgroup_id', + output_cols='probs', + slice_id=0, + slice_num=1, ) header_truth = 'probs;clk;buy;user_id;adgroup_id' @@ -287,24 +298,27 @@ def test_local_pred_with_part_col(self): @RunAsSubprocess def test_local_pred_rtp(self): test_input_path = 'data/test/inference/taobao_infer_rtp_data.txt' - self._test_output_path = os.path.join(self._test_dir, 'taobao_test_feature_result') + self._test_output_path = os.path.join(self._test_dir, + 'taobao_test_feature_result') saved_model_dir = 'data/test/inference/tb_multitower_rtp_export/' - pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) + pipeline_config_path = os.path.join(saved_model_dir, + 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path, False) predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - output_sep=';', - selected_cols='0,3', + saved_model_dir, + pipeline_config.data_config, + output_sep=';', + selected_cols='0,3', ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='ALL_COLUMNS', - output_cols='ALL_COLUMNS', - slice_id=0, - slice_num=1, + test_input_path, + self._test_output_path, + reserved_cols='ALL_COLUMNS', + output_cols='ALL_COLUMNS', + slice_id=0, + slice_num=1, ) header_truth = 'logits;probs;clk;no_used_1;no_used_2;features' with open(self._test_output_path + '/part-0.csv', 'r') as f: @@ -315,24 +329,27 @@ def test_local_pred_rtp(self): @RunAsSubprocess def test_local_pred_rtp_with_part_col(self): test_input_path = 'data/test/inference/taobao_infer_rtp_data.txt' - self._test_output_path = os.path.join(self._test_dir, 'taobao_test_feature_result') + self._test_output_path = os.path.join(self._test_dir, + 'taobao_test_feature_result') saved_model_dir = 'data/test/inference/tb_multitower_rtp_export/' - pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) + pipeline_config_path = os.path.join(saved_model_dir, + 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path, False) predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - output_sep=';', - selected_cols='0,3', + saved_model_dir, + pipeline_config.data_config, + output_sep=';', + selected_cols='0,3', ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='clk,features,no_used_1', - output_cols='ALL_COLUMNS', - slice_id=0, - slice_num=1, + test_input_path, + self._test_output_path, + reserved_cols='clk,features,no_used_1', + output_cols='ALL_COLUMNS', + slice_id=0, + slice_num=1, ) header_truth = 'logits;probs;clk;features;no_used_1' with open(self._test_output_path + '/part-0.csv', 'r') as f: @@ -345,37 +362,40 @@ def test_local_pred_embedding(self): test_input_path = 'data/test/inference/taobao_item_feature_data.csv' self._test_output_path = os.path.join(self._test_dir, 'taobao_item_feature') saved_model_dir = 'data/test/inference/dssm_item_model/' - pipeline_config_path = os.path.join(saved_model_dir, 'assets/pipeline.config') - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) + pipeline_config_path = os.path.join(saved_model_dir, + 'assets/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path, False) predictor = CSVPredictor( - saved_model_dir, - pipeline_config.data_config, - ds_vector_recall=True, - output_sep=';', - selected_cols='pid,adgroup_id,cate_id,campaign_id,customer,brand,price', + saved_model_dir, + pipeline_config.data_config, + ds_vector_recall=True, + output_sep=';', + selected_cols='pid,adgroup_id,cate_id,campaign_id,customer,brand,price', ) predictor.predict_impl( - test_input_path, - self._test_output_path, - reserved_cols='adgroup_id', - output_cols='item_emb', - slice_id=0, - slice_num=1, + test_input_path, + self._test_output_path, + reserved_cols='adgroup_id', + output_cols='item_emb', + slice_id=0, + slice_num=1, ) with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() self.assertTrue( - output_res[1] == '-0.187066,-0.027638,-0.117294,0.115318,-0.273561,0.035698,-0.055832,' - '0.226849,-0.105808,-0.152751,0.081528,-0.183329,0.134619,0.185392,' - '0.096774,0.104428,0.161868,0.269710,-0.268538,0.138760,-0.170105,' - '0.232625,-0.121130,0.198466,-0.078941,0.017774,0.268834,-0.238553,0.084058,' - '-0.269466,-0.289651,0.179517;620392\n' - ) + output_res[1] == + '-0.187066,-0.027638,-0.117294,0.115318,-0.273561,0.035698,-0.055832,' + '0.226849,-0.105808,-0.152751,0.081528,-0.183329,0.134619,0.185392,' + '0.096774,0.104428,0.161868,0.269710,-0.268538,0.138760,-0.170105,' + '0.232625,-0.121130,0.198466,-0.078941,0.017774,0.268834,-0.238553,0.084058,' + '-0.269466,-0.289651,0.179517;620392\n') class PredictorTestV2(tf.test.TestCase): + def setUp(self): self.gpus = test_utils.get_available_gpus() self.assertTrue(len(self.gpus) > 0, 'no available gpu on this machine') @@ -404,7 +424,8 @@ def test_pred_multi(self): for line_id, line_str in enumerate(fin): line_str = line_str.strip() line_pred = json.loads(line_str) - self.assertTrue(np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-6) + self.assertTrue( + np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-6) @RunAsSubprocess def test_pred_single(self): @@ -422,7 +443,8 @@ def test_pred_single(self): for line_id, line_str in enumerate(fin): line_str = line_str.strip() line_pred = json.loads(line_str) - self.assertTrue(np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-5) + self.assertTrue( + np.abs(line_pred['probs'] - output_res[line_id]['probs']) < 5e-5) if __name__ == '__main__': diff --git a/easy_rec/python/test/rtp_convert_test.py b/easy_rec/python/test/rtp_convert_test.py index 9c83a791f..8f7aad70e 100644 --- a/easy_rec/python/test/rtp_convert_test.py +++ b/easy_rec/python/test/rtp_convert_test.py @@ -6,10 +6,12 @@ import tensorflow as tf -from easy_rec.python.utils import config_util, test_utils +from easy_rec.python.utils import config_util +from easy_rec.python.utils import test_utils class RTPConvertTest(tf.test.TestCase): + def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) @@ -20,8 +22,7 @@ def test_rtp_convert(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_multi_tower.config') - convert_cmd = ( - """ + convert_cmd = (""" python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg.json --label clk @@ -31,21 +32,21 @@ def test_rtp_convert(self): --train_input_path data/test/rtp/taobao_train_feature.txt --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 - """ - % pipeline_config_path - ) - proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) + """ % pipeline_config_path) + proc = test_utils.run_cmd(convert_cmd, + '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) + self.assertTrue( + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) def test_rtp_convert_bucketize(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_multi_tower.config') - convert_cmd = ( - """ + convert_cmd = (""" python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg_bucketize.json --label clk @@ -55,21 +56,21 @@ def test_rtp_convert_bucketize(self): --train_input_path data/test/rtp/taobao_train_bucketize_feature.txt --eval_input_path data/test/rtp/taobao_test_bucketize_feature.txt --selected_cols 0,3 --num_steps 400 - """ - % pipeline_config_path - ) - proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) + """ % pipeline_config_path) + proc = test_utils.run_cmd(convert_cmd, + '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) + self.assertTrue( + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) def test_rtp_convert_bucketize_v2(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_multi_tower.config') - convert_cmd = ( - """ + convert_cmd = (""" python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg_bucketize_v2.json --label clk @@ -79,27 +80,28 @@ def test_rtp_convert_bucketize_v2(self): --train_input_path data/test/rtp/taobao_train_feature.txt --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 - """ - % pipeline_config_path - ) - proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) + """ % pipeline_config_path) + proc = test_utils.run_cmd(convert_cmd, + '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - tmp_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) + tmp_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path) for feature_config in tmp_config.feature_configs: if feature_config.input_names[0] == 'price': assert len(feature_config.boundaries) == 6 - self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) + self.assertTrue( + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) def test_rtp_convert_test_model_config(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_wide_and_deep.config') - convert_cmd = ( - """ + convert_cmd = (""" python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg_bucketize_model_config.json --label clk @@ -108,20 +110,22 @@ def test_rtp_convert_test_model_config(self): --train_input_path data/test/rtp/taobao_train_feature.txt --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 - """ - % pipeline_config_path - ) - proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) + """ % pipeline_config_path) + proc = test_utils.run_cmd(convert_cmd, + '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() self.assertTrue(proc.returncode == 0) - tmp_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) + tmp_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path) assert len(tmp_config.model_config.wide_and_deep.dnn.hidden_units) == 2 assert tmp_config.model_config.wide_and_deep.dnn.hidden_units[0] == 48 assert tmp_config.model_config.wide_and_deep.dnn.hidden_units[1] == 24 assert tmp_config.model_dir == 'experiments/rtp_fg/wide_and_deep_update_model' - self.assertTrue(test_utils.test_single_train_eval(pipeline_config_path, test_dir=test_dir)) + self.assertTrue( + test_utils.test_single_train_eval( + pipeline_config_path, test_dir=test_dir)) test_utils.clean_up(test_dir) diff --git a/easy_rec/python/test/run.py b/easy_rec/python/test/run.py index c5ef94d0c..0c7ac4c79 100644 --- a/easy_rec/python/test/run.py +++ b/easy_rec/python/test/run.py @@ -17,14 +17,18 @@ tf.app.flags.DEFINE_bool('list_tests', False, 'list all tests') tf.app.flags.DEFINE_string('list_test_to_file', None, 'list all tests') tf.app.flags.DEFINE_string('pattern', '*_test.py', 'test file pattern') -tf.app.flags.DEFINE_string('test_dir', 'easy_rec/python/test', 'directory to be tested') -tf.app.flags.DEFINE_integer('num_parallel', 10, 'number of parallel executed cases.') -tf.app.flags.DEFINE_integer('timeout', 3600, 'maximal execute time in seconds for each case.') +tf.app.flags.DEFINE_string('test_dir', 'easy_rec/python/test', + 'directory to be tested') +tf.app.flags.DEFINE_integer('num_parallel', 10, + 'number of parallel executed cases.') +tf.app.flags.DEFINE_integer('timeout', 3600, + 'maximal execute time in seconds for each case.') FLAGS = tf.flags.FLAGS def gather_test_cases(test_dir, pattern): - discover = unittest.defaultTestLoader.discover(test_dir, pattern=pattern, top_level_dir=None) + discover = unittest.defaultTestLoader.discover( + test_dir, pattern=pattern, top_level_dir=None) all_tests = [] for suite_discovered in discover: for test_case in suite_discovered: @@ -72,7 +76,8 @@ def main(argv): test_log_dir = os.path.join(test_dir, 'logs') if not os.path.exists(test_log_dir): os.makedirs(test_log_dir) - logging.info('Total number of cases: %d test_dir: %s' % (len(all_tests), test_dir)) + logging.info('Total number of cases: %d test_dir: %s' % + (len(all_tests), test_dir)) max_num_port_per_proc = 3 total_port_num = (max_num_port_per_proc + 2) * FLAGS.num_parallel * 10 @@ -95,7 +100,8 @@ def main(argv): del procs[proc] cmd = 'python -m easy_rec.python.test.%s %s' % (case_file, case_name) log_file = '%s/%s.%s.log' % (test_log_dir, case_file, case_name) - tmp_ports = ','.join([str(x) for x in all_available_ports[:max_num_port_per_proc]]) + tmp_ports = ','.join( + [str(x) for x in all_available_ports[:max_num_port_per_proc]]) all_available_ports = all_available_ports[max_num_port_per_proc:] logging.info('Run %s.%s Log: %s' % (case_file, case_name, log_file)) @@ -106,7 +112,8 @@ def main(argv): for proc in procs: try: - test_utils.proc_wait(proc, timeout=int(os.environ.get('TEST_TIME_OUT', 1200))) + test_utils.proc_wait( + proc, timeout=int(os.environ.get('TEST_TIME_OUT', 1200))) except Exception as ex: fail_file, fail_name = procs[proc] logging.info('Case Exception: %s.%s %s' % (fail_file, fail_name, str(ex))) @@ -119,9 +126,8 @@ def main(argv): if len(failed_cases) > 0: logging.info('Number Cases Failed: %d' % len(failed_cases)) for fail_file, fail_name, exit_code in failed_cases: - logging.info( - '\t%s.%s failed, exit_code:%d log: %s.%s.log' % (fail_file, fail_name, exit_code, fail_file, fail_name) - ) + logging.info('\t%s.%s failed, exit_code:%d log: %s.%s.log' % + (fail_file, fail_name, exit_code, fail_file, fail_name)) return 1 else: logging.info('TestSucceed.') diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 792f77b7c..7e6f5ed50 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -7,19 +7,17 @@ import threading import time import unittest -from distutils.version import LooseVersion import numpy as np import six import tensorflow as tf +from distutils.version import LooseVersion from tensorflow.python.platform import gfile from easy_rec.python.main import predict + from easy_rec.python.utils import ( # NOQA - config_util, - constant, - estimator_utils, - test_utils, + config_util, constant, estimator_utils, test_utils, ) try: @@ -43,6 +41,7 @@ class TrainEvalTest(tf.test.TestCase): + def setUp(self): logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) self._test_dir = test_utils.get_tmp_dir() @@ -55,93 +54,93 @@ def tearDown(self): test_utils.clean_up(self._test_dir) def test_deepfm_with_lookup_feature(self): - self._success = test_utils.test_single_train_eval('samples/model_config/deepfm_lookup.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/deepfm_lookup.config', self._test_dir) self.assertTrue(self._success) def test_deepfm_with_combo_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_ctr.config', self._test_dir - ) + 'samples/model_config/deepfm_combo_on_avazu_ctr.config', self._test_dir) self.assertTrue(self._success) def test_deepfm_with_combo_v2_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_v2_on_avazu_ctr.config', self._test_dir - ) + 'samples/model_config/deepfm_combo_v2_on_avazu_ctr.config', + self._test_dir) self.assertTrue(self._success) def test_deepfm_with_combo_v3_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_v3_on_avazu_ctr.config', self._test_dir - ) + 'samples/model_config/deepfm_combo_v3_on_avazu_ctr.config', + self._test_dir) self.assertTrue(self._success) def test_deepfm_freeze_gradient(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_freeze_gradient.config', self._test_dir - ) + 'samples/model_config/deepfm_freeze_gradient.config', self._test_dir) self.assertTrue(self._success) def test_deepfm_with_vocab_list(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_vocab_list_on_avazu_ctr.config', self._test_dir - ) + 'samples/model_config/deepfm_vocab_list_on_avazu_ctr.config', + self._test_dir) self.assertTrue(self._success) def test_deepfm_with_multi_class(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', self._test_dir - ) + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + self._test_dir) self.assertTrue(self._success) def test_wide_and_deep_no_final(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/wide_and_deep_no_final_on_avazau_ctr.config', - self._test_dir, + 'samples/model_config/wide_and_deep_no_final_on_avazau_ctr.config', + self._test_dir, ) self.assertTrue(self._success) def test_wide_and_deep(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/wide_and_deep_on_avazau_ctr.config', self._test_dir - ) + 'samples/model_config/wide_and_deep_on_avazau_ctr.config', + self._test_dir) self.assertTrue(self._success) def test_wide_and_deep_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/wide_and_deep_backbone_on_avazau.config', - self._test_dir, + 'samples/model_config/wide_and_deep_backbone_on_avazau.config', + self._test_dir, ) self.assertTrue(self._success) def test_dlrm(self): - self._success = test_utils.test_single_train_eval('samples/model_config/dlrm_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/dlrm_on_taobao.config', self._test_dir) def test_dlrm_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dlrm_backbone_on_taobao.config', self._test_dir - ) + 'samples/model_config/dlrm_backbone_on_taobao.config', self._test_dir) def test_adamw_optimizer(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_adamw_ctr.config', - self._test_dir, + 'samples/model_config/deepfm_combo_on_avazu_adamw_ctr.config', + self._test_dir, ) self.assertTrue(self._success) def test_momentumw_optimizer(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_momentumw_ctr.config', - self._test_dir, + 'samples/model_config/deepfm_combo_on_avazu_momentumw_ctr.config', + self._test_dir, ) self.assertTrue(self._success) def test_deepfm_with_param_edit(self): model_dir = os.path.join(self._test_dir, 'train_new') self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - self._test_dir, - hyperparam_str='{"model_dir":"%s", ' '"model_config.deepfm.wide_output_dim": 32}' % model_dir, + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + self._test_dir, + hyperparam_str='{"model_dir":"%s", ' + '"model_config.deepfm.wide_output_dim": 32}' % model_dir, ) self.assertTrue(self._success) config_path = os.path.join(model_dir, 'pipeline.config') @@ -151,34 +150,33 @@ def test_deepfm_with_param_edit(self): def test_multi_tower(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_on_taobao.config', self._test_dir - ) + 'samples/model_config/multi_tower_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_multi_tower_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_backbone_on_taobao.config', self._test_dir - ) + 'samples/model_config/multi_tower_backbone_on_taobao.config', + self._test_dir) self.assertTrue(self._success) def test_multi_tower_gauc(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_on_taobao_gauc.config', self._test_dir - ) + 'samples/model_config/multi_tower_on_taobao_gauc.config', + self._test_dir) self.assertTrue(self._success) def test_multi_tower_session_auc(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_on_taobao_session_auc.config', - self._test_dir, + 'samples/model_config/multi_tower_on_taobao_session_auc.config', + self._test_dir, ) self.assertTrue(self._success) def test_multi_tower_save_checkpoint_secs(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_save_secs_on_taobao.config', - self._test_dir, - total_steps=100, + 'samples/model_config/multi_tower_save_secs_on_taobao.config', + self._test_dir, + total_steps=100, ) ckpts_times = [] ckpt_dir = os.path.join(self._test_dir, 'train') @@ -190,23 +188,27 @@ def test_multi_tower_save_checkpoint_secs(self): # ensure interval is 20s diffs = list(ckpts_times[1:] - ckpts_times[:-1]) logging.info('nearby ckpts_times diff = %s' % diffs) - self.assertAllClose(ckpts_times[1:] - ckpts_times[:-1], [20] * (len(ckpts_times) - 1), atol=20) + self.assertAllClose( + ckpts_times[1:] - ckpts_times[:-1], [20] * (len(ckpts_times) - 1), + atol=20) self.assertTrue(self._success) def test_keep_ckpt_max(self): + def _post_check_func(pipeline_config): ckpt_prefix = os.path.join(pipeline_config.model_dir, 'model.ckpt-*.meta') ckpts = gfile.Glob(ckpt_prefix) assert len(ckpts) == 3, 'invalid number of checkpoints: %d' % len(ckpts) self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_ckpt_keep_3_on_taobao.config', - self._test_dir, - total_steps=500, - post_check_func=_post_check_func, + 'samples/model_config/multi_tower_ckpt_keep_3_on_taobao.config', + self._test_dir, + total_steps=500, + post_check_func=_post_check_func, ) def test_multi_tower_with_best_exporter(self): + def _post_check_func(pipeline_config): model_dir = pipeline_config.model_dir best_ckpts = os.path.join(model_dir, 'best_ckpt/model.ckpt-*.meta') @@ -214,15 +216,16 @@ def _post_check_func(pipeline_config): assert len(best_ckpts) <= 2, 'too many best ckpts: %s' % str(best_ckpts) best_exports = os.path.join(model_dir, 'export/best/*') best_exports = gfile.Glob(best_exports) - assert len(best_exports) <= 2, 'too many best exports: %s' % str(best_exports) + assert len( + best_exports) <= 2, 'too many best exports: %s' % str(best_exports) return True self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_best_export_on_taobao.config', - self._test_dir, - total_steps=800, - post_check_func=_post_check_func, - timeout=3000, + 'samples/model_config/multi_tower_best_export_on_taobao.config', + self._test_dir, + total_steps=800, + post_check_func=_post_check_func, + timeout=3000, ) self.assertTrue(self._success) @@ -233,17 +236,19 @@ def test_latest_ckpt(self): assert tmp.endswith('model.ckpt-500') def test_latest_ckpt_v2(self): + def _post_check_func(pipeline_config): logging.info('model_dir: %s' % pipeline_config.model_dir) - logging.info('latest_checkpoint: %s' % estimator_utils.latest_checkpoint(pipeline_config.model_dir)) - return tf.train.latest_checkpoint(pipeline_config.model_dir) == estimator_utils.latest_checkpoint( - pipeline_config.model_dir - ) + logging.info('latest_checkpoint: %s' % + estimator_utils.latest_checkpoint(pipeline_config.model_dir)) + return tf.train.latest_checkpoint( + pipeline_config.model_dir) == estimator_utils.latest_checkpoint( + pipeline_config.model_dir) self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.config', - self._test_dir, - post_check_func=_post_check_func, + 'samples/model_config/taobao_fg.config', + self._test_dir, + post_check_func=_post_check_func, ) self.assertTrue(self._success) @@ -266,9 +271,9 @@ def _watch_func(): watch_th.start() self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/taobao_fg_signal_stop.config', - self._test_dir, - total_steps=1000, + 'samples/model_config/taobao_fg_signal_stop.config', + self._test_dir, + total_steps=1000, ) self.assertTrue(self._success) watch_th.join() @@ -281,9 +286,9 @@ def _watch_func(): def test_dead_line_stop_signal(self): train_dir = os.path.join(self._test_dir, 'train/') self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/dead_line_stop.config', - self._test_dir, - total_steps=1000, + 'samples/model_config/dead_line_stop.config', + self._test_dir, + total_steps=1000, ) self.assertTrue(self._success) final_ckpt = estimator_utils.latest_checkpoint(train_dir) @@ -293,124 +298,136 @@ def test_dead_line_stop_signal(self): assert ckpt_version < 1000 def test_fine_tune_latest_ckpt_path(self): + def _post_check_func(pipeline_config): logging.info('model_dir: %s' % pipeline_config.model_dir) pipeline_config = config_util.get_configs_from_pipeline_file( - os.path.join(pipeline_config.model_dir, 'pipeline.config'), False - ) - logging.info('fine_tune_checkpoint: %s' % pipeline_config.train_config.fine_tune_checkpoint) + os.path.join(pipeline_config.model_dir, 'pipeline.config'), False) + logging.info('fine_tune_checkpoint: %s' % + pipeline_config.train_config.fine_tune_checkpoint) return pipeline_config.train_config.fine_tune_checkpoint == 'data/test/mt_ckpt/model.ckpt-100' self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_on_taobao.config', - self._test_dir, - fine_tune_checkpoint='data/test/mt_ckpt', - post_check_func=_post_check_func, + 'samples/model_config/multi_tower_on_taobao.config', + self._test_dir, + fine_tune_checkpoint='data/test/mt_ckpt', + post_check_func=_post_check_func, ) self.assertTrue(self._success) def test_fine_tune_ckpt(self): + def _post_check_func(pipeline_config): - pipeline_config.train_config.fine_tune_checkpoint = estimator_utils.latest_checkpoint(pipeline_config.model_dir) + pipeline_config.train_config.fine_tune_checkpoint = estimator_utils.latest_checkpoint( + pipeline_config.model_dir) test_dir = os.path.join(self._test_dir, 'fine_tune') pipeline_config.model_dir = os.path.join(test_dir, 'ckpt') return test_utils.test_single_train_eval(pipeline_config, test_dir) self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.config', - self._test_dir, - post_check_func=_post_check_func, + 'samples/model_config/taobao_fg.config', + self._test_dir, + post_check_func=_post_check_func, ) self.assertTrue(self._success) def test_multi_tower_multi_value_export(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_multi_value_export_on_taobao.config', - self._test_dir, + 'samples/model_config/multi_tower_multi_value_export_on_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_multi_tower_fg_input(self): - self._success = test_utils.test_single_train_eval('samples/model_config/taobao_fg.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/taobao_fg.config', self._test_dir) self.assertTrue(self._success) def test_multi_tower_fg_json_config(self): - self._success = test_utils.test_single_train_eval('samples/model_config/taobao_fg.json', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/taobao_fg.json', self._test_dir) self.assertTrue(self._success) def test_fm(self): - self._success = test_utils.test_single_train_eval('samples/model_config/fm_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/fm_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_place_embed_on_cpu(self): os.environ['place_embedding_on_cpu'] = 'True' - self._success = test_utils.test_single_train_eval('samples/model_config/fm_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/fm_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_din(self): - self._success = test_utils.test_single_train_eval('samples/model_config/din_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/din_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_din_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/din_backbone_on_taobao.config', self._test_dir - ) + 'samples/model_config/din_backbone_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_bst(self): - self._success = test_utils.test_single_train_eval('samples/model_config/bst_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/bst_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_bst_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/bst_backbone_on_taobao.config', self._test_dir - ) + 'samples/model_config/bst_backbone_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_cl4srec(self): - self._success = test_utils.test_single_train_eval('samples/model_config/cl4srec_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/cl4srec_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dcn(self): - self._success = test_utils.test_single_train_eval('samples/model_config/dcn_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/dcn_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_ziln_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mlp_on_taobao_with_ziln_loss.config', self._test_dir - ) + 'samples/model_config/mlp_on_taobao_with_ziln_loss.config', + self._test_dir) self.assertTrue(self._success) def test_fibinet(self): - self._success = test_utils.test_single_train_eval('samples/model_config/fibinet_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/fibinet_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_masknet(self): - self._success = test_utils.test_single_train_eval('samples/model_config/masknet_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/masknet_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dcn_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dcn_backbone_on_taobao.config', self._test_dir - ) + 'samples/model_config/dcn_backbone_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dcn_with_f1(self): - self._success = test_utils.test_single_train_eval('samples/model_config/dcn_f1_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/dcn_f1_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_autoint(self): - self._success = test_utils.test_single_train_eval('samples/model_config/autoint_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/autoint_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_uniter(self): - self._success = test_utils.test_single_train_eval('samples/model_config/uniter_on_movielens.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/uniter_on_movielens.config', self._test_dir) self.assertTrue(self._success) def test_highway(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/highway_on_movielens.config', self._test_dir - ) + 'samples/model_config/highway_on_movielens.config', self._test_dir) self.assertTrue(self._success) # @unittest.skipIf( @@ -423,116 +440,116 @@ def test_highway(self): # self.assertTrue(self._success) def test_cdn(self): - self._success = test_utils.test_single_train_eval('samples/model_config/cdn_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/cdn_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_ppnet(self): - self._success = test_utils.test_single_train_eval('samples/model_config/ppnet_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/ppnet_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_uniter_only_text_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/uniter_on_movielens_only_text_feature.config', - self._test_dir, + 'samples/model_config/uniter_on_movielens_only_text_feature.config', + self._test_dir, ) self.assertTrue(self._success) def test_uniter_only_image_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/uniter_on_movielens_only_image_feature.config', - self._test_dir, + 'samples/model_config/uniter_on_movielens_only_image_feature.config', + self._test_dir, ) self.assertTrue(self._success) def test_cmbf(self): - self._success = test_utils.test_single_train_eval('samples/model_config/cmbf_on_movielens.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/cmbf_on_movielens.config', self._test_dir) self.assertTrue(self._success) def test_cmbf_with_multi_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/cmbf_with_multi_loss.config', self._test_dir - ) + 'samples/model_config/cmbf_with_multi_loss.config', self._test_dir) self.assertTrue(self._success) def test_cmbf_has_other_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/cmbf_on_movielens_has_other_feature.config', - self._test_dir, + 'samples/model_config/cmbf_on_movielens_has_other_feature.config', + self._test_dir, ) self.assertTrue(self._success) def test_cmbf_only_text_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/cmbf_on_movielens_only_text_feature.config', - self._test_dir, + 'samples/model_config/cmbf_on_movielens_only_text_feature.config', + self._test_dir, ) self.assertTrue(self._success) def test_cmbf_only_image_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/cmbf_on_movielens_only_image_feature.config', - self._test_dir, + 'samples/model_config/cmbf_on_movielens_only_image_feature.config', + self._test_dir, ) self.assertTrue(self._success) def test_dssm(self): - self._success = test_utils.test_single_train_eval('samples/model_config/dssm_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/dssm_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dropoutnet(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dropoutnet_on_taobao.config', self._test_dir - ) + 'samples/model_config/dropoutnet_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_metric_learning(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/metric_learning_on_taobao.config', self._test_dir - ) + 'samples/model_config/metric_learning_on_taobao.config', self._test_dir) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_neg_sampler_on_taobao.config', self._test_dir - ) + 'samples/model_config/dssm_neg_sampler_on_taobao.config', + self._test_dir) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler_v2(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_neg_sampler_v2_on_taobao.config', self._test_dir - ) + 'samples/model_config/dssm_neg_sampler_v2_on_taobao.config', + self._test_dir) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_hard_neg_sampler(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_hard_neg_sampler_on_taobao.config', - self._test_dir, + 'samples/model_config/dssm_hard_neg_sampler_on_taobao.config', + self._test_dir, ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_hard_neg_regular_sampler(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_hard_neg_sampler_regular_on_taobao.config', - self._test_dir, + 'samples/model_config/dssm_hard_neg_sampler_regular_on_taobao.config', + self._test_dir, ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_hard_neg_sampler_v2(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_hard_neg_sampler_v2_on_taobao.config', - self._test_dir, + 'samples/model_config/dssm_hard_neg_sampler_v2_on_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_dssm_no_norm(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_inner_prod_on_taobao.config', self._test_dir - ) + 'samples/model_config/dssm_inner_prod_on_taobao.config', self._test_dir) self.assertTrue(self._success) # def test_dssm_with_regression(self): @@ -574,800 +591,807 @@ def _gen_kd_data(train_path, eval_path): pipeline_config = config_util.get_configs_from_pipeline_file(config1) pipeline_config.train_input_path = train_path pipeline_config.eval_input_path = eval_path - config_util.save_pipeline_config(pipeline_config, self._test_dir, 'kd_pipeline.config') + config_util.save_pipeline_config(pipeline_config, self._test_dir, + 'kd_pipeline.config') self._success = test_utils.test_single_train_eval( - os.path.join(self._test_dir, 'kd_pipeline.config'), - os.path.join(self._test_dir, 'kd'), + os.path.join(self._test_dir, 'kd_pipeline.config'), + os.path.join(self._test_dir, 'kd'), ) self.assertTrue(self._success) def test_dssm_with_kd(self): self._test_kd( - 'samples/model_config/multi_tower_on_taobao.config', - 'samples/model_config/dssm_kd_on_taobao.config', + 'samples/model_config/multi_tower_on_taobao.config', + 'samples/model_config/dssm_kd_on_taobao.config', ) def test_deepfm_multi_class_with_kd(self): self._test_kd( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - 'samples/model_config/deepfm_multi_cls_small.config', + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + 'samples/model_config/deepfm_multi_cls_small.config', ) def test_mind(self): - self._success = test_utils.test_single_train_eval('samples/model_config/mind_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/mind_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_mind_with_time_id(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mind_on_taobao_with_time.config', self._test_dir - ) + 'samples/model_config/mind_on_taobao_with_time.config', self._test_dir) self.assertTrue(self._success) def test_deepfm_with_regression(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_reg.config', self._test_dir - ) + 'samples/model_config/deepfm_combo_on_avazu_reg.config', self._test_dir) self.assertTrue(self._success) def test_deepfm_with_sigmoid_l2_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_sigmoid_l2.config', - self._test_dir, + 'samples/model_config/deepfm_combo_on_avazu_sigmoid_l2.config', + self._test_dir, ) self.assertTrue(self._success) def test_deepfm_with_embedding_learning_rate(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_emblr_ctr.config', - self._test_dir, + 'samples/model_config/deepfm_combo_on_avazu_emblr_ctr.config', + self._test_dir, ) self.assertTrue(self._success) def test_deepfm_with_eval_online(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_eval_online_ctr.config', - self._test_dir, + 'samples/model_config/deepfm_combo_on_avazu_eval_online_ctr.config', + self._test_dir, ) self.assertTrue(self._success) def test_deepfm_with_eval_online_gauc(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_eval_online_gauc_ctr.config', - self._test_dir, + 'samples/model_config/deepfm_combo_on_avazu_eval_online_gauc_ctr.config', + self._test_dir, ) self.assertTrue(self._success) def test_mmoe(self): - self._success = test_utils.test_single_train_eval('samples/model_config/mmoe_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/mmoe_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_mmoe_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mmoe_backbone_on_taobao.config', self._test_dir - ) + 'samples/model_config/mmoe_backbone_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_mmoe_with_multi_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mmoe_on_taobao_with_multi_loss.config', self._test_dir - ) + 'samples/model_config/mmoe_on_taobao_with_multi_loss.config', + self._test_dir) self.assertTrue(self._success) def test_mmoe_deprecated(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mmoe_on_taobao_deprecated.config', self._test_dir - ) + 'samples/model_config/mmoe_on_taobao_deprecated.config', self._test_dir) self.assertTrue(self._success) def test_simple_multi_task(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/simple_multi_task_on_taobao.config', self._test_dir - ) + 'samples/model_config/simple_multi_task_on_taobao.config', + self._test_dir) self.assertTrue(self._success) def test_simple_multi_task_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/simple_multi_task_backbone_on_taobao.config', - self._test_dir, + 'samples/model_config/simple_multi_task_backbone_on_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_esmm(self): - self._success = test_utils.test_single_train_eval('samples/model_config/esmm_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/esmm_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_tag_kv_input(self): - self._success = test_utils.test_single_train_eval('samples/model_config/kv_tag.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/kv_tag.config', self._test_dir) self.assertTrue(self._success) def test_aitm(self): - self._success = test_utils.test_single_train_eval('samples/model_config/aitm_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/aitm_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dbmtl(self): - self._success = test_utils.test_single_train_eval('samples/model_config/dbmtl_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/dbmtl_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dbmtl_backbone(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_backbone_on_taobao.config', self._test_dir - ) + 'samples/model_config/dbmtl_backbone_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_dbmtl_cmbf(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_cmbf_on_movielens.config', self._test_dir - ) + 'samples/model_config/dbmtl_cmbf_on_movielens.config', self._test_dir) self.assertTrue(self._success) def test_dbmtl_uniter(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_uniter_on_movielens.config', self._test_dir - ) + 'samples/model_config/dbmtl_uniter_on_movielens.config', self._test_dir) self.assertTrue(self._success) def test_dbmtl_with_multi_loss(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_taobao_with_multi_loss.config', - self._test_dir, + 'samples/model_config/dbmtl_on_taobao_with_multi_loss.config', + self._test_dir, ) self.assertTrue(self._success) def test_early_stop(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_early_stop_on_taobao.config', - self._test_dir, + 'samples/model_config/multi_tower_early_stop_on_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_early_stop_custom(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/custom_early_stop_on_taobao.config', self._test_dir - ) + 'samples/model_config/custom_early_stop_on_taobao.config', + self._test_dir) self.assertTrue(self._success) def test_early_stop_dis(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_early_stop_on_taobao.config', - self._test_dir, + 'samples/model_config/multi_tower_early_stop_on_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_latest_export_with_asset(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/din_on_taobao_latest_export.config', self._test_dir - ) + 'samples/model_config/din_on_taobao_latest_export.config', + self._test_dir) self.assertTrue(self._success) def test_incompatible_restore(self): + def _post_check_func(config): config.feature_config.features[0].hash_bucket_size += 20000 config.feature_config.features[1].hash_bucket_size += 100 config.train_config.fine_tune_checkpoint = config.model_dir config.model_dir += '_finetune' config.train_config.force_restore_shape_compatible = True - return test_utils.test_single_train_eval(config, os.path.join(self._test_dir, 'finetune')) + return test_utils.test_single_train_eval( + config, os.path.join(self._test_dir, 'finetune')) self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg.config', - self._test_dir, - post_check_func=_post_check_func, + 'samples/model_config/taobao_fg.config', + self._test_dir, + post_check_func=_post_check_func, ) self.assertTrue(self._success) def test_dbmtl_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_variational_dropout.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection, + 'samples/model_config/dbmtl_variational_dropout.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) def test_dbmtl_variational_dropout_feature_num(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_variational_dropout_feature_num.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection, + 'samples/model_config/dbmtl_variational_dropout_feature_num.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) def test_essm_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/esmm_variational_dropout_on_taobao.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection, + 'samples/model_config/esmm_variational_dropout_on_taobao.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) def test_fm_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/fm_variational_dropout_on_taobao.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection, + 'samples/model_config/fm_variational_dropout_on_taobao.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) def test_deepfm_with_combo_feature_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_combo_variational_dropout_on_avazu_ctr.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection, + 'samples/model_config/deepfm_combo_variational_dropout_on_avazu_ctr.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) def test_dbmtl_sequence_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_variational_dropout_on_sequence_feature_taobao.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection, + 'samples/model_config/dbmtl_variational_dropout_on_sequence_feature_taobao.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) def test_din_variational_dropout(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/din_varitional_dropout_on_taobao.config', - self._test_dir, - post_check_func=test_utils.test_feature_selection, + 'samples/model_config/din_varitional_dropout_on_taobao.config', + self._test_dir, + post_check_func=test_utils.test_feature_selection, ) self.assertTrue(self._success) def test_rocket_launching(self): - self._success = test_utils.test_single_train_eval('samples/model_config/rocket_launching.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/rocket_launching.config', self._test_dir) self.assertTrue(self._success) def test_rocket_launching_feature_based(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/rocket_launching_feature_based.config', self._test_dir - ) + 'samples/model_config/rocket_launching_feature_based.config', + self._test_dir) self.assertTrue(self._success) def test_rocket_launching_with_rtp_input(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/rocket_launching_with_rtp_input.config', - self._test_dir, + 'samples/model_config/rocket_launching_with_rtp_input.config', + self._test_dir, ) self.assertTrue(self._success) def test_dbmtl_mmoe(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_mmoe_on_taobao.config', self._test_dir - ) + 'samples/model_config/dbmtl_mmoe_on_taobao.config', self._test_dir) self.assertTrue(self._success) def test_train_with_ps_worker(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao.config', self._test_dir - ) + 'samples/model_config/multi_tower_on_taobao.config', self._test_dir) self.assertTrue(self._success) @unittest.skip('Timeout on CI machine') def test_fit_on_eval(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao.config', - self._test_dir, - total_steps=10, - num_evaluator=1, - fit_on_eval=True, + 'samples/model_config/multi_tower_on_taobao.config', + self._test_dir, + total_steps=10, + num_evaluator=1, + fit_on_eval=True, ) self.assertTrue(self._success) def test_unbalance_data(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao_unblanace.config', - self._test_dir, - total_steps=0, - num_epoch=1, - num_evaluator=1, + 'samples/model_config/multi_tower_on_taobao_unblanace.config', + self._test_dir, + total_steps=0, + num_epoch=1, + num_evaluator=1, ) self.assertTrue(self._success) def test_train_with_ps_worker_with_evaluator(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao.config', - self._test_dir, - num_evaluator=1, + 'samples/model_config/multi_tower_on_taobao.config', + self._test_dir, + num_evaluator=1, ) self.assertTrue(self._success) final_export_dir = os.path.join(self._test_dir, 'train/export/final') all_saved_files = glob.glob(final_export_dir + '/*/saved_model.pb') - logging.info('final_export_dir=%s all_saved_files=%s' % (final_export_dir, ','.join(all_saved_files))) + logging.info('final_export_dir=%s all_saved_files=%s' % + (final_export_dir, ','.join(all_saved_files))) self.assertTrue(len(all_saved_files) == 1) def test_train_with_ps_worker_chief_redundant(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao_chief_redundant.config', - self._test_dir, + 'samples/model_config/multi_tower_on_taobao_chief_redundant.config', + self._test_dir, ) self.assertTrue(self._success) def test_deepfm_embed_input(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_with_embed.config', self._test_dir - ) + 'samples/model_config/deepfm_with_embed.config', self._test_dir) self.assertTrue(self._success) def test_multi_tower_embed_input(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_with_embed.config', self._test_dir - ) + 'samples/model_config/multi_tower_with_embed.config', self._test_dir) self.assertTrue(self._success) def test_tfrecord_input(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_on_criteo_tfrecord.config', self._test_dir - ) + 'samples/model_config/deepfm_on_criteo_tfrecord.config', self._test_dir) self.assertTrue(self._success) def test_batch_tfrecord_input(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_on_criteo_batch_tfrecord.config', - self._test_dir, + 'samples/model_config/deepfm_on_criteo_batch_tfrecord.config', + self._test_dir, ) self.assertTrue(self._success) def test_autodis_embedding(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_on_criteo_with_autodis.config', self._test_dir - ) + 'samples/model_config/deepfm_on_criteo_with_autodis.config', + self._test_dir) self.assertTrue(self._success) def test_periodic_embedding(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_on_criteo_with_periodic.config', self._test_dir - ) + 'samples/model_config/deepfm_on_criteo_with_periodic.config', + self._test_dir) self.assertTrue(self._success) def test_sample_weight(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_with_sample_weight.config', self._test_dir - ) + 'samples/model_config/deepfm_with_sample_weight.config', self._test_dir) self.assertTrue(self._success) def test_dssm_sample_weight(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_with_sample_weight.config', self._test_dir - ) + 'samples/model_config/dssm_with_sample_weight.config', self._test_dir) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler_with_sample_weight(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_neg_sampler_with_sample_weight.config', - self._test_dir, + 'samples/model_config/dssm_neg_sampler_with_sample_weight.config', + self._test_dir, ) self.assertTrue(self._success) @unittest.skipIf( - LooseVersion(tf.__version__) != LooseVersion('2.3.0'), - 'MultiWorkerMirroredStrategy need tf version == 2.3', + LooseVersion(tf.__version__) != LooseVersion('2.3.0'), + 'MultiWorkerMirroredStrategy need tf version == 2.3', ) def test_train_with_multi_worker_mirror(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_multi_worker_mirrored_strategy_on_taobao.config', - self._test_dir, + 'samples/model_config/multi_tower_multi_worker_mirrored_strategy_on_taobao.config', + self._test_dir, ) self.assertTrue(self._success) @unittest.skipIf( - LooseVersion(tf.__version__) != LooseVersion('2.3.0'), - 'MultiWorkerMirroredStrategy need tf version == 2.3', + LooseVersion(tf.__version__) != LooseVersion('2.3.0'), + 'MultiWorkerMirroredStrategy need tf version == 2.3', ) def test_train_mmoe_with_multi_worker_mirror(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/mmoe_mirrored_strategy_on_taobao.config', - self._test_dir, + 'samples/model_config/mmoe_mirrored_strategy_on_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_fg_dtype(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/taobao_fg_test_dtype.config', self._test_dir - ) + 'samples/model_config/taobao_fg_test_dtype.config', self._test_dir) self.assertTrue(self._success) @unittest.skipIf(six.PY2, 'Only run in python3') def test_share_not_used(self): - self._success = test_utils.test_single_train_eval('samples/model_config/share_not_used.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/share_not_used.config', self._test_dir) self.assertTrue(self._success) def test_sequence_autoint(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/autoint_on_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/autoint_on_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_sequence_dcn(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dcn_on_sequence_feature_taobao.config', self._test_dir - ) + 'samples/model_config/dcn_on_sequence_feature_taobao.config', + self._test_dir) self.assertTrue(self._success) def test_sequence_dssm(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_on_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/dssm_on_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_sequence_esmm(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/esmm_on_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/esmm_on_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_sequence_mmoe(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/mmoe_on_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/mmoe_on_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_sequence_ple(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/ple_on_sequence_feature_taobao.config', self._test_dir - ) + 'samples/model_config/ple_on_sequence_feature_taobao.config', + self._test_dir) self.assertTrue(self._success) def test_sequence_rocket_launching(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/rocket_launching_on_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/rocket_launching_on_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_sequence_simple_multi_task(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/simple_multi_task_on_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/simple_multi_task_on_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_sequence_wide_and_deep(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/wide_and_deep_on_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/wide_and_deep_on_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_numeric_boundary_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_numeric_hash_bucket_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_numeric_hash_bucket_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_numeric_hash_bucket_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_numeric_raw_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_numeric_raw_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_numeric_raw_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_numeric_num_buckets_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_numeric_num_buckets_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_numeric_num_buckets_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_multi_numeric_boundary_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_boundary_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_multi_numeric_boundary_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_multi_numeric_hash_bucket_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_hash_bucket_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_multi_numeric_hash_bucket_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_multi_numeric_raw_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_raw_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_multi_numeric_raw_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_multi_numeric_num_buckets_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_num_buckets_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_multi_numeric_num_buckets_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_multi_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_sequence_feature_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_multi_sequence_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_multi_optimizer(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/wide_and_deep_two_opti.config', self._test_dir - ) + 'samples/model_config/wide_and_deep_two_opti.config', self._test_dir) self.assertTrue(self._success) def test_embedding_separate_optimizer(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_embed_adagrad.config', - self._test_dir, + 'samples/model_config/deepfm_combo_on_avazu_embed_adagrad.config', + self._test_dir, ) self.assertTrue(self._success) def test_expr_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_on_taobao_for_expr.config', self._test_dir - ) + 'samples/model_config/multi_tower_on_taobao_for_expr.config', + self._test_dir) self.assertTrue(self._success) def test_gzip_data(self): - self._success = test_utils.test_single_train_eval('samples/model_config/din_on_gzip_data.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/din_on_gzip_data.config', self._test_dir) self.assertTrue(self._success) def test_cmd_config_param(self): + def _post_check_config(pipeline_config): - train_saved_config_path = os.path.join(self._test_dir, 'train/pipeline.config') - pipeline_config = config_util.get_configs_from_pipeline_file(train_saved_config_path) + train_saved_config_path = os.path.join(self._test_dir, + 'train/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file( + train_saved_config_path) assert pipeline_config.model_config.deepfm.wide_output_dim == 8, ( - 'invalid model_config.deepfm.wide_output_dim=%d' % pipeline_config.model_config.deepfm.wide_output_dim - ) + 'invalid model_config.deepfm.wide_output_dim=%d' % + pipeline_config.model_config.deepfm.wide_output_dim) self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - self._test_dir, - post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim 8', + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + self._test_dir, + post_check_func=_post_check_config, + extra_cmd_args='--model_config.deepfm.wide_output_dim 8', ) def test_cmd_config_param_v2(self): + def _post_check_config(pipeline_config): - train_saved_config_path = os.path.join(self._test_dir, 'train/pipeline.config') - pipeline_config = config_util.get_configs_from_pipeline_file(train_saved_config_path) + train_saved_config_path = os.path.join(self._test_dir, + 'train/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file( + train_saved_config_path) assert pipeline_config.model_config.deepfm.wide_output_dim == 1, ( - 'invalid model_config.deepfm.wide_output_dim=%d' % pipeline_config.model_config.deepfm.wide_output_dim - ) + 'invalid model_config.deepfm.wide_output_dim=%d' % + pipeline_config.model_config.deepfm.wide_output_dim) self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - self._test_dir, - post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim=1', + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + self._test_dir, + post_check_func=_post_check_config, + extra_cmd_args='--model_config.deepfm.wide_output_dim=1', ) def test_cmd_config_param_v3(self): + def _post_check_config(pipeline_config): - train_saved_config_path = os.path.join(self._test_dir, 'train/pipeline.config') - pipeline_config = config_util.get_configs_from_pipeline_file(train_saved_config_path) + train_saved_config_path = os.path.join(self._test_dir, + 'train/pipeline.config') + pipeline_config = config_util.get_configs_from_pipeline_file( + train_saved_config_path) assert pipeline_config.model_config.deepfm.wide_output_dim == 3, ( - 'invalid model_config.deepfm.wide_output_dim=%d' % pipeline_config.model_config.deepfm.wide_output_dim - ) + 'invalid model_config.deepfm.wide_output_dim=%d' % + pipeline_config.model_config.deepfm.wide_output_dim) self._success = test_utils.test_single_train_eval( - 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - self._test_dir, - post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim="3"', + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + self._test_dir, + post_check_func=_post_check_config, + extra_cmd_args='--model_config.deepfm.wide_output_dim="3"', ) def test_distribute_eval_deepfm_multi_cls(self): cur_eval_path = 'data/test/distribute_eval_test/deepfm_distribute_eval_dwd_avazu_out_multi_cls' self._success = test_utils.test_distributed_eval( - 'samples/model_config/deepfm_distribute_eval_multi_cls_on_avazu_ctr.config', - cur_eval_path, - self._test_dir, + 'samples/model_config/deepfm_distribute_eval_multi_cls_on_avazu_ctr.config', + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) def test_distribute_eval_deepfm_single_cls(self): cur_eval_path = 'data/test/distribute_eval_test/dwd_distribute_eval_avazu_out_test_combo' self._success = test_utils.test_distributed_eval( - 'samples/model_config/deepfm_distribute_eval_combo_on_avazu_ctr.config', - cur_eval_path, - self._test_dir, + 'samples/model_config/deepfm_distribute_eval_combo_on_avazu_ctr.config', + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) def test_distribute_eval_dssm_pointwise_classification(self): cur_eval_path = 'data/test/distribute_eval_test/dssm_distribute_eval_pointwise_classification_taobao_ckpt' self._success = test_utils.test_distributed_eval( - 'samples/model_config/dssm_distribute_eval_pointwise_classification_on_taobao.config', - cur_eval_path, - self._test_dir, + 'samples/model_config/dssm_distribute_eval_pointwise_classification_on_taobao.config', + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) def test_distribute_eval_dssm_reg(self): cur_eval_path = 'data/test/distribute_eval_test/dssm_distribute_eval_reg_taobao_ckpt' self._success = test_utils.test_distributed_eval( - 'samples/model_config/dssm_distribute_eval_reg_on_taobao.config', - cur_eval_path, - self._test_dir, + 'samples/model_config/dssm_distribute_eval_reg_on_taobao.config', + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) def test_distribute_eval_dropout(self): cur_eval_path = 'data/test/distribute_eval_test/dropoutnet_distribute_eval_taobao_ckpt' self._success = test_utils.test_distributed_eval( - 'samples/model_config/dropoutnet_distribute_eval_on_taobao.config', - cur_eval_path, - self._test_dir, + 'samples/model_config/dropoutnet_distribute_eval_on_taobao.config', + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) def test_distribute_eval_esmm(self): cur_eval_path = 'data/test/distribute_eval_test/esmm_distribute_eval_taobao_ckpt' self._success = test_utils.test_distributed_eval( - 'samples/model_config/esmm_distribute_eval_on_taobao.config', - cur_eval_path, - self._test_dir, + 'samples/model_config/esmm_distribute_eval_on_taobao.config', + cur_eval_path, + self._test_dir, ) self.assertTrue(self._success) def test_share_no_used(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/share_embedding_not_used.config', self._test_dir - ) + 'samples/model_config/share_embedding_not_used.config', self._test_dir) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler_sequence_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_neg_sampler_sequence_feature.config', - self._test_dir, + 'samples/model_config/dssm_neg_sampler_sequence_feature.config', + self._test_dir, ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler_need_key_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_neg_sampler_need_key_feature.config', - self._test_dir, + 'samples/model_config/dssm_neg_sampler_need_key_feature.config', + self._test_dir, ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_need_key_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_boundary_need_key_feature_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_multi_numeric_boundary_need_key_feature_taobao.config', + self._test_dir, ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_allow_key_transform(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_multi_numeric_boundary_allow_key_transform.config', - self._test_dir, + 'samples/model_config/dbmtl_on_multi_numeric_boundary_allow_key_transform.config', + self._test_dir, ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_aux_hist_seq(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_aux_hist_seq_taobao.config', - self._test_dir, + 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_aux_hist_seq_taobao.config', + self._test_dir, ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_multi_tower_recall_neg_sampler_sequence_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_recall_neg_sampler_sequence_feature.config', - self._test_dir, + 'samples/model_config/multi_tower_recall_neg_sampler_sequence_feature.config', + self._test_dir, ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_multi_tower_recall_neg_sampler_only_sequence_feature(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/multi_tower_recall_neg_sampler_only_sequence_feature.config', - self._test_dir, + 'samples/model_config/multi_tower_recall_neg_sampler_only_sequence_feature.config', + self._test_dir, ) self.assertTrue(self._success) @unittest.skipIf(hvd is None, 'horovod is not installed') def test_horovod(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/deepfm_combo_on_avazu_ctr.config', - self._test_dir, - use_hvd=True, + 'samples/model_config/deepfm_combo_on_avazu_ctr.config', + self._test_dir, + use_hvd=True, ) self.assertTrue(self._success) - @unittest.skipIf(hvd is None or sok is None, 'horovod and sok is not installed') + @unittest.skipIf(hvd is None or sok is None, + 'horovod and sok is not installed') def test_sok(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/multi_tower_on_taobao_sok.config', - self._test_dir, - use_hvd=True, + 'samples/model_config/multi_tower_on_taobao_sok.config', + self._test_dir, + use_hvd=True, ) self.assertTrue(self._success) - @unittest.skipIf(six.PY2 or tf_version.split('.')[0] != '2', 'only run on python3 and tf 2.x') + @unittest.skipIf( + six.PY2 or tf_version.split('.')[0] != '2', + 'only run on python3 and tf 2.x') def test_train_parquet(self): os.environ[constant.NO_ARITHMETRIC_OPTI] = '1' self._success = test_utils.test_single_train_eval( - 'samples/model_config/dlrm_on_criteo_parquet.config', self._test_dir - ) + 'samples/model_config/dlrm_on_criteo_parquet.config', self._test_dir) self.assertTrue(self._success) @unittest.skipIf(hvd is None, 'horovod is not installed') def test_train_parquet_embedding_parallel(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/dlrm_on_criteo_parquet_ep.config', - self._test_dir, - use_hvd=True, + 'samples/model_config/dlrm_on_criteo_parquet_ep.config', + self._test_dir, + use_hvd=True, ) self.assertTrue(self._success) @unittest.skipIf(hvd is None, 'horovod is not installed') def test_train_parquet_embedding_parallel_v2(self): self._success = test_utils.test_distributed_train_eval( - 'samples/model_config/dlrm_on_criteo_parquet_ep_v2.config', - self._test_dir, - use_hvd=True, + 'samples/model_config/dlrm_on_criteo_parquet_ep_v2.config', + self._test_dir, + use_hvd=True, ) self.assertTrue(self._success) def test_pdn(self): - self._success = test_utils.test_single_train_eval('samples/model_config/pdn_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/pdn_on_taobao.config', self._test_dir) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_senet(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_senet_on_taobao.config', self._test_dir - ) + 'samples/model_config/dssm_senet_on_taobao.config', self._test_dir) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_backbone_on_taobao(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_on_taobao_backbone.config', self._test_dir - ) + 'samples/model_config/dssm_on_taobao_backbone.config', self._test_dir) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_senet_backbone_on_taobao(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/dssm_senet_on_taobao_backbone.config', self._test_dir - ) + 'samples/model_config/dssm_senet_on_taobao_backbone.config', + self._test_dir) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_parallel_dssm_backbone_on_taobao(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/parallel_dssm_on_taobao_backbone.config', - self._test_dir, + 'samples/model_config/parallel_dssm_on_taobao_backbone.config', + self._test_dir, ) self.assertTrue(self._success) def test_xdeefm_backbone_on_taobao(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/xdeepfm_on_taobao_backbone.config', self._test_dir - ) + 'samples/model_config/xdeepfm_on_taobao_backbone.config', + self._test_dir) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dat_on_taobao(self): - self._success = test_utils.test_single_train_eval('samples/model_config/dat_on_taobao.config', self._test_dir) + self._success = test_utils.test_single_train_eval( + 'samples/model_config/dat_on_taobao.config', self._test_dir) self.assertTrue(self._success) diff --git a/easy_rec/python/test/util_test.py b/easy_rec/python/test/util_test.py index 420069372..d3a0dbb70 100644 --- a/easy_rec/python/test/util_test.py +++ b/easy_rec/python/test/util_test.py @@ -13,43 +13,52 @@ class UtilTest(tf.test.TestCase): + def test_get_ckpt_version(self): - ver = estimator_utils.get_ckpt_version('oss://easyrec/ckpts/model.ckpt-6500.meta') + ver = estimator_utils.get_ckpt_version( + 'oss://easyrec/ckpts/model.ckpt-6500.meta') assert ver == 6500, 'invalid version: %s' % str(ver) - ver = estimator_utils.get_ckpt_version('oss://easyrec/ckpts/model.ckpt-6500') + ver = estimator_utils.get_ckpt_version( + 'oss://easyrec/ckpts/model.ckpt-6500') assert ver == 6500, 'invalid version: %s' % str(ver) def test_get_expression_greater(self): - result = get_expression('age_level>item_age_level', ['age_level', 'item_age_level']) + result = get_expression('age_level>item_age_level', + ['age_level', 'item_age_level']) assert result == "tf.greater(parsed_dict['age_level'], parsed_dict['item_age_level'])" def test_get_expression_greater_equal(self): - result = get_expression('age_level>=item_age_level', ['age_level', 'item_age_level']) + result = get_expression('age_level>=item_age_level', + ['age_level', 'item_age_level']) assert result == "tf.greater_equal(parsed_dict['age_level'], parsed_dict['item_age_level'])" def test_get_expression_less(self): - result = get_expression('age_level3)&(item_age_level<1)', ['age_level', 'item_age_level']) + result = get_expression('(age_level>3)&(item_age_level<1)', + ['age_level', 'item_age_level']) assert result == "tf.greater(parsed_dict['age_level'], 3) & tf.less(parsed_dict['item_age_level'], 1)" result = get_expression( - '(age_level>item_age_level) & (age_levelitem_age_level) & (age_level3)|(item_age_level<1)', ['age_level', 'item_age_level']) + result = get_expression('(age_level>3)|(item_age_level<1)', + ['age_level', 'item_age_level']) assert result == "tf.greater(parsed_dict['age_level'], 3) | tf.less(parsed_dict['item_age_level'], 1)" def test_dag(self): diff --git a/easy_rec/python/test/zero_inflated_lognormal_test.py b/easy_rec/python/test/zero_inflated_lognormal_test.py index 064209e77..f88f481fe 100644 --- a/easy_rec/python/test/zero_inflated_lognormal_test.py +++ b/easy_rec/python/test/zero_inflated_lognormal_test.py @@ -5,14 +5,14 @@ from scipy import stats from easy_rec.python.loss.zero_inflated_lognormal import ( # NOQA - zero_inflated_lognormal_loss, -) + zero_inflated_lognormal_loss,) if tf.__version__ >= '2.0': tf = tf.compat.v1 class ZeroInflatedLognormalLossTest(tf.test.TestCase): + def setUp(self): super(ZeroInflatedLognormalLossTest, self).setUp() self.logits = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]) @@ -42,10 +42,10 @@ def zero_inflated_lognormal(self, labels, logits, max_sigma=5.0): if np.any(mask): # scipy 参数化：s=shape= sigma, scale=exp(mu), loc=0 logprob[mask, 0] = stats.lognorm.logpdf( - x=labels[mask, 0].astype(np.float64), - s=sigma[mask, 0].astype(np.float64), - loc=0.0, - scale=np.exp(mu[mask, 0].astype(np.float64)), + x=labels[mask, 0].astype(np.float64), + s=sigma[mask, 0].astype(np.float64), + loc=0.0, + scale=np.exp(mu[mask, 0].astype(np.float64)), ) num_pos = np.sum(positive) + 1e-8 regression_loss = -(np.sum(positive * logprob) / num_pos) @@ -56,7 +56,8 @@ def zero_inflated_lognormal(self, labels, logits, max_sigma=5.0): def test_loss_value(self): expected_loss = self.zero_inflated_lognormal(self.labels, self.logits) expected_loss = np.average(expected_loss) - loss = zero_inflated_lognormal_loss(self.labels, self.logits, mu_reg=0, sigma_reg=0) + loss = zero_inflated_lognormal_loss( + self.labels, self.logits, mu_reg=0, sigma_reg=0) # Absolute error tolerance in asserting array near. _ERR_TOL = 1e-6 self.assertNear(self.evaluate(loss), expected_loss, _ERR_TOL) diff --git a/easy_rec/python/tools/add_boundaries_to_config.py b/easy_rec/python/tools/add_boundaries_to_config.py index cae47e133..f78b94e46 100644 --- a/easy_rec/python/tools/add_boundaries_to_config.py +++ b/easy_rec/python/tools/add_boundaries_to_config.py @@ -8,27 +8,34 @@ import common_io import tensorflow as tf -from easy_rec.python.utils import config_util, io_util +from easy_rec.python.utils import config_util +from easy_rec.python.utils import io_util if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) -tf.app.flags.DEFINE_string('template_config_path', None, 'Path to template pipeline config ' 'file.') -tf.app.flags.DEFINE_string('output_config_path', None, 'Path to output pipeline config ' 'file.') +tf.app.flags.DEFINE_string('template_config_path', None, + 'Path to template pipeline config ' + 'file.') +tf.app.flags.DEFINE_string('output_config_path', None, + 'Path to output pipeline config ' + 'file.') tf.app.flags.DEFINE_string('tables', '', 'quantile binning table') FLAGS = tf.app.flags.FLAGS def main(argv): - pipeline_config = config_util.get_configs_from_pipeline_file(FLAGS.template_config_path) + pipeline_config = config_util.get_configs_from_pipeline_file( + FLAGS.template_config_path) feature_boundaries_info = {} - reader = common_io.table.TableReader(FLAGS.tables, selected_cols='feature,json') + reader = common_io.table.TableReader( + FLAGS.tables, selected_cols='feature,json') while True: try: record = reader.read() diff --git a/easy_rec/python/tools/add_feature_info_to_config.py b/easy_rec/python/tools/add_feature_info_to_config.py index a564b2bed..d525bf893 100644 --- a/easy_rec/python/tools/add_feature_info_to_config.py +++ b/easy_rec/python/tools/add_feature_info_to_config.py @@ -7,35 +7,41 @@ import tensorflow as tf -from easy_rec.python.utils import config_util, io_util +from easy_rec.python.utils import config_util +from easy_rec.python.utils import io_util from easy_rec.python.utils.hive_utils import HiveUtils if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) -tf.app.flags.DEFINE_string('template_config_path', None, 'Path to template pipeline config ' 'file.') -tf.app.flags.DEFINE_string('output_config_path', None, 'Path to output pipeline config ' 'file.') +tf.app.flags.DEFINE_string('template_config_path', None, + 'Path to template pipeline config ' + 'file.') +tf.app.flags.DEFINE_string('output_config_path', None, + 'Path to output pipeline config ' + 'file.') tf.app.flags.DEFINE_string('config_table', '', 'config table') FLAGS = tf.app.flags.FLAGS def main(argv): - pipeline_config = config_util.get_configs_from_pipeline_file(FLAGS.template_config_path) + pipeline_config = config_util.get_configs_from_pipeline_file( + FLAGS.template_config_path) sels = 'feature,feature_info,message' feature_info_map = {} drop_feature_names = [] if pipeline_config.WhichOneof('train_path') == 'hive_train_input': hive_util = HiveUtils( - data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input, - selected_cols=sels, - record_defaults=['', '', ''], + data_config=pipeline_config.data_config, + hive_config=pipeline_config.hive_train_input, + selected_cols=sels, + record_defaults=['', '', ''], ) reader = hive_util.hive_read_line(FLAGS.config_table) for record in reader: @@ -70,18 +76,24 @@ def main(argv): feature_name = feature_config.input_names[0] if feature_name in feature_info_map: logging.info('edited %s' % feature_name) - feature_config.embedding_dim = int(feature_info_map[feature_name]['embedding_dim']) + feature_config.embedding_dim = int( + feature_info_map[feature_name]['embedding_dim']) logging.info('modify embedding_dim to %s' % feature_config.embedding_dim) if 'boundary' in feature_info_map[feature_name]: feature_config.ClearField('boundaries') - feature_config.boundaries.extend([float(i) for i in feature_info_map[feature_name]['boundary']]) + feature_config.boundaries.extend( + [float(i) for i in feature_info_map[feature_name]['boundary']]) logging.info('modify boundaries to %s' % feature_config.boundaries) elif 'hash_bucket_size' in feature_info_map[feature_name]: - feature_config.hash_bucket_size = int(feature_info_map[feature_name]['hash_bucket_size']) - logging.info('modify hash_bucket_size to %s' % feature_config.hash_bucket_size) + feature_config.hash_bucket_size = int( + feature_info_map[feature_name]['hash_bucket_size']) + logging.info('modify hash_bucket_size to %s' % + feature_config.hash_bucket_size) # modify num_steps - pipeline_config.train_config.num_steps = feature_info_map['__NUM_STEPS__']['num_steps'] - logging.info('modify num_steps to %s' % pipeline_config.train_config.num_steps) + pipeline_config.train_config.num_steps = feature_info_map['__NUM_STEPS__'][ + 'num_steps'] + logging.info('modify num_steps to %s' % + pipeline_config.train_config.num_steps) # modify decay_steps optimizer_configs = pipeline_config.train_config.optimizer_config for optimizer_config in optimizer_configs: @@ -90,7 +102,8 @@ def main(argv): learning_rate = optimizer.learning_rate.WhichOneof('learning_rate') learning_rate = getattr(optimizer.learning_rate, learning_rate) if hasattr(learning_rate, 'decay_steps'): - learning_rate.decay_steps = feature_info_map['__DECAY_STEPS__']['decay_steps'] + learning_rate.decay_steps = feature_info_map['__DECAY_STEPS__'][ + 'decay_steps'] logging.info('modify decay_steps to %s' % learning_rate.decay_steps) for feature_group in pipeline_config.model_config.feature_groups: diff --git a/easy_rec/python/tools/convert_config_format.py b/easy_rec/python/tools/convert_config_format.py index 49fbcc355..5cfda1219 100644 --- a/easy_rec/python/tools/convert_config_format.py +++ b/easy_rec/python/tools/convert_config_format.py @@ -2,7 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import os -from google.protobuf import json_format, text_format +from google.protobuf import json_format +from google.protobuf import text_format from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig @@ -25,7 +26,9 @@ def save_config(pipeline_config, save_path): if save_path.endswith('.config'): fout.write(text_format.MessageToString(pipeline_config, as_utf8=True)) elif save_path.endswith('.json'): - fout.write(json_format.MessageToJson(pipeline_config, preserving_proto_field_name=True)) + fout.write( + json_format.MessageToJson( + pipeline_config, preserving_proto_field_name=True)) else: assert False, 'only .config/.json are supported(%s)' % save_path @@ -34,8 +37,10 @@ def save_config(pipeline_config, save_path): import argparse parser = argparse.ArgumentParser() - parser.add_argument('--input_config', type=str, help='input_config path', default=None) - parser.add_argument('--output_config', type=str, help='output_config path', default=None) + parser.add_argument( + '--input_config', type=str, help='input_config path', default=None) + parser.add_argument( + '--output_config', type=str, help='output_config path', default=None) args = parser.parse_args() assert os.path.exists(args.input_config) diff --git a/easy_rec/python/tools/convert_rtp_data.py b/easy_rec/python/tools/convert_rtp_data.py index 60ae1fcc7..97513231d 100644 --- a/easy_rec/python/tools/convert_rtp_data.py +++ b/easy_rec/python/tools/convert_rtp_data.py @@ -19,8 +19,8 @@ import tensorflow as tf logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) if tf.__version__ >= '2.0': @@ -28,7 +28,8 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--rtp_fg', type=str, default='', help='rtp fg path(.json)') + parser.add_argument( + '--rtp_fg', type=str, default='', help='rtp fg path(.json)') parser.add_argument('--input_path', type=str, default='', help='input path') parser.add_argument('--output_path', type=str, default='', help='output path') parser.add_argument('--label', type=str, default='', help='label for train') diff --git a/easy_rec/python/tools/convert_rtp_fg.py b/easy_rec/python/tools/convert_rtp_fg.py index 5c9954610..cfe7434bd 100644 --- a/easy_rec/python/tools/convert_rtp_fg.py +++ b/easy_rec/python/tools/convert_rtp_fg.py @@ -12,8 +12,8 @@ from easy_rec.python.utils.convert_rtp_fg import convert_rtp_fg logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) if tf.__version__ >= '2.0': @@ -23,52 +23,64 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--model_type', - type=str, - choices=model_types, - default='', - help='model type, currently support: %s' % ','.join(model_types), + '--model_type', + type=str, + choices=model_types, + default='', + help='model type, currently support: %s' % ','.join(model_types), ) parser.add_argument('--rtp_fg', type=str, help='rtp fg path') - parser.add_argument('--embedding_dim', type=int, default=16, help='embedding_dimension') - parser.add_argument('--batch_size', type=int, default=1024, help='batch_size for train') - parser.add_argument('--label', type=str, default='', nargs='+', required=True, help='label fields') parser.add_argument( - '--num_steps', - type=int, - default=1000, - help='number of train steps = num_samples * num_epochs / batch_size / num_workers', + '--embedding_dim', type=int, default=16, help='embedding_dimension') + parser.add_argument( + '--batch_size', type=int, default=1024, help='batch_size for train') + parser.add_argument( + '--label', + type=str, + default='', + nargs='+', + required=True, + help='label fields') + parser.add_argument( + '--num_steps', + type=int, + default=1000, + help='number of train steps = num_samples * num_epochs / batch_size / num_workers', ) parser.add_argument('--output_path', type=str, help='generated config path') parser.add_argument( - '--incol_separator', - type=str, - default='\003', - help='separator for multi_value features', + '--incol_separator', + type=str, + default='\003', + help='separator for multi_value features', ) parser.add_argument( - '--separator', - type=str, - default='\002', - help='separator between different features', + '--separator', + type=str, + default='\002', + help='separator between different features', ) - parser.add_argument('--train_input_path', type=str, default=None, help='train data path') - parser.add_argument('--eval_input_path', type=str, default=None, help='eval data path') parser.add_argument( - '--selected_cols', - type=str, - default=None, - help='selected cols, for csv input, it is in the format of: label_col_id0,...,lable_cold_idn,feature_col_id ' - 'for odps table input, it is in the format of: label_col_name0,...,label_col_namen,feature_col_name ', + '--train_input_path', type=str, default=None, help='train data path') + parser.add_argument( + '--eval_input_path', type=str, default=None, help='eval data path') + parser.add_argument( + '--selected_cols', + type=str, + default=None, + help='selected cols, for csv input, it is in the format of: label_col_id0,...,lable_cold_idn,feature_col_id ' + 'for odps table input, it is in the format of: label_col_name0,...,label_col_namen,feature_col_name ', ) - parser.add_argument('--rtp_separator', type=str, default=';', help='separator') parser.add_argument( - '--input_type', - type=str, - default='OdpsRTPInput', - help='default to OdpsRTPInput, if test local, change it to RTPInput', + '--rtp_separator', type=str, default=';', help='separator') + parser.add_argument( + '--input_type', + type=str, + default='OdpsRTPInput', + help='default to OdpsRTPInput, if test local, change it to RTPInput', ) - parser.add_argument('--is_async', action='store_true', help='async mode, debug to false') + parser.add_argument( + '--is_async', action='store_true', help='async mode, debug to false') args = parser.parse_args() @@ -81,29 +93,29 @@ sys.exit(1) pipeline_config = convert_rtp_fg( - args.rtp_fg, - args.embedding_dim, - args.batch_size, - args.label, - args.num_steps, - args.model_type, - args.separator, - args.incol_separator, - args.train_input_path, - args.eval_input_path, - args.selected_cols, - args.input_type, - args.is_async, + args.rtp_fg, + args.embedding_dim, + args.batch_size, + args.label, + args.num_steps, + args.model_type, + args.separator, + args.incol_separator, + args.train_input_path, + args.eval_input_path, + args.selected_cols, + args.input_type, + args.is_async, ) save_message(pipeline_config, args.output_path) logging.info('Conversion done.') logging.info('Tips:') logging.info( - 'if run on local, please change data_config.input_type to RTPInput, ' - 'and model_dir/train_input_path/eval_input_path must also be set, ' - ) + 'if run on local, please change data_config.input_type to RTPInput, ' + 'and model_dir/train_input_path/eval_input_path must also be set, ') logging.info( - 'if run local, please set data_config.selected_cols in the format ' - 'label_col_id0,label_col_id1,...,label_col_idn,feature_col_id' - ) - logging.info('if run on odps, selected_cols must be set, which are label0_col,' 'label1_col, ..., feature_col_name') + 'if run local, please set data_config.selected_cols in the format ' + 'label_col_id0,label_col_id1,...,label_col_idn,feature_col_id') + logging.info( + 'if run on odps, selected_cols must be set, which are label0_col,' + 'label1_col, ..., feature_col_name') diff --git a/easy_rec/python/tools/create_config_from_excel.py b/easy_rec/python/tools/create_config_from_excel.py index f04689408..cf31bdb1b 100644 --- a/easy_rec/python/tools/create_config_from_excel.py +++ b/easy_rec/python/tools/create_config_from_excel.py @@ -9,20 +9,22 @@ from easy_rec.python.utils import config_util -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') class ModelConfigConverter: + def __init__( - self, - excel_path, - output_path, - model_type, - column_separator, - incol_separator, - train_input_path, - eval_input_path, - model_dir, + self, + excel_path, + output_path, + model_type, + column_separator, + incol_separator, + train_input_path, + eval_input_path, + model_dir, ): self._excel_path = excel_path self._output_path = output_path @@ -43,21 +45,21 @@ def __init__( def _get_type_name(self, input_name): type_dict = { - 'bigint': 'INT64', - 'double': 'DOUBLE', - 'float': 'FLOAT', - 'string': 'STRING', - 'bool': 'BOOL', + 'bigint': 'INT64', + 'double': 'DOUBLE', + 'float': 'FLOAT', + 'string': 'STRING', + 'bool': 'BOOL', } return type_dict[input_name] def _get_type_default(self, input_name): type_dict = { - 'bigint': '0', - 'double': '0.0', - 'float': '0.0', - 'string': '', - 'bool': 'false', + 'bigint': '0', + 'double': '0.0', + 'float': '0.0', + 'string': '', + 'bool': 'false', } return type_dict[input_name] @@ -87,9 +89,9 @@ def _add_to_tower(self, tower_name, field): tower_names = ['wide', 'deep'] else: raise ValueError( - 'invalid tower_name[%s] for deepfm model, ' - 'only[label, deep, wide, wide_and_deep are supported]' % tower_name - ) + 'invalid tower_name[%s] for deepfm model, ' + 'only[label, deep, wide, wide_and_deep are supported]' % + tower_name) for tower_name in tower_names: if tower_name in self._tower_dicts: self._tower_dicts[tower_name].append(field) @@ -135,21 +137,24 @@ def _is_good(v): return str(v) not in ['nan', ''] if _is_good(self._dict_global[field['global']]['default_value']): - field['default_value'] = self._dict_global[field['global']]['default_value'] + field['default_value'] = self._dict_global[ + field['global']]['default_value'] if _is_good(self._dict_global[field['global']]['hash_bucket_size']): - field['hash_bucket_size'] = self._dict_global[field['global']]['hash_bucket_size'] + field['hash_bucket_size'] = self._dict_global[ + field['global']]['hash_bucket_size'] if _is_good(self._dict_global[field['global']]['embedding_dim']): - field['embedding_dim'] = self._dict_global[field['global']]['embedding_dim'] + field['embedding_dim'] = self._dict_global[ + field['global']]['embedding_dim'] field['embedding_name'] = field['global'] for t in [ - 'type', - 'global', - 'hash_bucket_size', - 'embedding_dim', - 'default_value', - 'weights', - 'boundaries', + 'type', + 'global', + 'hash_bucket_size', + 'embedding_dim', + 'default_value', + 'weights', + 'boundaries', ]: if t not in row: continue @@ -188,14 +193,14 @@ def _is_good(v): # check that tag features weights are one of the fields for name, config in self._feature_details.items(): if config['type'] == 'tags': - if 'weights' in config and config['weights'] not in self._feature_details: + if 'weights' in config and config[ + 'weights'] not in self._feature_details: raise ValueError(config['weights'] + ' not in field names') def _write_train_eval_config(self, fout): fout.write('train_input_path: "%s"\n' % self._train_input_path) fout.write('eval_input_path: "%s"\n' % self._eval_input_path) - fout.write( - """ + fout.write(""" model_dir: "%s" train_config { @@ -221,9 +226,7 @@ def _write_train_eval_config(self, fout): metrics_set: { auc {} } - }""" - % self._model_dir - ) + }""" % self._model_dir) def _write_deepfm_config(self, fout): # write model_config @@ -245,8 +248,7 @@ def _write_deepfm_config(self, fout): fout.write(' }\n') # write deepfm configs - fout.write( - """ + fout.write(""" deepfm { dnn { hidden_units: [128, 64, 32] @@ -259,8 +261,7 @@ def _write_deepfm_config(self, fout): } embedding_regularization: 1e-5 } - """ - ) + """) def _write_multi_tower_config(self, fout): # write model_config @@ -285,27 +286,22 @@ def _write_multi_tower_config(self, fout): fout.write('multi_tower { \n') for tower_name in tower_names: - fout.write( - """ + fout.write(""" towers { input: "%s" dnn { hidden_units: [256, 192, 128] } - }""" - % tower_name - ) + }""" % tower_name) - fout.write( - """ + fout.write(""" final_dnn { hidden_units: [192, 128, 64] } l2_regularization: 1e-5 } embedding_regularization: 1e-5 - }""" - ) + }""") def _write_data_config(self, fout): fout.write('data_config {\n') @@ -313,19 +309,19 @@ def _write_data_config(self, fout): for name in self._feature_names: fout.write(' input_fields: {\n') fout.write(' input_name: "%s"\n' % name) - fout.write(' input_type: %s\n' % self._get_type_name(self._feature_details[name]['data_type'])) + fout.write(' input_type: %s\n' % + self._get_type_name(self._feature_details[name]['data_type'])) if 'default_value' in self._feature_details[name]: - fout.write(' default_val:"%s"\n' % self._feature_details[name]['default_value']) + fout.write(' default_val:"%s"\n' % + self._feature_details[name]['default_value']) fout.write(' }\n') fout.write(' label_fields: "%s"\n' % self._label) - fout.write( - """ + fout.write(""" batch_size: 1024 prefetch_size: 32 input_type: CSVInput - }""" - ) + }""") def _write_feature_config(self, fout): for name in self._feature_names: @@ -345,9 +341,11 @@ def _write_feature_config(self, fout): elif feature['type'] == 'dense': fout.write(' feature_type: RawFeature\n') if self._model_type == 'deepfm': - assert feature['boundaries'] != '', 'raw features must be discretized by specifying boundaries' + assert feature[ + 'boundaries'] != '', 'raw features must be discretized by specifying boundaries' if 'boundaries' in feature and feature['boundaries'] != '': - fout.write(' boundaries: [%s]\n' % str(feature['boundaries']).strip()) + fout.write(' boundaries: [%s]\n' % + str(feature['boundaries']).strip()) fout.write(' embedding_dim: %d\n' % int(feature['embedding_dim'])) elif feature['type'] == 'tags': if 'weights' in feature: @@ -373,7 +371,9 @@ def _write_feature_config(self, fout): def convert(self): self._parse_features() - logging.info('TOWERS[%d]: %s' % (len(self._tower_dicts), ','.join(list(self._tower_dicts.keys())))) + logging.info( + 'TOWERS[%d]: %s' % + (len(self._tower_dicts), ','.join(list(self._tower_dicts.keys())))) with open(self._output_path, 'w') as fout: self._write_train_eval_config(fout) self._write_data_config(fout) @@ -384,10 +384,11 @@ def convert(self): self._write_multi_tower_config(fout) else: logging.warning( - 'the model_config could not be generated automatically, you have to write the model_config manually' + 'the model_config could not be generated automatically, you have to write the model_config manually' ) # reformat the config - pipeline_config = config_util.get_configs_from_pipeline_file(self._output_path) + pipeline_config = config_util.get_configs_from_pipeline_file( + self._output_path) config_util.save_message(pipeline_config, self._output_path) @@ -398,27 +399,29 @@ def convert(self): parser = argparse.ArgumentParser() parser.add_argument( - '--model_type', - type=str, - choices=model_types, - help='model type, currently support: %s' % ','.join(model_types), + '--model_type', + type=str, + choices=model_types, + help='model type, currently support: %s' % ','.join(model_types), ) parser.add_argument('--excel_path', type=str, help='excel config path') parser.add_argument('--output_path', type=str, help='generated config path') parser.add_argument( - '--column_separator', - type=str, - default=',', - help='column separator, separator between features', + '--column_separator', + type=str, + default=',', + help='column separator, separator between features', ) parser.add_argument( - '--incol_separator', - type=str, - default='|', - help='separator within features, such as tag features', + '--incol_separator', + type=str, + default='|', + help='separator within features, such as tag features', ) - parser.add_argument('--train_input_path', type=str, default='', help='train input path') - parser.add_argument('--eval_input_path', type=str, default='', help='eval input path') + parser.add_argument( + '--train_input_path', type=str, default='', help='train input path') + parser.add_argument( + '--eval_input_path', type=str, default='', help='eval input path') parser.add_argument('--model_dir', type=str, default='', help='model dir') args = parser.parse_args() @@ -426,17 +429,18 @@ def convert(self): parser.print_usage() sys.exit(1) - logging.info('column_separator = %s in_column_separator = %s' % (args.column_separator, args.incol_separator)) + logging.info('column_separator = %s in_column_separator = %s' % + (args.column_separator, args.incol_separator)) converter = ModelConfigConverter( - args.excel_path, - args.output_path, - args.model_type, - args.column_separator, - args.incol_separator, - args.train_input_path, - args.eval_input_path, - args.model_dir, + args.excel_path, + args.output_path, + args.model_type, + args.column_separator, + args.incol_separator, + args.train_input_path, + args.eval_input_path, + args.model_dir, ) converter.convert() logging.info('Conversion done') diff --git a/easy_rec/python/tools/criteo/convert_data.py b/easy_rec/python/tools/criteo/convert_data.py index af8fe304c..222a25a75 100644 --- a/easy_rec/python/tools/criteo/convert_data.py +++ b/easy_rec/python/tools/criteo/convert_data.py @@ -12,7 +12,8 @@ import six from tensorflow.python.platform import gfile -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') def save_np_bin(labels, dense_arr, cate_arr, prefix): @@ -37,7 +38,8 @@ def save_parquet(labels, dense_arr, cate_arr, prefix): def convert(input_path, prefix, part_record_num, save_format): - logging.info('start to convert %s, part_record_num=%d, save_format=%s' % (input_path, part_record_num, save_format)) + logging.info('start to convert %s, part_record_num=%d, save_format=%s' % + (input_path, part_record_num, save_format)) save_func = save_np_bin if save_format == 'parquet': save_func = save_parquet @@ -74,10 +76,10 @@ def convert(input_path, prefix, part_record_num, save_format): sid = 0 if sid > 0: save_func( - labels[:sid], - dense_arr[:sid], - cate_arr[:sid], - prefix + '_' + str(part_id), + labels[:sid], + dense_arr[:sid], + cate_arr[:sid], + prefix + '_' + str(part_id), ) logging.info('\t%s write final part: %d' % (input_path, part_id)) part_id += 1 @@ -86,7 +88,8 @@ def convert(input_path, prefix, part_record_num, save_format): logging.error('convert %s failed: %s' % (input_path, str(ex))) logging.error(traceback.format_exc()) return - logging.info('done convert %s, total_line=%d, part_num=%d' % (input_path, total_line, part_id)) + logging.info('done convert %s, total_line=%d, part_num=%d' % + (input_path, total_line, part_id)) if __name__ == '__main__': @@ -105,25 +108,30 @@ def convert(input_path, prefix, part_record_num, save_format): """ parser = argparse.ArgumentParser() - parser.add_argument('--input_dir', type=str, default=None, help='criteo 1t data dir') - parser.add_argument('--save_dir', type=str, default=None, help='criteo binary data output dir ') parser.add_argument( - '--save_format', - type=str, - default='npy', - help='save format, choices: npy|parquet', + '--input_dir', type=str, default=None, help='criteo 1t data dir') + parser.add_argument( + '--save_dir', + type=str, + default=None, + help='criteo binary data output dir ') + parser.add_argument( + '--save_format', + type=str, + default='npy', + help='save format, choices: npy|parquet', ) parser.add_argument( - '--part_record_num', - type=int, - default=1024 * 1024 * 8, - help='the maximal number of samples in each binary file', + '--part_record_num', + type=int, + default=1024 * 1024 * 8, + help='the maximal number of samples in each binary file', ) parser.add_argument( - '--dt', - nargs='*', - type=int, - help='select days to convert, default to select all: 0-23', + '--dt', + nargs='*', + type=int, + help='select days to convert, default to select all: 0-23', ) args = parser.parse_args() @@ -147,8 +155,8 @@ def convert(input_path, prefix, part_record_num, save_format): input_path = os.path.join(args.input_dir, 'day_%d.gz' % d) prefix = os.path.join(args.save_dir, str(d)) proc = multiprocessing.Process( - target=convert, - args=(input_path, prefix, args.part_record_num, args.save_format), + target=convert, + args=(input_path, prefix, args.part_record_num, args.save_format), ) convert(input_path, prefix, args.part_record_num, args.save_format) proc.start() diff --git a/easy_rec/python/tools/edit_lookup_graph.py b/easy_rec/python/tools/edit_lookup_graph.py index 00d5aa8ee..284087772 100644 --- a/easy_rec/python/tools/edit_lookup_graph.py +++ b/easy_rec/python/tools/edit_lookup_graph.py @@ -7,15 +7,16 @@ import tensorflow as tf from tensorflow.core.protobuf import saved_model_pb2 -from tensorflow.python.lib.io.file_io import file_exists, recursive_create_dir +from tensorflow.python.lib.io.file_io import file_exists +from tensorflow.python.lib.io.file_io import recursive_create_dir from tensorflow.python.platform.gfile import GFile import easy_rec from easy_rec.python.utils.meta_graph_editor import MetaGraphEditor logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) if __name__ == '__main__': @@ -31,12 +32,16 @@ """ parser = argparse.ArgumentParser() - parser.add_argument('--saved_model_dir', type=str, default=None, help='saved model dir') + parser.add_argument( + '--saved_model_dir', type=str, default=None, help='saved model dir') parser.add_argument('--output_dir', type=str, default=None, help='output dir') - parser.add_argument('--redis_url', type=str, default='127.0.0.1:6379', help='redis url') - parser.add_argument('--redis_passwd', type=str, default='', help='redis password') + parser.add_argument( + '--redis_url', type=str, default='127.0.0.1:6379', help='redis url') + parser.add_argument( + '--redis_passwd', type=str, default='', help='redis password') parser.add_argument('--time_out', type=int, default=1500, help='timeout') - parser.add_argument('--test_data_path', type=str, default='', help='test data path') + parser.add_argument( + '--test_data_path', type=str, default='', help='test data path') parser.add_argument('--verbose', action='store_true', default=False) args = parser.parse_args() @@ -55,19 +60,21 @@ recursive_create_dir(args.output_dir) meta_graph_editor = MetaGraphEditor( - lookup_lib_path, - args.saved_model_dir, - args.redis_url, - args.redis_passwd, - args.time_out, - meta_graph_def=None, - debug_dir=args.output_dir if args.verbose else '', + lookup_lib_path, + args.saved_model_dir, + args.redis_url, + args.redis_passwd, + args.time_out, + meta_graph_def=None, + debug_dir=args.output_dir if args.verbose else '', ) meta_graph_editor.edit_graph() meta_graph_version = meta_graph_editor.meta_graph_version if meta_graph_version == '': - export_ts = [x for x in args.saved_model_dir.split('/') if x != '' and x is not None] + export_ts = [ + x for x in args.saved_model_dir.split('/') if x != '' and x is not None + ] meta_graph_version = export_ts[-1] # import edit graph @@ -77,10 +84,13 @@ embed_name_to_id_file = os.path.join(args.output_dir, 'embed_name_to_ids.txt') with GFile(embed_name_to_id_file, 'w') as fout: for tmp_norm_name in meta_graph_editor._embed_name_to_ids: - fout.write('%s\t%s\n' % (tmp_norm_name, meta_graph_editor._embed_name_to_ids[tmp_norm_name])) + fout.write( + '%s\t%s\n' % + (tmp_norm_name, meta_graph_editor._embed_name_to_ids[tmp_norm_name])) tf.add_to_collection( - tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt'), + tf.GraphKeys.ASSET_FILEPATHS, + tf.constant( + embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt'), ) graph = tf.get_default_graph() @@ -98,14 +108,16 @@ with tf.Session() as sess: saver.restore(sess, args.saved_model_dir + '/variables/variables') output_dir = os.path.join(args.output_dir, meta_graph_version) - tf.saved_model.simple_save(sess, output_dir, inputs=inputs_map, outputs=outputs_map) + tf.saved_model.simple_save( + sess, output_dir, inputs=inputs_map, outputs=outputs_map) # the meta_graph_version could not be passed via existing interfaces # so we could only write it by the raw methods saved_model = saved_model_pb2.SavedModel() with GFile(os.path.join(output_dir, 'saved_model.pb'), 'rb') as fin: saved_model.ParseFromString(fin.read()) - saved_model.meta_graphs[0].meta_info_def.meta_graph_version = meta_graph_editor.meta_graph_version + saved_model.meta_graphs[ + 0].meta_info_def.meta_graph_version = meta_graph_editor.meta_graph_version with GFile(os.path.join(output_dir, 'saved_model.pb'), 'wb') as fout: fout.write(saved_model.SerializeToString()) @@ -120,5 +132,6 @@ feature_vals.append(''.join(line_toks)) if len(feature_vals) >= 32: break - out_vals = sess.run(outputs_map, feed_dict={inputs_map['features']: feature_vals}) + out_vals = sess.run( + outputs_map, feed_dict={inputs_map['features']: feature_vals}) logging.info('test_data probs:' + str(out_vals)) diff --git a/easy_rec/python/tools/faiss_index_pai.py b/easy_rec/python/tools/faiss_index_pai.py index 13c43c8ff..e9ebe3f89 100644 --- a/easy_rec/python/tools/faiss_index_pai.py +++ b/easy_rec/python/tools/faiss_index_pai.py @@ -12,7 +12,8 @@ from easy_rec.python.utils import io_util -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') tf.app.flags.DEFINE_string('tables', '', 'tables passed by pai command') tf.app.flags.DEFINE_integer('batch_size', 1024, 'batch size') @@ -28,9 +29,11 @@ def main(argv): - reader = tf.python_io.TableReader(FLAGS.tables, slice_id=0, slice_count=1, capacity=FLAGS.batch_size * 2) + reader = tf.python_io.TableReader( + FLAGS.tables, slice_id=0, slice_count=1, capacity=FLAGS.batch_size * 2) i = 0 - id_map_f = tf.gfile.GFile(os.path.join(FLAGS.index_output_dir, 'id_mapping'), 'w') + id_map_f = tf.gfile.GFile( + os.path.join(FLAGS.index_output_dir, 'id_mapping'), 'w') embeddings = [] while True: try: @@ -40,7 +43,8 @@ def main(argv): eid = record[0].decode('utf-8') id_map_f.write('%s\n' % eid) - embeddings.extend([list(map(float, record[1].split(b','))) for record in records]) + embeddings.extend( + [list(map(float, record[1].split(b','))) for record in records]) i += 1 if i % 100 == 0: logging.info('read %d embeddings.' % (i * FLAGS.batch_size)) @@ -52,9 +56,11 @@ def main(argv): logging.info('Building faiss index..') if FLAGS.index_type == 'IVFFlat': quantizer = faiss.IndexFlatIP(FLAGS.embedding_dim) - index = faiss.IndexIVFFlat(quantizer, FLAGS.embedding_dim, FLAGS.ivf_nlist, faiss.METRIC_INNER_PRODUCT) + index = faiss.IndexIVFFlat(quantizer, FLAGS.embedding_dim, FLAGS.ivf_nlist, + faiss.METRIC_INNER_PRODUCT) elif FLAGS.index_type == 'HNSWFlat': - index = faiss.IndexHNSWFlat(FLAGS.embedding_dim, FLAGS.hnsw_M, faiss.METRIC_INNER_PRODUCT) + index = faiss.IndexHNSWFlat(FLAGS.embedding_dim, FLAGS.hnsw_M, + faiss.METRIC_INNER_PRODUCT) index.hnsw.efConstruction = FLAGS.hnsw_efConstruction else: raise NotImplementedError @@ -68,7 +74,8 @@ def main(argv): index.add(embeddings) faiss.write_index(index, 'faiss_index') - with tf.gfile.GFile(os.path.join(FLAGS.index_output_dir, 'faiss_index'), 'wb') as f_out: + with tf.gfile.GFile( + os.path.join(FLAGS.index_output_dir, 'faiss_index'), 'wb') as f_out: with open('faiss_index', 'rb') as f_in: f_out.write(f_in.read()) @@ -76,12 +83,14 @@ def main(argv): # IVFFlat for ivf_nlist in [100, 500, 1000, 2000]: quantizer = faiss.IndexFlatIP(FLAGS.embedding_dim) - index = faiss.IndexIVFFlat(quantizer, FLAGS.embedding_dim, ivf_nlist, faiss.METRIC_INNER_PRODUCT) + index = faiss.IndexIVFFlat(quantizer, FLAGS.embedding_dim, ivf_nlist, + faiss.METRIC_INNER_PRODUCT) index.train(embeddings) index.add(embeddings) index_name = 'faiss_index_ivfflat_nlist%d' % ivf_nlist faiss.write_index(index, index_name) - with tf.gfile.GFile(os.path.join(FLAGS.index_output_dir, index_name), 'wb') as f_out: + with tf.gfile.GFile( + os.path.join(FLAGS.index_output_dir, index_name), 'wb') as f_out: with open(index_name, 'rb') as f_in: f_out.write(f_in.read()) @@ -90,12 +99,14 @@ def main(argv): for hnsw_efConstruction in [64, 128, 256, 512, 1024, 2048, 4096, 8196]: if hnsw_efConstruction < hnsw_M * 2: continue - index = faiss.IndexHNSWFlat(FLAGS.embedding_dim, hnsw_M, faiss.METRIC_INNER_PRODUCT) + index = faiss.IndexHNSWFlat(FLAGS.embedding_dim, hnsw_M, + faiss.METRIC_INNER_PRODUCT) index.hnsw.efConstruction = hnsw_efConstruction index.add(embeddings) index_name = 'faiss_index_hnsw_M%d_ef%d' % (hnsw_M, hnsw_efConstruction) faiss.write_index(index, index_name) - with tf.gfile.GFile(os.path.join(FLAGS.index_output_dir, index_name), 'wb') as f_out: + with tf.gfile.GFile( + os.path.join(FLAGS.index_output_dir, index_name), 'wb') as f_out: with open(index_name, 'rb') as f_in: f_out.write(f_in.read()) diff --git a/easy_rec/python/tools/feature_selection.py b/easy_rec/python/tools/feature_selection.py index f64a6518c..491d0f6fb 100644 --- a/easy_rec/python/tools/feature_selection.py +++ b/easy_rec/python/tools/feature_selection.py @@ -1,4 +1,5 @@ -from __future__ import division, print_function +from __future__ import division +from __future__ import print_function import json import os @@ -10,7 +11,8 @@ import tensorflow as tf from tensorflow.python.framework.meta_graph import read_meta_graph_file -from easy_rec.python.utils import config_util, io_util +from easy_rec.python.utils import config_util +from easy_rec.python.utils import io_util if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -20,25 +22,32 @@ matplotlib.use('Agg') # NOQA import matplotlib.pyplot as plt # NOQA -tf.app.flags.DEFINE_string('model_type', 'variational_dropout', 'feature selection model type') -tf.app.flags.DEFINE_string('config_path', '', 'feature selection model config path') -tf.app.flags.DEFINE_string('checkpoint_path', None, 'feature selection model checkpoint path') -tf.app.flags.DEFINE_string('output_dir', '', 'feature selection result directory') -tf.app.flags.DEFINE_integer('topk', 100, 'select topk importance features for each feature group') +tf.app.flags.DEFINE_string('model_type', 'variational_dropout', + 'feature selection model type') +tf.app.flags.DEFINE_string('config_path', '', + 'feature selection model config path') +tf.app.flags.DEFINE_string('checkpoint_path', None, + 'feature selection model checkpoint path') +tf.app.flags.DEFINE_string('output_dir', '', + 'feature selection result directory') +tf.app.flags.DEFINE_integer( + 'topk', 100, 'select topk importance features for each feature group') tf.app.flags.DEFINE_string('fg_path', '', 'fg config path') -tf.app.flags.DEFINE_bool('visualize', False, 'visualization feature selection result or not') +tf.app.flags.DEFINE_bool('visualize', False, + 'visualization feature selection result or not') FLAGS = tf.app.flags.FLAGS class VariationalDropoutFS: + def __init__( - self, - config_path, - output_dir, - topk, - checkpoint_path=None, - fg_path=None, - visualize=False, + self, + config_path, + output_dir, + topk, + checkpoint_path=None, + fg_path=None, + visualize=False, ): self._config_path = config_path self._output_dir = output_dir @@ -52,14 +61,15 @@ def __init__( def process(self): tf.logging.info('Loading logit_p of VariationalDropout layer ...') ( - feature_dim_dropout_p_map, - embedding_wise_variational_dropout, + feature_dim_dropout_p_map, + embedding_wise_variational_dropout, ) = self._feature_dim_dropout_ratio() feature_importance_map = {} for group_name, feature_dim_dropout_p in feature_dim_dropout_p_map.items(): tf.logging.info('Calculating %s feature importance ...' % group_name) - feature_importance = self._get_feature_importance(feature_dim_dropout_p, embedding_wise_variational_dropout) + feature_importance = self._get_feature_importance( + feature_dim_dropout_p, embedding_wise_variational_dropout) feature_importance_map[group_name] = feature_importance tf.logging.info('Dump %s feature importance to csv ...' % group_name) @@ -77,7 +87,8 @@ def process(self): def _feature_dim_dropout_ratio(self): """Get dropout ratio of embedding-wise or feature-wise.""" config = config_util.get_configs_from_pipeline_file(self._config_path) - assert config.model_config.HasField('variational_dropout'), 'variational_dropout must be in model_config' + assert config.model_config.HasField( + 'variational_dropout'), 'variational_dropout must be in model_config' embedding_wise_variational_dropout = config.model_config.variational_dropout.embedding_wise_variational_dropout @@ -88,7 +99,8 @@ def _feature_dim_dropout_ratio(self): meta_graph_def = read_meta_graph_file(checkpoint_path + '.meta') features_dimension_map = dict() - for col_def in meta_graph_def.collection_def['variational_dropout'].bytes_list.value: + for col_def in meta_graph_def.collection_def[ + 'variational_dropout'].bytes_list.value: name, features_dimension = json.loads(col_def) name = 'all' if name == '' else name features_dimension_map[name] = OrderedDict(features_dimension) @@ -113,10 +125,12 @@ def _feature_dim_dropout_ratio(self): feature_dim_dropout_p = {} if embedding_wise_variational_dropout: index_end = 0 - for feature_name, feature_dim in features_dimension_map[group_name].items(): + for feature_name, feature_dim in features_dimension_map[ + group_name].items(): index_start = index_end index_end = index_start + feature_dim - feature_dim_dropout_p[feature_name] = feature_dims_importance[index_start:index_end] + feature_dim_dropout_p[feature_name] = feature_dims_importance[ + index_start:index_end] else: index = 0 for feature_name in features_dimension_map[group_name].keys(): @@ -126,16 +140,19 @@ def _feature_dim_dropout_ratio(self): feature_dim_dropout_p_map[group_name] = feature_dim_dropout_p return feature_dim_dropout_p_map, embedding_wise_variational_dropout - def _get_feature_importance(self, feature_dim_dropout_p, embedding_wise_variational_dropout): + def _get_feature_importance(self, feature_dim_dropout_p, + embedding_wise_variational_dropout): """Calculate feature importance.""" if embedding_wise_variational_dropout: feature_importance = {} for item in feature_dim_dropout_p.items(): dropout_rate_mean = np.mean(item[1]) feature_importance[item[0]] = dropout_rate_mean - feature_importance = OrderedDict(sorted(feature_importance.items(), key=lambda e: e[1])) + feature_importance = OrderedDict( + sorted(feature_importance.items(), key=lambda e: e[1])) else: - feature_importance = OrderedDict(sorted(feature_dim_dropout_p.items(), key=lambda e: e[1])) + feature_importance = OrderedDict( + sorted(feature_dim_dropout_p.items(), key=lambda e: e[1])) return feature_importance def _process_config(self, feature_importance_map): @@ -168,8 +185,8 @@ def _process_config(self, feature_importance_map): feature_configs = [] for feature_config in config_util.get_compatible_feature_configs(config): feature_name = ( - feature_config.feature_name if feature_config.HasField('feature_name') else feature_config.input_names[0] - ) + feature_config.feature_name if feature_config.HasField('feature_name') + else feature_config.input_names[0]) if feature_name not in excluded_features: feature_configs.append(feature_config) @@ -187,7 +204,9 @@ def _process_config(self, feature_importance_map): feature_names.append(feature_name) feature_group.ClearField('feature_names') feature_group.feature_names.extend(feature_names) - config_util.save_message(config, os.path.join(self._output_dir, os.path.basename(self._config_path))) + config_util.save_message( + config, + os.path.join(self._output_dir, os.path.basename(self._config_path))) if self._fg_path is not None and len(self._fg_path) > 0: with tf.gfile.Open(self._fg_path) as f: @@ -200,18 +219,21 @@ def _process_config(self, feature_importance_map): else: features.append(feature) fg_json['features'] = features - with tf.gfile.Open(os.path.join(self._output_dir, os.path.basename(self._fg_path)), 'w') as f: + with tf.gfile.Open( + os.path.join(self._output_dir, os.path.basename(self._fg_path)), + 'w') as f: json.dump(fg_json, f, indent=4) def _dump_to_csv(self, feature_importance, group_name): """Dump feature importance data to a csv file.""" with tf.gfile.Open( - os.path.join(self._output_dir, 'feature_dropout_ratio_%s.csv' % group_name), - 'w', + os.path.join(self._output_dir, + 'feature_dropout_ratio_%s.csv' % group_name), + 'w', ) as f: df = pd.DataFrame( - columns=['feature_name', 'mean_drop_p'], - data=[list(kv) for kv in feature_importance.items()], + columns=['feature_name', 'mean_drop_p'], + data=[list(kv) for kv in feature_importance.items()], ) df.to_csv(f, encoding='gbk') @@ -233,21 +255,21 @@ def _visualize_embedding_dim_importance(self, feature_dim_dropout_p): performance_list.append(feature_dropout_p[i]) fig, ax = plt.subplots() b = ax.barh( - y_pos, - performance_list, - align='center', - alpha=0.4, - label='dropout_rate', - lw=1, + y_pos, + performance_list, + align='center', + alpha=0.4, + label='dropout_rate', + lw=1, ) for rect in b: w = rect.get_width() ax.text( - w, - rect.get_y() + rect.get_height() / 2, - '%.4f' % w, - ha='left', - va='center', + w, + rect.get_y() + rect.get_height() / 2, + '%.4f' % w, + ha='left', + va='center', ) plt.yticks(y_pos, embedding_dims) plt.xlabel(feature_name) @@ -259,8 +281,8 @@ def _visualize_embedding_dim_importance(self, feature_dim_dropout_p): def _visualize_feature_importance(self, feature_importance, group_name): """Draw feature importance histogram.""" df = pd.DataFrame( - columns=['feature_name', 'mean_drop_p'], - data=[list(kv) for kv in feature_importance.items()], + columns=['feature_name', 'mean_drop_p'], + data=[list(kv) for kv in feature_importance.items()], ) df['color'] = ['red' if x < 0.5 else 'green' for x in df['mean_drop_p']] df.sort_values('mean_drop_p', inplace=True, ascending=False) @@ -270,12 +292,15 @@ def _visualize_feature_importance(self, feature_importance, group_name): plt.hlines(y=df.index, xmin=0, xmax=df.mean_drop_p) for x, y, tex in zip(df.mean_drop_p, df.index, df.mean_drop_p): plt.text( - x, - y, - round(tex, 2), - horizontalalignment='right' if x < 0 else 'left', - verticalalignment='center', - fontdict={'color': 'red' if x < 0 else 'green', 'size': 14}, + x, + y, + round(tex, 2), + horizontalalignment='right' if x < 0 else 'left', + verticalalignment='center', + fontdict={ + 'color': 'red' if x < 0 else 'green', + 'size': 14 + }, ) # Decorations plt.yticks(df.index, df.feature_name, fontsize=20) @@ -283,8 +308,9 @@ def _visualize_feature_importance(self, feature_importance, group_name): plt.grid(linestyle='--', alpha=0.5) plt.xlim(0, 1) with tf.gfile.GFile( - os.path.join(self._output_dir, 'feature_dropout_pic_%s.png' % group_name), - 'wb', + os.path.join(self._output_dir, + 'feature_dropout_pic_%s.png' % group_name), + 'wb', ) as f: plt.savefig(f, format='png') @@ -293,13 +319,14 @@ def _visualize_feature_importance(self, feature_importance, group_name): sys.argv = io_util.filter_unknown_args(FLAGS, sys.argv) if FLAGS.model_type == 'variational_dropout': fs = VariationalDropoutFS( - FLAGS.config_path, - FLAGS.output_dir, - FLAGS.topk, - checkpoint_path=FLAGS.checkpoint_path, - fg_path=FLAGS.fg_path, - visualize=FLAGS.visualize, + FLAGS.config_path, + FLAGS.output_dir, + FLAGS.topk, + checkpoint_path=FLAGS.checkpoint_path, + fg_path=FLAGS.fg_path, + visualize=FLAGS.visualize, ) fs.process() else: - raise ValueError('Unknown feature selection model type %s' % FLAGS.model_type) + raise ValueError('Unknown feature selection model type %s' % + FLAGS.model_type) diff --git a/easy_rec/python/tools/hit_rate_ds.py b/easy_rec/python/tools/hit_rate_ds.py index 8c8ef1631..24c0b6a73 100644 --- a/easy_rec/python/tools/hit_rate_ds.py +++ b/easy_rec/python/tools/hit_rate_ds.py @@ -13,7 +13,9 @@ # limitations under the License. # ============================================================================= # """Evaluation of Top k hitrate.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import json import logging @@ -24,31 +26,32 @@ import tensorflow as tf from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.utils import config_util, io_util +from easy_rec.python.utils import config_util +from easy_rec.python.utils import io_util from easy_rec.python.utils.config_util import process_multi_file_input_path +from easy_rec.python.utils.hive_utils import HiveUtils + from easy_rec.python.utils.hit_rate_utils import ( # NOQA - compute_hitrate_batch, - load_graph, - reduce_hitrate, + compute_hitrate_batch, load_graph, reduce_hitrate, ) -from easy_rec.python.utils.hive_utils import HiveUtils if tf.__version__ >= '2.0': tf = tf.compat.v1 from easy_rec.python.utils.distribution_utils import ( # NOQA - set_tf_config_and_get_train_worker_num_on_ds, -) + set_tf_config_and_get_train_worker_num_on_ds,) logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) tf.app.flags.DEFINE_string('item_emb_table', '', 'item embedding table name') tf.app.flags.DEFINE_string('gt_table', '', 'ground truth table name') -tf.app.flags.DEFINE_string('hitrate_details_result', '', 'hitrate detail file path') -tf.app.flags.DEFINE_string('total_hitrate_result', '', 'total hitrate result file path') +tf.app.flags.DEFINE_string('hitrate_details_result', '', + 'hitrate detail file path') +tf.app.flags.DEFINE_string('total_hitrate_result', '', + 'total hitrate result file path') tf.app.flags.DEFINE_string('pipeline_config_path', '', 'pipeline config path') tf.app.flags.DEFINE_integer('batch_size', 512, 'batch size') @@ -60,7 +63,8 @@ tf.app.flags.DEFINE_integer('timeout', '60', 'timeout') tf.app.flags.DEFINE_integer('num_interests', 1, 'max number of interests') tf.app.flags.DEFINE_string('gt_table_field_sep', '\t', 'gt_table_field_sep') -tf.app.flags.DEFINE_string('item_emb_table_field_sep', '\t', 'item_emb_table_field_sep') +tf.app.flags.DEFINE_string('item_emb_table_field_sep', '\t', + 'item_emb_table_field_sep') tf.app.flags.DEFINE_bool('is_on_ds', False, help='is on ds') FLAGS = tf.app.flags.FLAGS @@ -85,40 +89,41 @@ def compute_hitrate(g, gt_all, hitrate_writer, gt_table=None): for gt_record in gt_all: gt_record = list(gt_record) ( - hits, - gt_count, - src_ids, - recall_ids, - recall_distances, - hitrates, - bad_cases, - bad_dists, - ) = compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, FLAGS.num_interests, FLAGS.top_k) + hits, + gt_count, + src_ids, + recall_ids, + recall_distances, + hitrates, + bad_cases, + bad_dists, + ) = compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, FLAGS.num_interests, + FLAGS.top_k) total_hits += hits total_gt_count += gt_count src_ids = [str(ids) for ids in src_ids] hitrates = [str(hitrate) for hitrate in hitrates] topk_recalls = [','.join(str(x) for x in ids) for ids in recall_ids] - topk_dists = [','.join('|'.join(str(x) for x in dist) for dist in dists) for dists in recall_distances] + topk_dists = [ + ','.join('|'.join(str(x) + for x in dist) + for dist in dists) + for dists in recall_distances + ] bad_cases = [','.join(str(x) for x in bad_case) for bad_case in bad_cases] bad_dists = [','.join(str(x) for x in dist) for dist in bad_dists] - hitrate_writer.write( - '\n'.join( - [ - '\t'.join(line) - for line in zip( + hitrate_writer.write('\n'.join([ + '\t'.join(line) for line in zip( src_ids, topk_recalls, topk_dists, hitrates, bad_cases, bad_dists, - ) - ] - ) - ) + ) + ])) print('total_hits: ', total_hits) print('total_gt_count: ', total_gt_count) return total_hits, total_gt_count @@ -159,7 +164,8 @@ def main(): i_emb_table = FLAGS.item_emb_table gt_table = FLAGS.gt_table - pipeline_config = config_util.get_configs_from_pipeline_file(FLAGS.pipeline_config_path) + pipeline_config = config_util.get_configs_from_pipeline_file( + FLAGS.pipeline_config_path) logging.info('i_emb_table %s', i_emb_table) input_type = pipeline_config.data_config.input_type @@ -168,22 +174,29 @@ def main(): i_emb_table = process_multi_file_input_path(i_emb_table) else: hive_utils = HiveUtils( - data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input, + data_config=pipeline_config.data_config, + hive_config=pipeline_config.hive_train_input, ) i_emb_table = hive_utils.get_table_location(i_emb_table) - g = load_graph(i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, FLAGS.knn_strict) + g = load_graph(i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, + FLAGS.knn_strict) gl.set_tracker_mode(0) gl.set_field_delimiter(FLAGS.item_emb_table_field_sep) - cluster = tf.train.ClusterSpec({'ps': tf_config['cluster']['ps'], 'worker': tf_config['cluster']['worker']}) + cluster = tf.train.ClusterSpec({ + 'ps': tf_config['cluster']['ps'], + 'worker': tf_config['cluster']['worker'] + }) server = tf.train.Server(cluster, job_name=job_name, task_index=task_index) if job_name == 'ps': server.join() else: - worker_hosts = [str(host.split(':')[0]) + ':888' + str(i) for i, host in enumerate(tf_config['cluster']['worker'])] + worker_hosts = [ + str(host.split(':')[0]) + ':888' + str(i) + for i, host in enumerate(tf_config['cluster']['worker']) + ] worker_hosts = ','.join(worker_hosts) g.init(task_index=task_index, task_count=worker_count, hosts=worker_hosts) # Your model, use g to do some operation, such as sampling @@ -192,20 +205,24 @@ def main(): gt_all = gt_hdfs(gt_table, FLAGS.batch_size, FLAGS.gt_table_field_sep) else: gt_reader = HiveUtils( - data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input, - selected_cols='*', + data_config=pipeline_config.data_config, + hive_config=pipeline_config.hive_train_input, + selected_cols='*', ) gt_all = gt_reader.hive_read_lines(gt_table, FLAGS.batch_size) if not tf.gfile.IsDirectory(hitrate_details_result): tf.gfile.MakeDirs(hitrate_details_result) - hitrate_details_result = os.path.join(hitrate_details_result, 'part-%s' % task_index) + hitrate_details_result = os.path.join(hitrate_details_result, + 'part-%s' % task_index) details_writer = tf.gfile.GFile(hitrate_details_result, 'w') print('Start compute hitrate...') - total_hits, total_gt_count = compute_hitrate(g, gt_all, details_writer, gt_table) - var_total_hitrate, var_worker_count = reduce_hitrate(cluster, total_hits, total_gt_count, task_index) + total_hits, total_gt_count = compute_hitrate(g, gt_all, details_writer, + gt_table) + var_total_hitrate, var_worker_count = reduce_hitrate( + cluster, total_hits, total_gt_count, task_index) - with tf.train.MonitoredTrainingSession(master=server.target, is_chief=(task_index == 0)) as sess: + with tf.train.MonitoredTrainingSession( + master=server.target, is_chief=(task_index == 0)) as sess: outs = sess.run([var_total_hitrate, var_worker_count]) # write after all workers have completed the calculation of hitrate. diff --git a/easy_rec/python/tools/hit_rate_pai.py b/easy_rec/python/tools/hit_rate_pai.py index db016ec56..4db7795f1 100644 --- a/easy_rec/python/tools/hit_rate_pai.py +++ b/easy_rec/python/tools/hit_rate_pai.py @@ -14,17 +14,18 @@ # ============================================================================= """Evaluation of Top k hitrate.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import sys import tensorflow as tf from easy_rec.python.utils import io_util + from easy_rec.python.utils.hit_rate_utils import ( # NOQA - compute_hitrate_batch, - load_graph, - reduce_hitrate, + compute_hitrate_batch, load_graph, reduce_hitrate, ) flags = tf.app.flags @@ -64,34 +65,36 @@ def compute_hitrate(g, gt_reader, hitrate_writer): try: gt_record = gt_reader.read(FLAGS.batch_size) ( - hits, - gt_count, - src_ids, - recall_ids, - recall_distances, - hitrates, - bad_cases, - bad_dists, - ) = compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, FLAGS.num_interests, FLAGS.top_k) + hits, + gt_count, + src_ids, + recall_ids, + recall_distances, + hitrates, + bad_cases, + bad_dists, + ) = compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, + FLAGS.num_interests, FLAGS.top_k) total_hits += hits total_gt_count += gt_count topk_recalls = [','.join(str(x) for x in ids) for ids in recall_ids] - topk_dists = [','.join(str(x) for x in dists) for dists in recall_distances] + topk_dists = [ + ','.join(str(x) for x in dists) for dists in recall_distances + ] bad_cases = [','.join(str(x) for x in case) for case in bad_cases] bad_dists = [','.join(str(x) for x in dist) for dist in bad_dists] hitrate_writer.write( - list( - zip( - src_ids, - topk_recalls, - topk_dists, - hitrates, - bad_cases, - bad_dists, - ) - ), - indices=[0, 1, 2, 3, 4, 5], + list( + zip( + src_ids, + topk_recalls, + topk_dists, + hitrates, + bad_cases, + bad_dists, + )), + indices=[0, 1, 2, 3, 4, 5], ) except tf.python_io.OutOfRangeException: break @@ -104,36 +107,47 @@ def main(): if FLAGS.recall_type == 'u2i': i_emb_table, gt_table = input_tables g = load_graph( - i_emb_table, - FLAGS.emb_dim, - FLAGS.knn_metric, - FLAGS.timeout, - FLAGS.knn_strict, + i_emb_table, + FLAGS.emb_dim, + FLAGS.knn_metric, + FLAGS.timeout, + FLAGS.knn_strict, ) else: i_emb_table, gt_table = input_tables[-2], input_tables[-1] g = load_graph( - i_emb_table, - FLAGS.emb_dim, - FLAGS.knn_metric, - FLAGS.timeout, - FLAGS.knn_strict, + i_emb_table, + FLAGS.emb_dim, + FLAGS.knn_metric, + FLAGS.timeout, + FLAGS.knn_strict, ) hitrate_details_table, total_hitrate_table = FLAGS.outputs.split(',') - cluster = tf.train.ClusterSpec({'ps': FLAGS.ps_hosts.split(','), 'worker': FLAGS.worker_hosts.split(',')}) - server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index) + cluster = tf.train.ClusterSpec({ + 'ps': FLAGS.ps_hosts.split(','), + 'worker': FLAGS.worker_hosts.split(',') + }) + server = tf.train.Server( + cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index) if FLAGS.job_name == 'ps': server.join() else: g.init(task_index=FLAGS.task_index, task_count=worker_count) - gt_reader = tf.python_io.TableReader(gt_table, slice_id=FLAGS.task_index, slice_count=worker_count, capacity=2048) - details_writer = tf.python_io.TableWriter(hitrate_details_table, slice_id=FLAGS.task_index) + gt_reader = tf.python_io.TableReader( + gt_table, + slice_id=FLAGS.task_index, + slice_count=worker_count, + capacity=2048) + details_writer = tf.python_io.TableWriter( + hitrate_details_table, slice_id=FLAGS.task_index) print('Start compute hitrate...') total_hits, total_gt_count = compute_hitrate(g, gt_reader, details_writer) - var_total_hitrate, var_worker_count = reduce_hitrate(cluster, total_hits, total_gt_count, FLAGS.task_index) + var_total_hitrate, var_worker_count = reduce_hitrate( + cluster, total_hits, total_gt_count, FLAGS.task_index) - with tf.train.MonitoredTrainingSession(master=server.target, is_chief=(FLAGS.task_index == 0)) as sess: + with tf.train.MonitoredTrainingSession( + master=server.target, is_chief=(FLAGS.task_index == 0)) as sess: outs = sess.run([var_total_hitrate, var_worker_count]) # write after all workers have completed the calculation of hitrate. diff --git a/easy_rec/python/tools/pre_check.py b/easy_rec/python/tools/pre_check.py index 12a009b6c..34be0332d 100644 --- a/easy_rec/python/tools/pre_check.py +++ b/easy_rec/python/tools/pre_check.py @@ -8,26 +8,34 @@ import tensorflow as tf from easy_rec.python.input.input import Input -from easy_rec.python.utils import config_util, fg_util, io_util +from easy_rec.python.utils import config_util +from easy_rec.python.utils import fg_util +from easy_rec.python.utils import io_util + from easy_rec.python.utils.check_utils import ( # NOQA - check_env_and_input_path, - check_sequence, + check_env_and_input_path, check_sequence, ) if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) -tf.app.flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') -tf.app.flags.DEFINE_multi_string('data_input_path', None, help='data input path') +tf.app.flags.DEFINE_string('pipeline_config_path', None, + 'Path to pipeline config ' + 'file.') +tf.app.flags.DEFINE_multi_string( + 'data_input_path', None, help='data input path') FLAGS = tf.app.flags.FLAGS -def _get_input_fn(data_config, feature_configs, data_path=None, export_config=None): +def _get_input_fn(data_config, + feature_configs, + data_path=None, + export_config=None): """Build estimator input function. Args: @@ -53,19 +61,20 @@ def _get_input_fn(data_config, feature_configs, data_path=None, export_config=No task_index = 0 input_obj = input_class( - data_config, - feature_configs, - data_path, - task_index=task_index, - task_num=worker_num, - check_mode=True, + data_config, + feature_configs, + data_path, + task_index=task_index, + task_num=worker_num, + check_mode=True, ) input_fn = input_obj.create_input(export_config) return input_fn def loda_pipeline_config(pipeline_config_path): - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path, False) + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path, False) if pipeline_config.fg_json_path: fg_util.load_fg_json_to_config(pipeline_config) config_util.auto_expand_share_feature_configs(pipeline_config) @@ -76,9 +85,16 @@ def run_check(pipeline_config, input_path): logging.info('data_input_path: %s' % input_path) check_env_and_input_path(pipeline_config, input_path) feature_configs = config_util.get_compatible_feature_configs(pipeline_config) - eval_input_fn = _get_input_fn(pipeline_config.data_config, feature_configs, input_path) - eval_spec = tf.estimator.EvalSpec(name='val', input_fn=eval_input_fn, steps=None, throttle_secs=10, exporters=[]) - input_iter = eval_spec.input_fn(mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() + eval_input_fn = _get_input_fn(pipeline_config.data_config, feature_configs, + input_path) + eval_spec = tf.estimator.EvalSpec( + name='val', + input_fn=eval_input_fn, + steps=None, + throttle_secs=10, + exporters=[]) + input_iter = eval_spec.input_fn( + mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() with tf.Session() as sess: try: while True: @@ -96,9 +112,8 @@ def main(argv): if FLAGS.data_input_path: input_path = ','.join(FLAGS.data_input_path) else: - assert ( - pipeline_config.train_input_path or pipeline_config.eval_input_path - ), 'input_path should not be empty when checking!' + assert (pipeline_config.train_input_path or pipeline_config.eval_input_path + ), 'input_path should not be empty when checking!' input_path = pipeline_config.train_input_path + ',' + pipeline_config.eval_input_path run_check(pipeline_config, input_path) diff --git a/easy_rec/python/tools/predict_and_chk.py b/easy_rec/python/tools/predict_and_chk.py index fe605d17a..bc7353f76 100644 --- a/easy_rec/python/tools/predict_and_chk.py +++ b/easy_rec/python/tools/predict_and_chk.py @@ -13,40 +13,45 @@ try: import tensorflow as tf - tf.load_op_library(os.path.join(easy_rec.ops_dir, 'libembed_op.so')) except Exception as ex: logging.warning('exception: %s' % str(ex)) -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--saved_model_dir', type=str, default=None, help='saved model directory') - parser.add_argument('--input_path', type=str, default=None, help='input feature path') + parser.add_argument( + '--saved_model_dir', type=str, default=None, help='saved model directory') + parser.add_argument( + '--input_path', type=str, default=None, help='input feature path') parser.add_argument('--save_path', type=str, default=None, help='save path') - parser.add_argument('--cmp_res_path', type=str, default=None, help='compare result path') - parser.add_argument('--cmp_key', type=str, default='probs', help='compare key') parser.add_argument( - '--rtp_fea_id', - type=int, - default=-1, - help='rtp feature column index, default to the last column', - ) + '--cmp_res_path', type=str, default=None, help='compare result path') + parser.add_argument( + '--cmp_key', type=str, default='probs', help='compare key') + parser.add_argument( + '--rtp_fea_id', + type=int, + default=-1, + help='rtp feature column index, default to the last column') parser.add_argument('--tol', type=float, default=1e-5, help='tolerance') parser.add_argument( - '--label_id', - nargs='*', - type=int, - help='the label column, which is to be excluded', - ) + '--label_id', + nargs='*', + type=int, + help='the label column, which is to be excluded') + parser.add_argument( + '--separator', + type=str, + default='', + help='separator between features, default to \\u0002') parser.add_argument( - '--separator', - type=str, - default='', - help='separator between features, default to \\u0002', - ) - parser.add_argument('--rtp_separator', type=str, default='', help='separator, default to \\u0001') + '--rtp_separator', + type=str, + default='', + help='separator, default to \\u0001') args = parser.parse_args() if not args.saved_model_dir: @@ -66,7 +71,9 @@ predictor = Predictor(args.saved_model_dir) if len(predictor.input_names) == 1: - assert len(args.label_id) == 0, 'label_id should not be set if rtp feature format is used.' + assert len( + args.label_id + ) == 0, 'label_id should not be set if rtp feature format is used.' with open(args.input_path, 'r') as fin: batch_input = [] @@ -74,7 +81,10 @@ line_str = line_str.strip() line_tok = line_str.split(args.rtp_separator) feature = line_tok[args.rtp_fea_id] - feature = [x for fid, x in enumerate(feature.split(args.separator)) if fid not in args.label_id] + feature = [ + x for fid, x in enumerate(feature.split(args.separator)) + if fid not in args.label_id + ] if 'features' in predictor.input_names: feature = args.separator.join(feature) batch_input.append(feature) @@ -94,7 +104,8 @@ for line_id, line_str in enumerate(fin): line_str = line_str.strip() line_pred = json.loads(line_str) - assert np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( - line_id, - np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]), - ) + assert np.abs( + line_pred[args.cmp_key] - + output[line_id][args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( + line_id, + np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key])) diff --git a/easy_rec/python/tools/read_kafka.py b/easy_rec/python/tools/read_kafka.py index 79af83c40..60c1d5d65 100644 --- a/easy_rec/python/tools/read_kafka.py +++ b/easy_rec/python/tools/read_kafka.py @@ -8,7 +8,8 @@ from kafka import KafkaConsumer from kafka.structs import TopicPartition -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -26,9 +27,9 @@ servers = args.servers.split(',') consumer = KafkaConsumer( - group_id=args.group, - bootstrap_servers=servers, - consumer_timeout_ms=args.timeout * 1000, + group_id=args.group, + bootstrap_servers=servers, + consumer_timeout_ms=args.timeout * 1000, ) if args.partitions is not None: @@ -37,15 +38,17 @@ partitions = consumer.partitions_for_topic(args.topic) logging.info('partitions: %s' % partitions) - topics = [TopicPartition(topic=args.topic, partition=part_id) for part_id in partitions] + topics = [ + TopicPartition(topic=args.topic, partition=part_id) + for part_id in partitions + ] consumer.assign(topics) consumer.seek_to_beginning() record_id = 0 for x in consumer: - logging.info( - '%d: key=%s\toffset=%d\ttimestamp=%d\tlen=%d' % (record_id, x.key, x.offset, x.timestamp, len(x.value)) - ) + logging.info('%d: key=%s\toffset=%d\ttimestamp=%d\tlen=%d' % + (record_id, x.key, x.offset, x.timestamp, len(x.value))) if args.save_dir is not None: save_path = os.path.join(args.save_dir, x.key) with open(save_path, 'wb') as fout: diff --git a/easy_rec/python/tools/split_model_pai.py b/easy_rec/python/tools/split_model_pai.py index b7e005b15..c763ce005 100644 --- a/easy_rec/python/tools/split_model_pai.py +++ b/easy_rec/python/tools/split_model_pai.py @@ -6,7 +6,8 @@ import tensorflow as tf from tensorflow.core.framework import graph_pb2 -from tensorflow.python.framework import importer, ops +from tensorflow.python.framework import importer +from tensorflow.python.framework import ops from tensorflow.python.framework.dtypes import _TYPE_TO_STRING from tensorflow.python.ops.resource_variable_ops import _from_proto_fn from tensorflow.python.saved_model import signature_constants @@ -28,7 +29,8 @@ tf.app.flags.DEFINE_string('user_fg_json_path', '', '') tf.app.flags.DEFINE_string('item_fg_json_path', '', '') -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') def search_pb(directory): @@ -128,7 +130,8 @@ def load_meta_graph_def(model_dir): output_tensor_names = {} variable_protos = {} - meta_graph_def = saved_model_utils.get_meta_graph_def(model_dir, tf.saved_model.tag_constants.SERVING) + meta_graph_def = saved_model_utils.get_meta_graph_def( + model_dir, tf.saved_model.tag_constants.SERVING) signatures = meta_graph_def.signature_def collections = meta_graph_def.collection_def @@ -147,21 +150,19 @@ def load_meta_graph_def(model_dir): # parse signature info for SavedModel for sig_name in signatures: - if signatures[sig_name].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: + if signatures[ + sig_name].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: tf.logging.info('[Signature] inputs:') for input_name in signatures[sig_name].inputs: input_tensor_shape = [] input_tensor = signatures[sig_name].inputs[input_name] for dim in input_tensor.tensor_shape.dim: input_tensor_shape.append(int(dim.size)) - tf.logging.info( - '"%s": %s; %s' - % ( + tf.logging.info('"%s": %s; %s' % ( input_name, _TYPE_TO_STRING[input_tensor.dtype], input_tensor_shape, - ) - ) + )) input_tensor_names[input_name] = input_tensor.name tf.logging.info('[Signature] outputs:') for output_name in signatures[sig_name].outputs: @@ -169,27 +170,24 @@ def load_meta_graph_def(model_dir): output_tensor = signatures[sig_name].outputs[output_name] for dim in output_tensor.tensor_shape.dim: output_tensor_shape.append(int(dim.size)) - tf.logging.info( - '"%s": %s; %s' - % ( + tf.logging.info('"%s": %s; %s' % ( output_name, _TYPE_TO_STRING[output_tensor.dtype], output_tensor_shape, - ) - ) + )) output_tensor_names[output_name] = output_tensor.name return meta_graph_def, variable_protos, input_tensor_names, output_tensor_names def export( - model_dir, - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, - part_name, - part_dir, + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name, + part_dir, ): """Export subpart saved model. @@ -202,10 +200,17 @@ def export( part_name: subpart model name, user or item. part_dir: subpart model export directory. """ - output_tensor_names = {x: output_tensor_names[x] for x in output_tensor_names.keys() if part_name in x} - output_node_names = [_node_name(output_tensor_names[x]) for x in output_tensor_names.keys()] - - inference_graph, variables_to_keep = extract_sub_graph(meta_graph_def.graph_def, output_node_names, variable_protos) + output_tensor_names = { + x: output_tensor_names[x] + for x in output_tensor_names.keys() + if part_name in x + } + output_node_names = [ + _node_name(output_tensor_names[x]) for x in output_tensor_names.keys() + ] + + inference_graph, variables_to_keep = extract_sub_graph( + meta_graph_def.graph_def, output_node_names, variable_protos) tf.reset_default_graph() with tf.Session() as sess: @@ -222,28 +227,31 @@ def export( signature_inputs = {} for input_name in input_tensor_names: try: - tensor_info = tf.saved_model.utils.build_tensor_info(graph.get_tensor_by_name(input_tensor_names[input_name])) + tensor_info = tf.saved_model.utils.build_tensor_info( + graph.get_tensor_by_name(input_tensor_names[input_name])) signature_inputs[input_name] = tensor_info except Exception: print('ignore input: %s' % input_name) signature_outputs = {} for output_name in output_tensor_names: - tensor_info = tf.saved_model.utils.build_tensor_info(graph.get_tensor_by_name(output_tensor_names[output_name])) + tensor_info = tf.saved_model.utils.build_tensor_info( + graph.get_tensor_by_name(output_tensor_names[output_name])) signature_outputs[output_name] = tensor_info prediction_signature = tf.saved_model.signature_def_utils.build_signature_def( - inputs=signature_inputs, - outputs=signature_outputs, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, + inputs=signature_inputs, + outputs=signature_outputs, + method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, ) builder.add_meta_graph_and_variables( - sess, - [tf.saved_model.tag_constants.SERVING], - signature_def_map={ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_signature, - }, + sess, + [tf.saved_model.tag_constants.SERVING], + signature_def_map={ + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + prediction_signature, + }, ) builder.save() config_path = os.path.join(model_dir, 'assets/pipeline.config') @@ -264,30 +272,30 @@ def main(argv): model_dir = search_pb(FLAGS.model_dir) tf.logging.info('Loading meta graph...') ( - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, ) = load_meta_graph_def(model_dir) tf.logging.info('Exporting user part model...') export( - model_dir, - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, - part_name='user', - part_dir=FLAGS.user_model_dir, + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name='user', + part_dir=FLAGS.user_model_dir, ) tf.logging.info('Exporting item part model...') export( - model_dir, - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, - part_name='item', - part_dir=FLAGS.item_model_dir, + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name='item', + part_dir=FLAGS.item_model_dir, ) diff --git a/easy_rec/python/tools/split_pdn_model_pai.py b/easy_rec/python/tools/split_pdn_model_pai.py index 287d7eace..5cd414a75 100644 --- a/easy_rec/python/tools/split_pdn_model_pai.py +++ b/easy_rec/python/tools/split_pdn_model_pai.py @@ -6,7 +6,8 @@ import tensorflow as tf from tensorflow.core.framework import graph_pb2 -from tensorflow.python.framework import importer, ops +from tensorflow.python.framework import importer +from tensorflow.python.framework import ops from tensorflow.python.framework.dtypes import _TYPE_TO_STRING from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model.utils_impl import get_variables_path @@ -20,7 +21,8 @@ tf.app.flags.DEFINE_string('trigger_model_dir', '', '') tf.app.flags.DEFINE_string('sim_model_dir', '', '') -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') def search_pb(directory): @@ -120,7 +122,8 @@ def load_meta_graph_def(model_dir): output_tensor_names = {} variable_protos = {} - meta_graph_def = saved_model_utils.get_meta_graph_def(model_dir, tf.saved_model.tag_constants.SERVING) + meta_graph_def = saved_model_utils.get_meta_graph_def( + model_dir, tf.saved_model.tag_constants.SERVING) signatures = meta_graph_def.signature_def collections = meta_graph_def.collection_def @@ -139,21 +142,19 @@ def load_meta_graph_def(model_dir): # parse signature info for SavedModel for sig_name in signatures: - if signatures[sig_name].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: + if signatures[ + sig_name].method_name == tf.saved_model.signature_constants.PREDICT_METHOD_NAME: tf.logging.info('[Signature] inputs:') for input_name in signatures[sig_name].inputs: input_tensor_shape = [] input_tensor = signatures[sig_name].inputs[input_name] for dim in input_tensor.tensor_shape.dim: input_tensor_shape.append(int(dim.size)) - tf.logging.info( - '"%s": %s; %s' - % ( + tf.logging.info('"%s": %s; %s' % ( input_name, _TYPE_TO_STRING[input_tensor.dtype], input_tensor_shape, - ) - ) + )) input_tensor_names[input_name] = input_tensor.name tf.logging.info('[Signature] outputs:') for output_name in signatures[sig_name].outputs: @@ -161,27 +162,24 @@ def load_meta_graph_def(model_dir): output_tensor = signatures[sig_name].outputs[output_name] for dim in output_tensor.tensor_shape.dim: output_tensor_shape.append(int(dim.size)) - tf.logging.info( - '"%s": %s; %s' - % ( + tf.logging.info('"%s": %s; %s' % ( output_name, _TYPE_TO_STRING[output_tensor.dtype], output_tensor_shape, - ) - ) + )) output_tensor_names[output_name] = output_tensor.name return meta_graph_def, variable_protos, input_tensor_names, output_tensor_names def export( - model_dir, - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, - part_name, - part_dir, + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name, + part_dir, ): """Export subpart saved model. @@ -194,10 +192,17 @@ def export( part_name: subpart model name, user or item. part_dir: subpart model export directory. """ - output_tensor_names = {x: output_tensor_names[x] for x in output_tensor_names.keys() if part_name in x} - output_node_names = [_node_name(output_tensor_names[x]) for x in output_tensor_names.keys()] - - inference_graph, variables_to_keep = extract_sub_graph(meta_graph_def.graph_def, output_node_names, variable_protos) + output_tensor_names = { + x: output_tensor_names[x] + for x in output_tensor_names.keys() + if part_name in x + } + output_node_names = [ + _node_name(output_tensor_names[x]) for x in output_tensor_names.keys() + ] + + inference_graph, variables_to_keep = extract_sub_graph( + meta_graph_def.graph_def, output_node_names, variable_protos) tf.reset_default_graph() with tf.Session() as sess: @@ -214,28 +219,31 @@ def export( signature_inputs = {} for input_name in input_tensor_names: try: - tensor_info = tf.saved_model.utils.build_tensor_info(graph.get_tensor_by_name(input_tensor_names[input_name])) + tensor_info = tf.saved_model.utils.build_tensor_info( + graph.get_tensor_by_name(input_tensor_names[input_name])) signature_inputs[input_name] = tensor_info except Exception: print('ignore input: %s' % input_name) signature_outputs = {} for output_name in output_tensor_names: - tensor_info = tf.saved_model.utils.build_tensor_info(graph.get_tensor_by_name(output_tensor_names[output_name])) + tensor_info = tf.saved_model.utils.build_tensor_info( + graph.get_tensor_by_name(output_tensor_names[output_name])) signature_outputs[output_name] = tensor_info prediction_signature = tf.saved_model.signature_def_utils.build_signature_def( - inputs=signature_inputs, - outputs=signature_outputs, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, + inputs=signature_inputs, + outputs=signature_outputs, + method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, ) builder.add_meta_graph_and_variables( - sess, - [tf.saved_model.tag_constants.SERVING], - signature_def_map={ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_signature, - }, + sess, + [tf.saved_model.tag_constants.SERVING], + signature_def_map={ + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + prediction_signature, + }, ) builder.save() config_path = os.path.join(model_dir, 'assets/pipeline.config') @@ -250,30 +258,30 @@ def main(argv): model_dir = search_pb(FLAGS.model_dir) tf.logging.info('Loading meta graph...') ( - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, ) = load_meta_graph_def(model_dir) tf.logging.info('Exporting trigger part model...') export( - model_dir, - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, - part_name='trigger_out', - part_dir=FLAGS.trigger_model_dir, + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name='trigger_out', + part_dir=FLAGS.trigger_model_dir, ) tf.logging.info('Exporting sim part model...') export( - model_dir, - meta_graph_def, - variable_protos, - input_tensor_names, - output_tensor_names, - part_name='sim_out', - part_dir=FLAGS.sim_model_dir, + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name='sim_out', + part_dir=FLAGS.sim_model_dir, ) diff --git a/easy_rec/python/tools/test_saved_model.py b/easy_rec/python/tools/test_saved_model.py index 9dbfd7a4b..35889a668 100644 --- a/easy_rec/python/tools/test_saved_model.py +++ b/easy_rec/python/tools/test_saved_model.py @@ -12,9 +12,8 @@ from easy_rec.python.inference.predictor import Predictor logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, -) + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO) lookup_op_path = os.path.join(easy_rec.ops_dir, 'libkv_lookup.so') lookup_op = tf.load_op_library(lookup_op_path) @@ -29,19 +28,21 @@ """ parser = argparse.ArgumentParser() - parser.add_argument('--saved_model_dir', type=str, default=None, help='saved model dir') + parser.add_argument( + '--saved_model_dir', type=str, default=None, help='saved model dir') parser.add_argument('--input_path', type=str, default=None, help='output dir') parser.add_argument('--save_path', type=str, default=None, help='save path') parser.add_argument('--separator', type=str, default=',', help='separator') - parser.add_argument('--cmp_res_path', type=str, default=None, help='compare result path') - parser.add_argument('--cmp_key', type=str, default='probs', help='compare key') + parser.add_argument( + '--cmp_res_path', type=str, default=None, help='compare result path') + parser.add_argument( + '--cmp_key', type=str, default='probs', help='compare key') parser.add_argument('--tol', type=float, default=1e-5, help='tolerance') parser.add_argument( - '--with_lbl', - action='store_true', - default=False, - help='whether the test data has label field', - ) + '--with_lbl', + action='store_true', + default=False, + help='whether the test data has label field') args = parser.parse_args() logging.info('saved_model_dir: %s' % args.saved_model_dir) @@ -72,7 +73,8 @@ for line_id, line_str in enumerate(fin): line_str = line_str.strip() line_pred = json.loads(line_str) - assert np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( - line_id, - np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key]), - ) + assert np.abs( + line_pred[args.cmp_key] - + output[line_id][args.cmp_key]) < args.tol, 'line[%d]: %.8f' % ( + line_id, + np.abs(line_pred[args.cmp_key] - output[line_id][args.cmp_key])) diff --git a/easy_rec/python/tools/view_saved_model.py b/easy_rec/python/tools/view_saved_model.py index 222ea696b..7b11589f2 100644 --- a/easy_rec/python/tools/view_saved_model.py +++ b/easy_rec/python/tools/view_saved_model.py @@ -8,14 +8,16 @@ from tensorflow.python.platform.gfile import GFile logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--input', type=str, default=None, help='saved model path') - parser.add_argument('--output', type=str, default=None, help='saved model save path') + parser.add_argument( + '--input', type=str, default=None, help='saved model path') + parser.add_argument( + '--output', type=str, default=None, help='saved model save path') args = parser.parse_args() assert args.input is not None and args.output is not None diff --git a/easy_rec/python/tools/write_kafka.py b/easy_rec/python/tools/write_kafka.py index 71f6d2931..37902de02 100644 --- a/easy_rec/python/tools/write_kafka.py +++ b/easy_rec/python/tools/write_kafka.py @@ -5,12 +5,14 @@ import sys # from kafka import KafkaConsumer -from kafka import KafkaAdminClient, KafkaProducer +from kafka import KafkaAdminClient +from kafka import KafkaProducer from kafka.admin import NewTopic # from kafka.structs import TopicPartition -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -36,23 +38,23 @@ admin_clt = KafkaAdminClient(bootstrap_servers=servers) if args.topic not in admin_clt.list_topics(): admin_clt.create_topics( - new_topics=[ - NewTopic( - name=args.topic, - num_partitions=1, - replication_factor=1, - topic_configs={'max.message.bytes': 1024 * 1024 * 1024}, - ) - ], - validate_only=False, + new_topics=[ + NewTopic( + name=args.topic, + num_partitions=1, + replication_factor=1, + topic_configs={'max.message.bytes': 1024 * 1024 * 1024}, + ) + ], + validate_only=False, ) logging.info('create increment save topic: %s' % args.topic) admin_clt.close() producer = KafkaProducer( - bootstrap_servers=servers, - request_timeout_ms=args.timeout * 1000, - api_version=(0, 10, 1), + bootstrap_servers=servers, + request_timeout_ms=args.timeout * 1000, + api_version=(0, 10, 1), ) i = 1 diff --git a/easy_rec/python/train_eval.py b/easy_rec/python/train_eval.py index ab116de5c..84c5a08b9 100644 --- a/easy_rec/python/train_eval.py +++ b/easy_rec/python/train_eval.py @@ -9,17 +9,12 @@ from easy_rec.python.main import _train_and_evaluate_impl from easy_rec.python.protos.train_pb2 import DistributionStrategy + from easy_rec.python.utils import ( # NOQA - config_util, - ds_util, - estimator_utils, - fg_util, - hpo_util, + config_util, ds_util, estimator_utils, fg_util, hpo_util, ) from easy_rec.python.utils.config_util import ( # NOQA - process_neg_sampler_data_path, - set_eval_input_path, - set_train_input_path, + process_neg_sampler_data_path, set_eval_input_path, set_train_input_path, ) if tf.__version__.startswith('1.'): @@ -28,88 +23,102 @@ import tensorflow.io.gfile as gfile from easy_rec.python.utils.distribution_utils import ( # NOQA - set_tf_config_and_get_train_worker_num_on_ds, -) + set_tf_config_and_get_train_worker_num_on_ds,) if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', + level=logging.INFO, ) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--pipeline_config_path', - type=str, - default=None, - help='Path to pipeline config file.', + '--pipeline_config_path', + type=str, + default=None, + help='Path to pipeline config file.', ) parser.add_argument( - '--continue_train', - action='store_true', - default=False, - help='continue train using existing model_dir', + '--continue_train', + action='store_true', + default=False, + help='continue train using existing model_dir', ) - parser.add_argument('--hpo_param_path', type=str, default=None, help='hyperparam tuning param path') parser.add_argument( - '--hpo_metric_save_path', - type=str, - default=None, - help='hyperparameter save metric path', + '--hpo_param_path', + type=str, + default=None, + help='hyperparam tuning param path') + parser.add_argument( + '--hpo_metric_save_path', + type=str, + default=None, + help='hyperparameter save metric path', ) parser.add_argument( - '--model_dir', - type=str, - default=None, - help='will update the model_dir in pipeline_config', + '--model_dir', + type=str, + default=None, + help='will update the model_dir in pipeline_config', ) parser.add_argument( - '--train_input_path', - type=str, - nargs='*', - default=None, - help='train data input path', + '--train_input_path', + type=str, + nargs='*', + default=None, + help='train data input path', ) parser.add_argument( - '--eval_input_path', - type=str, - nargs='*', - default=None, - help='eval data input path', + '--eval_input_path', + type=str, + nargs='*', + default=None, + help='eval data input path', ) parser.add_argument( - '--fit_on_eval', - action='store_true', - default=False, - help='Fit evaluation data after fitting and evaluating train data', + '--fit_on_eval', + action='store_true', + default=False, + help='Fit evaluation data after fitting and evaluating train data', ) - parser.add_argument('--fit_on_eval_steps', type=int, default=None, help='Fit evaluation data steps') parser.add_argument( - '--fine_tune_checkpoint', - type=str, - default=None, - help='will update the train_config.fine_tune_checkpoint in pipeline_config', + '--fit_on_eval_steps', + type=int, + default=None, + help='Fit evaluation data steps') + parser.add_argument( + '--fine_tune_checkpoint', + type=str, + default=None, + help='will update the train_config.fine_tune_checkpoint in pipeline_config', ) parser.add_argument( - '--edit_config_json', - type=str, - default=None, - help='edit pipeline config str, example: {"model_dir":"experiments/",' - '"feature_config.feature[0].boundaries":[4,5,6,7]}', + '--edit_config_json', + type=str, + default=None, + help='edit pipeline config str, example: {"model_dir":"experiments/",' + '"feature_config.feature[0].boundaries":[4,5,6,7]}', ) parser.add_argument( - '--ignore_finetune_ckpt_error', - action='store_true', - default=False, - help='During incremental training, ignore the problem of missing fine_tune_checkpoint files', + '--ignore_finetune_ckpt_error', + action='store_true', + default=False, + help='During incremental training, ignore the problem of missing fine_tune_checkpoint files', ) - parser.add_argument('--odps_config', type=str, default=None, help='odps config path') - parser.add_argument('--is_on_ds', action='store_true', default=False, help='is on ds') - parser.add_argument('--check_mode', action='store_true', default=False, help='is use check mode') - parser.add_argument('--selected_cols', type=str, default=None, help='select input columns') + parser.add_argument( + '--odps_config', type=str, default=None, help='odps config path') + parser.add_argument( + '--is_on_ds', action='store_true', default=False, help='is on ds') + parser.add_argument( + '--check_mode', + action='store_true', + default=False, + help='is use check mode') + parser.add_argument( + '--selected_cols', type=str, default=None, help='select input columns') parser.add_argument('--gpu', type=str, default=None, help='gpu id') args, extra_args = parser.parse_known_args() @@ -124,7 +133,8 @@ config_util.parse_extra_config_param(extra_args, edit_config_json) if args.pipeline_config_path is not None: - pipeline_config = config_util.get_configs_from_pipeline_file(args.pipeline_config_path, False) + pipeline_config = config_util.get_configs_from_pipeline_file( + args.pipeline_config_path, False) if args.selected_cols: pipeline_config.data_config.selected_cols = args.selected_cols if args.model_dir: @@ -137,8 +147,7 @@ if args.fine_tune_checkpoint: ckpt_path = estimator_utils.get_latest_checkpoint_from_checkpoint_path( - args.fine_tune_checkpoint, args.ignore_finetune_ckpt_error - ) + args.fine_tune_checkpoint, args.ignore_finetune_ckpt_error) if ckpt_path: pipeline_config.train_config.fine_tune_checkpoint = ckpt_path @@ -150,11 +159,11 @@ os.environ['ODPS_CONFIG_FILE_PATH'] = args.odps_config if len(edit_config_json) > 0: - fine_tune_checkpoint = edit_config_json.get('train_config', {}).get('fine_tune_checkpoint', None) + fine_tune_checkpoint = edit_config_json.get('train_config', {}).get( + 'fine_tune_checkpoint', None) if fine_tune_checkpoint: ckpt_path = estimator_utils.get_latest_checkpoint_from_checkpoint_path( - args.fine_tune_checkpoint, args.ignore_finetune_ckpt_error - ) + args.fine_tune_checkpoint, args.ignore_finetune_ckpt_error) edit_config_json['train_config']['fine_tune_checkpoint'] = ckpt_path config_util.edit_config(pipeline_config, edit_config_json) @@ -167,12 +176,12 @@ ds_util.cache_ckpt(pipeline_config) if pipeline_config.train_config.train_distribute in [ - DistributionStrategy.HorovodStrategy, + DistributionStrategy.HorovodStrategy, ]: estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ - DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy, + DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy, ]: estimator_utils.init_hvd() estimator_utils.init_sok() @@ -183,20 +192,21 @@ hpo_params = hpo_config['param'] config_util.edit_config(pipeline_config, hpo_params) config_util.auto_expand_share_feature_configs(pipeline_config) - _train_and_evaluate_impl(pipeline_config, args.continue_train, args.check_mode) + _train_and_evaluate_impl(pipeline_config, args.continue_train, + args.check_mode) hpo_util.save_eval_metrics( - pipeline_config.model_dir, - metric_save_path=args.hpo_metric_save_path, - has_evaluator=False, + pipeline_config.model_dir, + metric_save_path=args.hpo_metric_save_path, + has_evaluator=False, ) else: config_util.auto_expand_share_feature_configs(pipeline_config) _train_and_evaluate_impl( - pipeline_config, - args.continue_train, - args.check_mode, - fit_on_eval=args.fit_on_eval, - fit_on_eval_steps=args.fit_on_eval_steps, + pipeline_config, + args.continue_train, + args.check_mode, + fit_on_eval=args.fit_on_eval, + fit_on_eval_steps=args.fit_on_eval_steps, ) else: raise ValueError('pipeline_config_path should not be empty when training!') diff --git a/easy_rec/python/utils/activation.py b/easy_rec/python/utils/activation.py index 0abdd32dc..9e3b14516 100644 --- a/easy_rec/python/utils/activation.py +++ b/easy_rec/python/utils/activation.py @@ -27,18 +27,18 @@ def dice(_x, axis=-1, epsilon=1e-9, name='dice', training=True): ACM, 2018: 1059-1068.] (https://arxiv.org/pdf/1706.06978.pdf) """ alphas = tf.get_variable( - 'alpha_' + name, - _x.get_shape()[-1], - initializer=tf.constant_initializer(0.0), - dtype=tf.float32, + 'alpha_' + name, + _x.get_shape()[-1], + initializer=tf.constant_initializer(0.0), + dtype=tf.float32, ) inputs_normed = tf.layers.batch_normalization( - inputs=_x, - axis=axis, - epsilon=epsilon, - center=False, - scale=False, - training=training, + inputs=_x, + axis=axis, + epsilon=epsilon, + center=False, + scale=False, + training=training, ) x_p = tf.sigmoid(inputs_normed) return alphas * (1.0 - x_p) * _x + x_p * _x @@ -58,7 +58,8 @@ def gelu(x, name='gelu'): `x` with the GELU activation applied. """ with tf.name_scope(name): - cdf = 0.5 * (1.0 + tf.tanh((np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + cdf = 0.5 * (1.0 + tf.tanh( + (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) return x * cdf diff --git a/easy_rec/python/utils/check_utils.py b/easy_rec/python/utils/check_utils.py index c2901054c..eea7aeee6 100644 --- a/easy_rec/python/utils/check_utils.py +++ b/easy_rec/python/utils/check_utils.py @@ -10,20 +10,21 @@ def check_split(line, sep, requried_field_num, field_name=''): - assert sep, 'must have separator.' + (' field: %s.' % field_name) if field_name else '' + assert sep, 'must have separator.' + (' field: %s.' % + field_name) if field_name else '' for one_line in line: field_num = len(one_line.split(sep)) if field_name: assert_info = ( - 'sep[%s] maybe invalid. field_num=%d, required_num=%d, field: %s, value: %s, ' - 'please check separator and data.' % (sep, field_num, requried_field_num, field_name, one_line) - ) + 'sep[%s] maybe invalid. field_num=%d, required_num=%d, field: %s, value: %s, ' + 'please check separator and data.' % + (sep, field_num, requried_field_num, field_name, one_line)) else: assert_info = ( - 'sep[%s] maybe invalid. field_num=%d, required_num=%d, current line is: %s, ' - 'please check separator and data.' % (sep, field_num, requried_field_num, one_line) - ) + 'sep[%s] maybe invalid. field_num=%d, required_num=%d, current line is: %s, ' + 'please check separator and data.' % + (sep, field_num, requried_field_num, one_line)) assert field_num == requried_field_num, assert_info return True @@ -34,13 +35,11 @@ def check_string_to_number(field_vals, field_name): float(val) except: # noqa: E722 assert False, ( - 'StringToNumber ERROR: cannot convert string_to_number, field: %s, value: %s. ' - 'please check data.' - % ( - field_name, - val, - ) - ) + 'StringToNumber ERROR: cannot convert string_to_number, field: %s, value: %s. ' + 'please check data.' % ( + field_name, + val, + )) return True @@ -54,7 +53,7 @@ def check_sequence(pipeline_config_path, features): return for seq_att_map in seq_att_maps: assert len(seq_att_map.key) == len( - seq_att_map.hist_seq + seq_att_map.hist_seq ), 'The size of hist_seq must equal to the size of key in one seq_att_map.' size_list = [] for hist_seq in seq_att_map.hist_seq: @@ -62,30 +61,29 @@ def check_sequence(pipeline_config_path, features): size_list.append(cur_seq_size) hist_seqs = ' '.join(seq_att_map.hist_seq) assert len(set(size_list)) == 1, ( - 'SequenceFeature Error: The size in [%s] should be consistent. Please check input: [%s].' - % ( - hist_seqs, - hist_seqs, - ) - ) + 'SequenceFeature Error: The size in [%s] should be consistent. Please check input: [%s].' + % ( + hist_seqs, + hist_seqs, + )) def check_env_and_input_path(pipeline_config, input_path): input_type = pipeline_config.data_config.input_type input_type_name = DatasetConfig.InputType.Name(input_type) ignore_input_list = [ - DatasetConfig.InputType.TFRecordInput, - DatasetConfig.InputType.BatchTFRecordInput, - DatasetConfig.InputType.KafkaInput, - DatasetConfig.InputType.DataHubInput, - DatasetConfig.InputType.HiveInput, - DatasetConfig.InputType.DummyInput, + DatasetConfig.InputType.TFRecordInput, + DatasetConfig.InputType.BatchTFRecordInput, + DatasetConfig.InputType.KafkaInput, + DatasetConfig.InputType.DataHubInput, + DatasetConfig.InputType.HiveInput, + DatasetConfig.InputType.DummyInput, ] if input_type in ignore_input_list: return True assert_info = 'Current InputType is %s, InputPath is %s. Please check InputType and InputPath.' % ( - input_type_name, - input_path, + input_type_name, + input_path, ) if input_type_name.startswith('Odps'): # is on pai diff --git a/easy_rec/python/utils/config_util.py b/easy_rec/python/utils/config_util.py index fbdccc3f4..bc76f3489 100644 --- a/easy_rec/python/utils/config_util.py +++ b/easy_rec/python/utils/config_util.py @@ -15,7 +15,8 @@ import numpy as np import six import tensorflow as tf -from google.protobuf import json_format, text_format +from google.protobuf import json_format +from google.protobuf import text_format from tensorflow.python.lib.io import file_io from easy_rec.python.protos import pipeline_pb2 @@ -57,7 +58,9 @@ def get_configs_from_pipeline_file(pipeline_config_path, auto_expand=True): if isinstance(pipeline_config_path, pipeline_pb2.EasyRecConfig): return pipeline_config_path - assert tf.gfile.Exists(pipeline_config_path), 'pipeline_config_path [%s] not exists' % pipeline_config_path + assert tf.gfile.Exists( + pipeline_config_path + ), 'pipeline_config_path [%s] not exists' % pipeline_config_path pipeline_config = pipeline_pb2.EasyRecConfig() with tf.gfile.GFile(pipeline_config_path, 'r') as f: @@ -155,7 +158,9 @@ def create_pipeline_proto_from_configs(configs): return pipeline_config -def save_pipeline_config(pipeline_config, directory, filename='pipeline.config'): +def save_pipeline_config(pipeline_config, + directory, + filename='pipeline.config'): """Saves a pipeline config text file to disk. Args: @@ -173,26 +178,26 @@ def save_pipeline_config(pipeline_config, directory, filename='pipeline.config') def _get_basic_types(): dtypes = [ - bool, - int, - str, - float, - type(''), - np.float16, - np.float32, - np.float64, - np.char, - np.byte, - np.uint8, - np.int8, - np.int16, - np.uint16, - np.uint32, - np.int32, - np.uint64, - np.int64, - bool, - str, + bool, + int, + str, + float, + type(''), + np.float16, + np.float32, + np.float64, + np.char, + np.byte, + np.uint8, + np.int8, + np.int16, + np.uint16, + np.uint32, + np.int32, + np.uint64, + np.int64, + bool, + str, ] if six.PY2: dtypes.append(long) # noqa: F821 @@ -236,7 +241,7 @@ def _get_attr(obj, attr, only_last=False): for obj in objs: if '[' in key: pos = key.find('[') - name, cond = key[:pos], key[pos + 1 :] + name, cond = key[:pos], key[pos + 1:] cond = cond[:-1] update_objs = getattr(obj, name) # select all update_objs @@ -254,7 +259,7 @@ def _get_attr(obj, attr, only_last=False): sid = 0 else: sid = int(sid) - eid = cond[(colon_pos + 1) :] + eid = cond[(colon_pos + 1):] if len(eid) == 0: eid = len(update_objs) else: @@ -276,11 +281,11 @@ def _get_attr(obj, attr, only_last=False): # for complex conditions a[optimizer.lr=20] op_func_map = { - '>=': lambda x, y: x >= y, - '<=': lambda x, y: x <= y, - '<': lambda x, y: x < y, - '>': lambda x, y: x > y, - '=': lambda x, y: x == y, + '>=': lambda x, y: x >= y, + '<=': lambda x, y: x <= y, + '<': lambda x, y: x < y, + '>': lambda x, y: x > y, + '=': lambda x, y: x == y, } cond_key = None cond_val = None @@ -289,7 +294,7 @@ def _get_attr(obj, attr, only_last=False): tmp_pos = cond.rfind(op) if tmp_pos != -1: cond_key = cond[:tmp_pos] - cond_val = cond[(tmp_pos + len(op)) :] + cond_val = cond[(tmp_pos + len(op)):] op_func = op_func_map[op] break @@ -297,7 +302,8 @@ def _get_attr(obj, attr, only_last=False): assert cond_val is not None, 'invalid cond: %s' % cond for tid, update_obj in enumerate(update_objs): - tmp, tmp_parent, _, _ = _get_attr(update_obj, cond_key, only_last=True) + tmp, tmp_parent, _, _ = _get_attr( + update_obj, cond_key, only_last=True) cond_val = _type_convert(tmp, cond_val, tmp_parent) @@ -391,8 +397,8 @@ def add_boundaries_to_config(pipeline_config, tables): if feature_name in feature_boundaries_info: if feature_config.feature_type != feature_config.SequenceFeature: logging.info( - 'feature = {0}, type = {1}, will turn to RawFeature.'.format(feature_name, feature_config.feature_type) - ) + 'feature = {0}, type = {1}, will turn to RawFeature.'.format( + feature_name, feature_config.feature_type)) feature_config.feature_type = feature_config.RawFeature feature_config.hash_bucket_size = 0 feature_config.ClearField('boundaries') @@ -418,7 +424,9 @@ def parse_time(time_data): """ if isinstance(time_data, str) or isinstance(time_data, type('')): if len(time_data) == 17: - return int(datetime.datetime.strptime(time_data, '%Y%m%d %H:%M:%S').strftime('%s')) + return int( + datetime.datetime.strptime(time_data, + '%Y%m%d %H:%M:%S').strftime('%s')) elif len(time_data) == 10: return int(time_data) else: @@ -480,12 +488,17 @@ def get_model_dir_path(pipeline_config): def set_train_input_path(pipeline_config, train_input_path): if pipeline_config.WhichOneof('train_path') == 'hive_train_input': if isinstance(train_input_path, list): - assert len(train_input_path) <= 1, 'only support one hive_train_input.table_name when hive input' + assert len( + train_input_path + ) <= 1, 'only support one hive_train_input.table_name when hive input' pipeline_config.hive_train_input.table_name = train_input_path[0] else: - assert len(train_input_path.split(',')) <= 1, 'only support one hive_train_input.table_name when hive input' + assert len( + train_input_path.split(',') + ) <= 1, 'only support one hive_train_input.table_name when hive input' pipeline_config.hive_train_input.table_name = train_input_path - logging.info('update hive_train_input.table_name to %s' % pipeline_config.hive_train_input.table_name) + logging.info('update hive_train_input.table_name to %s' % + pipeline_config.hive_train_input.table_name) elif pipeline_config.WhichOneof('train_path') == 'kafka_train_input': if isinstance(train_input_path, list): @@ -502,19 +515,25 @@ def set_train_input_path(pipeline_config, train_input_path): pipeline_config.train_input_path = ','.join(train_input_path) else: pipeline_config.train_input_path = train_input_path - logging.info('update train_input_path to %s' % pipeline_config.train_input_path) + logging.info('update train_input_path to %s' % + pipeline_config.train_input_path) return pipeline_config def set_eval_input_path(pipeline_config, eval_input_path): if pipeline_config.WhichOneof('eval_path') == 'hive_eval_input': if isinstance(eval_input_path, list): - assert len(eval_input_path) <= 1, 'only support one hive_eval_input.table_name when hive input' + assert len( + eval_input_path + ) <= 1, 'only support one hive_eval_input.table_name when hive input' pipeline_config.hive_eval_input.table_name = eval_input_path[0] else: - assert len(eval_input_path.split(',')) <= 1, 'only support one hive_eval_input.table_name when hive input' + assert len( + eval_input_path.split(',') + ) <= 1, 'only support one hive_eval_input.table_name when hive input' pipeline_config.hive_eval_input.table_name = eval_input_path - logging.info('update hive_eval_input.table_name to %s' % pipeline_config.hive_eval_input.table_name) + logging.info('update hive_eval_input.table_name to %s' % + pipeline_config.hive_eval_input.table_name) elif pipeline_config.WhichOneof('eval_path') == 'parquet_eval_input': if isinstance(eval_input_path, list): pipeline_config.parquet_eval_input = ','.join(eval_input_path) @@ -530,7 +549,8 @@ def set_eval_input_path(pipeline_config, eval_input_path): pipeline_config.eval_input_path = ','.join(eval_input_path) else: pipeline_config.eval_input_path = eval_input_path - logging.info('update eval_input_path to %s' % pipeline_config.eval_input_path) + logging.info('update eval_input_path to %s' % + pipeline_config.eval_input_path) return pipeline_config @@ -555,40 +575,43 @@ def process_neg_sampler_data_path(pipeline_config): if pipeline_config.WhichOneof('train_path') != 'hive_train_input': return hive_util = HiveUtils( - data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input, + data_config=pipeline_config.data_config, + hive_config=pipeline_config.hive_train_input, ) sampler_type = pipeline_config.data_config.WhichOneof('sampler') sampler_config = getattr(pipeline_config.data_config, sampler_type) if hasattr(sampler_config, 'input_path'): - sampler_config.input_path = process_data_path(sampler_config.input_path, hive_util) + sampler_config.input_path = process_data_path(sampler_config.input_path, + hive_util) if hasattr(sampler_config, 'user_input_path'): - sampler_config.user_input_path = process_data_path(sampler_config.user_input_path, hive_util) + sampler_config.user_input_path = process_data_path( + sampler_config.user_input_path, hive_util) if hasattr(sampler_config, 'item_input_path'): - sampler_config.item_input_path = process_data_path(sampler_config.item_input_path, hive_util) + sampler_config.item_input_path = process_data_path( + sampler_config.item_input_path, hive_util) if hasattr(sampler_config, 'pos_edge_input_path'): - sampler_config.pos_edge_input_path = process_data_path(sampler_config.pos_edge_input_path, hive_util) + sampler_config.pos_edge_input_path = process_data_path( + sampler_config.pos_edge_input_path, hive_util) if hasattr(sampler_config, 'hard_neg_edge_input_path'): - sampler_config.hard_neg_edge_input_path = process_data_path(sampler_config.hard_neg_edge_input_path, hive_util) + sampler_config.hard_neg_edge_input_path = process_data_path( + sampler_config.hard_neg_edge_input_path, hive_util) def parse_extra_config_param(extra_args, edit_config_json): arg_num = len(extra_args) arg_id = 0 while arg_id < arg_num: - if ( - extra_args[arg_id].startswith('--data_config.') - or extra_args[arg_id].startswith('--train_config.') - or extra_args[arg_id].startswith('--feature_config.') - or extra_args[arg_id].startswith('--model_config.') - or extra_args[arg_id].startswith('--export_config.') - or extra_args[arg_id].startswith('--eval_config.') - ): + if (extra_args[arg_id].startswith('--data_config.') or + extra_args[arg_id].startswith('--train_config.') or + extra_args[arg_id].startswith('--feature_config.') or + extra_args[arg_id].startswith('--model_config.') or + extra_args[arg_id].startswith('--export_config.') or + extra_args[arg_id].startswith('--eval_config.')): tmp_arg = extra_args[arg_id][2:] if '=' in tmp_arg: sep_pos = tmp_arg.find('=') k = tmp_arg[:sep_pos] - v = tmp_arg[(sep_pos + 1) :] + v = tmp_arg[(sep_pos + 1):] v = v.strip(' "\'') edit_config_json[k] = v arg_id += 1 @@ -605,7 +628,9 @@ def parse_extra_config_param(extra_args, edit_config_json): def process_multi_file_input_path(sampler_config_input_path): if '*' in sampler_config_input_path: - input_path = ','.join(file_path for file_path in tf.gfile.Glob(sampler_config_input_path.split(','))) + input_path = ','.join( + file_path + for file_path in tf.gfile.Glob(sampler_config_input_path.split(','))) else: input_path = sampler_config_input_path diff --git a/easy_rec/python/utils/constant.py b/easy_rec/python/utils/constant.py index b2d6e3044..84366fbf5 100644 --- a/easy_rec/python/utils/constant.py +++ b/easy_rec/python/utils/constant.py @@ -35,7 +35,8 @@ def enable_avx_str_split(): def has_avx_str_split(): - return ENABLE_AVX_STR_SPLIT in os.environ and os.environ[ENABLE_AVX_STR_SPLIT] == '1' + return ENABLE_AVX_STR_SPLIT in os.environ and os.environ[ + ENABLE_AVX_STR_SPLIT] == '1' def disable_avx_str_split(): diff --git a/easy_rec/python/utils/convert_rtp_fg.py b/easy_rec/python/utils/convert_rtp_fg.py index 9bcb150cc..13fdd503e 100644 --- a/easy_rec/python/utils/convert_rtp_fg.py +++ b/easy_rec/python/utils/convert_rtp_fg.py @@ -9,21 +9,21 @@ from google.protobuf import text_format from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.protos.feature_config_pb2 import ( # NOQA - FeatureConfig, - FeatureGroupConfig, - WideOrDeep, -) from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig from easy_rec.python.utils import config_util +from easy_rec.python.protos.feature_config_pb2 import ( # NOQA + FeatureConfig, FeatureGroupConfig, WideOrDeep, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 MAX_HASH_BUCKET_SIZE = 9223372036854775807 -def _gen_raw_config(feature, input_field, feature_config, is_multi, curr_embed_dim): +def _gen_raw_config(feature, input_field, feature_config, is_multi, + curr_embed_dim): if 'bucketize_boundaries' in feature: if is_multi: input_field.input_type = DatasetConfig.STRING @@ -31,7 +31,8 @@ def _gen_raw_config(feature, input_field, feature_config, is_multi, curr_embed_d else: input_field.input_type = DatasetConfig.INT32 feature_config.feature_type = feature_config.IdFeature - feature_config.num_buckets = len(feature['bucketize_boundaries'].split(',')) + 1 + feature_config.num_buckets = len( + feature['bucketize_boundaries'].split(',')) + 1 feature_config.embedding_dim = curr_embed_dim else: feature_config.feature_type = feature_config.RawFeature @@ -56,7 +57,9 @@ def _set_hash_bucket(feature, feature_config, input_field): feature_config.hash_bucket_size = feature['hash_bucket_size'] if feature_config.hash_bucket_size > 10000000: if 'max_partitions' not in feature: - logging.error('it is suggested to set max_partitions > 1 for large hash buckets[%s]' % feature['feature_name']) + logging.error( + 'it is suggested to set max_partitions > 1 for large hash buckets[%s]' + % feature['feature_name']) sys.exit(1) if feature.get('filter_freq', -1) >= 0: feature_config.ev_params.filter_freq = feature['filter_freq'] @@ -76,20 +79,21 @@ def _set_hash_bucket(feature, feature_config, input_field): def process_features( - feature_type, - feature_name, - feature, - pipeline_config, - embedding_dim, - incol_separator, - is_sequence=False, + feature_type, + feature_name, + feature, + pipeline_config, + embedding_dim, + incol_separator, + is_sequence=False, ): feature_config = FeatureConfig() feature_config.input_names.append(feature_name) feature_config.separator = incol_separator input_field = DatasetConfig.Field() input_field.input_name = feature_name - curr_embed_dim = feature.get('embedding_dimension', feature.get('embedding_dim', embedding_dim)) + curr_embed_dim = feature.get('embedding_dimension', + feature.get('embedding_dim', embedding_dim)) curr_combiner = feature.get('combiner', 'sum') if feature.get('is_cache', False): logging.info('will cache %s' % feature_name) @@ -127,7 +131,8 @@ def process_features( elif feature_type == 'lookup_feature': need_discrete = feature.get('needDiscrete', True) if not need_discrete: - _gen_raw_config(feature, input_field, feature_config, is_multi, curr_embed_dim) + _gen_raw_config(feature, input_field, feature_config, is_multi, + curr_embed_dim) else: feature_config.feature_type = feature_config.TagFeature if feature.get('needWeighting', False): @@ -136,7 +141,8 @@ def process_features( _set_hash_bucket(feature, feature_config, input_field) feature_config.combiner = curr_combiner elif feature_type == 'raw_feature': - _gen_raw_config(feature, input_field, feature_config, is_multi, curr_embed_dim) + _gen_raw_config(feature, input_field, feature_config, is_multi, + curr_embed_dim) elif feature_type == 'match_feature': need_discrete = feature.get('needDiscrete', True) if feature.get('matchType', '') == 'multihit': @@ -150,7 +156,8 @@ def process_features( feature_config.combiner = curr_combiner else: assert 'bucketize_boundaries' not in feature - _gen_raw_config(feature, input_field, feature_config, is_multi, curr_embed_dim) + _gen_raw_config(feature, input_field, feature_config, is_multi, + curr_embed_dim) elif feature_type == 'combo_feature': feature_config.feature_type = feature_config.TagFeature _set_hash_bucket(feature, feature_config, input_field) @@ -178,7 +185,9 @@ def process_features( if 'extra_combo_info' in feature: extra_combo_info = feature['extra_combo_info'] feature_names = extra_combo_info.get('feature_names', []) - assert len(feature_names) >= 1, 'The feature number for ComboFeature must be greater than 2.' + assert len( + feature_names + ) >= 1, 'The feature number for ComboFeature must be greater than 2.' combo_feature_config = FeatureConfig() combo_feature_config.input_names.append(feature_name) @@ -186,10 +195,13 @@ def process_features( combo_feature_config.input_names.append(fea_name) final_feature_name = 'combo__' + '_'.join(combo_feature_config.input_names) - final_feature_name = extra_combo_info.get('final_feature_name', final_feature_name) + final_feature_name = extra_combo_info.get('final_feature_name', + final_feature_name) combo_feature_config.feature_name = final_feature_name combo_feature_config.feature_type = combo_feature_config.ComboFeature - curr_embed_dim = extra_combo_info.get('embedding_dimension', extra_combo_info.get('embedding_dim', embedding_dim)) + curr_embed_dim = extra_combo_info.get( + 'embedding_dimension', + extra_combo_info.get('embedding_dim', embedding_dim)) curr_combiner = extra_combo_info.get('combiner', 'mean') combo_feature_config.embedding_dim = curr_embed_dim combo_feature_config.combiner = curr_combiner @@ -203,7 +215,10 @@ def process_features( return pipeline_config -def load_input_field_and_feature_config(rtp_fg, label_fields, embedding_dim=16, incol_separator='\003'): +def load_input_field_and_feature_config(rtp_fg, + label_fields, + embedding_dim=16, + incol_separator='\003'): embedding_dim = rtp_fg.get('embedding_dim', embedding_dim) logging.info('embedding_dim = %s' % embedding_dim) logging.info('label_fields = %s' % ','.join(label_fields)) @@ -226,12 +241,12 @@ def load_input_field_and_feature_config(rtp_fg, label_fields, embedding_dim=16, feature_type = feature['feature_type'] feature_name = feature['feature_name'] pipeline_config = process_features( - feature_type, - feature_name, - feature, - pipeline_config, - embedding_dim, - incol_separator, + feature_type, + feature_name, + feature, + pipeline_config, + embedding_dim, + incol_separator, ) elif 'sequence_name' in feature: logging.info('Set sequence_features group later.') @@ -241,34 +256,35 @@ def load_input_field_and_feature_config(rtp_fg, label_fields, embedding_dim=16, sub_feature_name = sub_feature['feature_name'] all_sub_feature_name = sequence_name + '_' + sub_feature_name pipeline_config = process_features( - sub_feature_type, - all_sub_feature_name, - sub_feature, - pipeline_config, - embedding_dim, - incol_separator, - is_sequence=True, + sub_feature_type, + all_sub_feature_name, + sub_feature, + pipeline_config, + embedding_dim, + incol_separator, + is_sequence=True, ) except Exception: - logging.info('convert feature[%s] exception[%s]' % (str(feature), traceback.format_exc())) + logging.info('convert feature[%s] exception[%s]' % + (str(feature), traceback.format_exc())) sys.exit(1) return pipeline_config def convert_rtp_fg( - rtp_fg, - embedding_dim=16, - batch_size=1024, - label_fields=[], - num_steps=10, - model_type='', - separator='\002', - incol_separator='\003', - train_input_path=None, - eval_input_path=None, - selected_cols='', - input_type='OdpsRTPInput', - is_async=False, + rtp_fg, + embedding_dim=16, + batch_size=1024, + label_fields=[], + num_steps=10, + model_type='', + separator='\002', + incol_separator='\003', + train_input_path=None, + eval_input_path=None, + selected_cols='', + input_type='OdpsRTPInput', + is_async=False, ): with tf.gfile.GFile(rtp_fg, 'r') as fin: rtp_fg = json.load(fin) @@ -287,7 +303,9 @@ def convert_rtp_fg( logging.info('model_path = %s' % model_path) logging.info('edit_config_json = %s' % edit_config_json) - pipeline_config = load_input_field_and_feature_config(rtp_fg, label_fields, embedding_dim, incol_separator) + pipeline_config = load_input_field_and_feature_config(rtp_fg, label_fields, + embedding_dim, + incol_separator) pipeline_config.model_dir = model_dir pipeline_config.data_config.separator = separator if selected_cols: @@ -330,8 +348,8 @@ def convert_rtp_fg( sync_replicas: %s } """ % ( - 'adam_optimizer' if not is_async else 'adam_async_optimizer', - 'true' if not is_async else 'false', + 'adam_optimizer' if not is_async else 'adam_async_optimizer', + 'true' if not is_async else 'false', ) text_format.Merge(train_config_str, pipeline_config) @@ -405,11 +423,11 @@ def convert_rtp_fg( feature_groups = {} group_map = { - 'u': 'user', - 'i': 'item', - 'ctx': 'combo', - 'q': 'combo', - 'comb': 'combo', + 'u': 'user', + 'i': 'item', + 'ctx': 'combo', + 'q': 'combo', + 'comb': 'combo', } for feature in rtp_features: feature_name = feature['feature_name'].strip() @@ -427,8 +445,8 @@ def convert_rtp_fg( feature_groups[group_name] = [feature_name] logging.info( - 'if group is specified, group will be used as feature group name; ' - 'otherwise, the prefix of feature_name in fg.json is used as feature group name' + 'if group is specified, group will be used as feature group name; ' + 'otherwise, the prefix of feature_name in fg.json is used as feature group name' ) logging.info('prefix map: %s' % str(group_map)) for group_name in feature_groups: @@ -442,28 +460,23 @@ def convert_rtp_fg( multi_tower_config_str = ' multi_tower {\n' for group_name in feature_groups: - multi_tower_config_str += ( - """ + multi_tower_config_str += (""" towers { input: "%s" dnn { hidden_units: [256, 192, 128] } } - """ - % group_name - ) + """ % group_name) multi_tower_config_str = ( - multi_tower_config_str - + """ + multi_tower_config_str + """ final_dnn { hidden_units: [192, 128, 64] } l2_regularization: 1e-4 } - """ - ) + """) text_format.Merge(multi_tower_config_str, pipeline_config.model_config) pipeline_config.model_config.embedding_regularization = 1e-5 @@ -490,16 +503,13 @@ def convert_rtp_fg( esmm_config_str = ' esmm {\n' for group_name in feature_groups: - esmm_config_str += ( - """ + esmm_config_str += (""" groups { input: "%s" dnn { hidden_units: [256, 128, 96, 64] } - }""" - % group_name - ) + }""" % group_name) esmm_config_str += """ ctr_tower { @@ -530,8 +540,8 @@ def convert_rtp_fg( } l2_regularization: 1e-6 }""" % ( - label_fields[0], - label_fields[1], + label_fields[0], + label_fields[1], ) text_format.Merge(esmm_config_str, pipeline_config.model_config) pipeline_config.model_config.embedding_regularization = 5e-5 @@ -599,28 +609,28 @@ def convert_rtp_fg( l2_regularization: 1e-6 } """ % ( - label_fields[0], - label_fields[1], + label_fields[0], + label_fields[1], ) text_format.Merge(dbmtl_config_str, pipeline_config.model_config) pipeline_config.model_config.embedding_regularization = 5e-6 if model_type in ['wide_and_deep', 'deepfm', 'multi_tower']: text_format.Merge( - """ + """ metrics_set { auc {} } """, - pipeline_config.eval_config, + pipeline_config.eval_config, ) text_format.Merge( - """ export_config { + """ export_config { multi_placeholder: false } """, - pipeline_config, + pipeline_config, ) if edit_config_json: diff --git a/easy_rec/python/utils/dag.py b/easy_rec/python/utils/dag.py index acfabe103..b9ae24725 100644 --- a/easy_rec/python/utils/dag.py +++ b/easy_rec/python/utils/dag.py @@ -1,6 +1,8 @@ import logging -from collections import OrderedDict, defaultdict -from copy import copy, deepcopy +from collections import OrderedDict +from collections import defaultdict +from copy import copy +from copy import deepcopy class DAG(object): @@ -109,7 +111,9 @@ def all_downstreams(self, node, graph=None): nodes_seen.add(downstream_node) nodes.append(downstream_node) i += 1 - return list(filter(lambda node: node in nodes_seen, self.topological_sort(graph=graph))) + return list( + filter(lambda node: node in nodes_seen, + self.topological_sort(graph=graph))) def all_leaves(self, graph=None): """Return a list of all leaves (nodes with no downstreams).""" @@ -140,7 +144,8 @@ def independent_nodes(self, graph=None): if graph is None: graph = self.graph - dependent_nodes = set(node for dependents in graph.values() for node in dependents) + dependent_nodes = set( + node for dependents in graph.values() for node in dependents) return [node for node in graph.keys() if node not in dependent_nodes] def validate(self, graph=None): diff --git a/easy_rec/python/utils/distribution_utils.py b/easy_rec/python/utils/distribution_utils.py index 43015e2fd..1be75b59c 100644 --- a/easy_rec/python/utils/distribution_utils.py +++ b/easy_rec/python/utils/distribution_utils.py @@ -10,26 +10,27 @@ from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import estimator_utils + from easy_rec.python.utils.estimator_utils import ( # NOQA - chief_to_master, - master_to_chief, + chief_to_master, master_to_chief, ) DistributionStrategyMap = { - '': DistributionStrategy.NoStrategy, - 'ps': DistributionStrategy.PSStrategy, - 'ess': DistributionStrategy.ExascaleStrategy, - 'mirrored': DistributionStrategy.MirroredStrategy, - 'collective': DistributionStrategy.CollectiveAllReduceStrategy, + '': DistributionStrategy.NoStrategy, + 'ps': DistributionStrategy.PSStrategy, + 'ess': DistributionStrategy.ExascaleStrategy, + 'mirrored': DistributionStrategy.MirroredStrategy, + 'collective': DistributionStrategy.CollectiveAllReduceStrategy, } -def set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy): +def set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, + distribute_strategy): if distribute_strategy in [ - DistributionStrategy.PSStrategy, - DistributionStrategy.MirroredStrategy, - DistributionStrategy.CollectiveAllReduceStrategy, - DistributionStrategy.ExascaleStrategy, + DistributionStrategy.PSStrategy, + DistributionStrategy.MirroredStrategy, + DistributionStrategy.CollectiveAllReduceStrategy, + DistributionStrategy.ExascaleStrategy, ]: pipeline_config.train_config.sync_replicas = False pipeline_config.train_config.train_distribute = distribute_strategy @@ -39,14 +40,16 @@ def set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, di def set_tf_config_and_get_train_worker_num( - ps_hosts, - worker_hosts, - task_index, - job_name, - distribute_strategy=DistributionStrategy.NoStrategy, - eval_method='none', + ps_hosts, + worker_hosts, + task_index, + job_name, + distribute_strategy=DistributionStrategy.NoStrategy, + eval_method='none', ): - logging.info('set_tf_config_and_get_train_worker_num: distribute_strategy = %d' % distribute_strategy) + logging.info( + 'set_tf_config_and_get_train_worker_num: distribute_strategy = %d' % + distribute_strategy) worker_hosts = worker_hosts.split(',') ps_hosts = ps_hosts.split(',') if ps_hosts else [] @@ -54,19 +57,17 @@ def set_tf_config_and_get_train_worker_num( train_worker_num = total_worker_num print('Original TF_CONFIG=%s' % os.environ.get('TF_CONFIG', '')) - print( - 'worker_hosts=%s ps_hosts=%s task_index=%d job_name=%s' - % (','.join(worker_hosts), ','.join(ps_hosts), task_index, job_name) - ) + print('worker_hosts=%s ps_hosts=%s task_index=%d job_name=%s' % + (','.join(worker_hosts), ','.join(ps_hosts), task_index, job_name)) print('eval_method=%s' % eval_method) if distribute_strategy == DistributionStrategy.MirroredStrategy: assert total_worker_num == 1, 'mirrored distribute strategy only need 1 worker' elif distribute_strategy in [ - DistributionStrategy.NoStrategy, - DistributionStrategy.PSStrategy, - DistributionStrategy.CollectiveAllReduceStrategy, - DistributionStrategy.ExascaleStrategy, + DistributionStrategy.NoStrategy, + DistributionStrategy.PSStrategy, + DistributionStrategy.CollectiveAllReduceStrategy, + DistributionStrategy.ExascaleStrategy, ]: cluster, task_type, task_index_ = estimator_utils.parse_tf_config() train_worker_num = 0 @@ -83,8 +84,11 @@ def set_tf_config_and_get_train_worker_num( if distribute_strategy == DistributionStrategy.NoStrategy: del cluster['evaluator'] tf_config = { - 'cluster': cluster, - 'task': {'type': task_type, 'index': task_index_}, + 'cluster': cluster, + 'task': { + 'type': task_type, + 'index': task_index_ + }, } os.environ['TF_CONFIG'] = json.dumps(tf_config) else: @@ -92,8 +96,8 @@ def set_tf_config_and_get_train_worker_num( # -Dcluster, we use first worker for chief, second for evaluation train_worker_num = total_worker_num - 1 assert train_worker_num > 0, ( - 'in distribution mode worker num must be greater than 1, ' 'the second worker will be used as evaluator' - ) + 'in distribution mode worker num must be greater than 1, ' + 'the second worker will be used as evaluator') if len(worker_hosts) > 1: cluster = {'chief': [worker_hosts[0]], 'worker': worker_hosts[2:]} if distribute_strategy != DistributionStrategy.NoStrategy: @@ -101,34 +105,38 @@ def set_tf_config_and_get_train_worker_num( if len(ps_hosts) > 0: cluster['ps'] = ps_hosts if job_name == 'ps': - os.environ['TF_CONFIG'] = json.dumps( - { + os.environ['TF_CONFIG'] = json.dumps({ 'cluster': cluster, - 'task': {'type': job_name, 'index': task_index}, - } - ) + 'task': { + 'type': job_name, + 'index': task_index + }, + }) elif job_name == 'worker': if task_index == 0: - os.environ['TF_CONFIG'] = json.dumps( - { + os.environ['TF_CONFIG'] = json.dumps({ 'cluster': cluster, - 'task': {'type': 'chief', 'index': 0}, - } - ) + 'task': { + 'type': 'chief', + 'index': 0 + }, + }) elif task_index == 1: - os.environ['TF_CONFIG'] = json.dumps( - { + os.environ['TF_CONFIG'] = json.dumps({ 'cluster': cluster, - 'task': {'type': 'evaluator', 'index': 0}, - } - ) + 'task': { + 'type': 'evaluator', + 'index': 0 + }, + }) else: - os.environ['TF_CONFIG'] = json.dumps( - { + os.environ['TF_CONFIG'] = json.dumps({ 'cluster': cluster, - 'task': {'type': job_name, 'index': task_index - 2}, - } - ) + 'task': { + 'type': job_name, + 'index': task_index - 2 + }, + }) else: if 'evaluator' in cluster: evaluator = cluster['evaluator'] @@ -146,13 +154,19 @@ def set_tf_config_and_get_train_worker_num( cluster['worker'] = [evaluator[0]] if task_type == 'evaluator': tf_config = { - 'cluster': cluster, - 'task': {'type': 'worker', 'index': train_worker_num - 2}, + 'cluster': cluster, + 'task': { + 'type': 'worker', + 'index': train_worker_num - 2 + }, } else: tf_config = { - 'cluster': cluster, - 'task': {'type': task_type, 'index': task_index_}, + 'cluster': cluster, + 'task': { + 'type': task_type, + 'index': task_index_ + }, } os.environ['TF_CONFIG'] = json.dumps(tf_config) else: @@ -161,22 +175,30 @@ def set_tf_config_and_get_train_worker_num( if len(ps_hosts) > 0: cluster['ps'] = ps_hosts if job_name == 'ps': - os.environ['TF_CONFIG'] = json.dumps( - { + os.environ['TF_CONFIG'] = json.dumps({ 'cluster': cluster, - 'task': {'type': job_name, 'index': task_index}, - } - ) + 'task': { + 'type': job_name, + 'index': task_index + }, + }) else: if task_index == 0: - os.environ['TF_CONFIG'] = json.dumps({'cluster': cluster, 'task': {'type': 'chief', 'index': 0}}) + os.environ['TF_CONFIG'] = json.dumps({ + 'cluster': cluster, + 'task': { + 'type': 'chief', + 'index': 0 + } + }) else: - os.environ['TF_CONFIG'] = json.dumps( - { + os.environ['TF_CONFIG'] = json.dumps({ 'cluster': cluster, - 'task': {'type': 'worker', 'index': task_index - 1}, - } - ) + 'task': { + 'type': 'worker', + 'index': task_index - 1 + }, + }) if eval_method == 'none': # change master to chief, will not evaluate master_to_chief() @@ -188,7 +210,8 @@ def set_tf_config_and_get_train_worker_num( cluster, task_type, task_index = estimator_utils.parse_tf_config() print('Final TF_CONFIG = %s' % os.environ.get('TF_CONFIG', '')) tf.logging.info('TF_CONFIG %s' % os.environ.get('TF_CONFIG', '')) - tf.logging.info('distribute_stategy %s, train_worker_num: %d' % (distribute_strategy, train_worker_num)) + tf.logging.info('distribute_stategy %s, train_worker_num: %d' % + (distribute_strategy, train_worker_num)) # remove pai chief-worker waiting strategy # which is conflicted with worker waiting strategy in easyrec @@ -201,7 +224,8 @@ def set_tf_config_and_get_train_worker_num_on_ds(): if 'TF_CONFIG' not in os.environ: return tf_config = json.loads(os.environ['TF_CONFIG']) - if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and ('evaluator' not in tf_config['cluster']): + if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and ( + 'evaluator' not in tf_config['cluster']): easyrec_tf_config = dict() easyrec_tf_config['cluster'] = {} easyrec_tf_config['task'] = {} @@ -212,7 +236,8 @@ def set_tf_config_and_get_train_worker_num_on_ds(): if tf_config['task']['type'] == 'worker' and tf_config['task']['index'] == 0: easyrec_tf_config['task']['type'] = 'chief' easyrec_tf_config['task']['index'] = 0 - elif tf_config['task']['type'] == 'worker' and tf_config['task']['index'] == 1: + elif tf_config['task']['type'] == 'worker' and tf_config['task'][ + 'index'] == 1: easyrec_tf_config['task']['type'] = 'evaluator' easyrec_tf_config['task']['index'] = 0 elif tf_config['task']['type'] == 'worker': @@ -227,7 +252,8 @@ def set_tf_config_and_get_train_worker_num_on_ds(): def set_tf_config_and_get_distribute_eval_worker_num_on_ds(): assert 'TF_CONFIG' in os.environ, "'TF_CONFIG' must in os.environ" tf_config = json.loads(os.environ['TF_CONFIG']) - if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and ('evaluator' not in tf_config['cluster']): + if 'cluster' in tf_config and 'ps' in tf_config['cluster'] and ( + 'evaluator' not in tf_config['cluster']): easyrec_tf_config = dict() easyrec_tf_config['cluster'] = {} easyrec_tf_config['task'] = {} diff --git a/easy_rec/python/utils/ds_util.py b/easy_rec/python/utils/ds_util.py index 9b2117c22..25bb880a2 100644 --- a/easy_rec/python/utils/ds_util.py +++ b/easy_rec/python/utils/ds_util.py @@ -27,7 +27,8 @@ def cache_ckpt(pipeline_config): # there is no need to cache if remote directories are mounted return - if estimator_utils.is_ps() or estimator_utils.is_chief() or estimator_utils.is_master(): + if estimator_utils.is_ps() or estimator_utils.is_chief( + ) or estimator_utils.is_master(): tmpdir = os.path.dirname(fine_tune_ckpt_path.replace('hdfs://', '')) tmpdir = os.path.join('/tmp/experiments', tmpdir) logging.info('will cache fine_tune_ckpt to local dir: %s' % tmpdir) @@ -50,7 +51,8 @@ def cache_ckpt(pipeline_config): dst_path = os.path.join(tmpdir, os.path.basename(src_path)) logging.info('will copy %s to local path %s' % (src_path, dst_path)) try: - output = subprocess.check_output('hadoop fs -get %s %s' % (src_path, dst_path), shell=True) + output = subprocess.check_output( + 'hadoop fs -get %s %s' % (src_path, dst_path), shell=True) logging.info('copy succeed: %s' % output) except Exception: logging.warning('exception: %s' % traceback.format_exc()) diff --git a/easy_rec/python/utils/embedding_utils.py b/easy_rec/python/utils/embedding_utils.py index 064ce5043..89f326e4f 100644 --- a/easy_rec/python/utils/embedding_utils.py +++ b/easy_rec/python/utils/embedding_utils.py @@ -5,7 +5,8 @@ import tensorflow as tf from tensorflow.python.framework import ops -from easy_rec.python.utils import constant, proto_util +from easy_rec.python.utils import constant +from easy_rec.python.utils import proto_util if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/utils/estimator_utils.py b/easy_rec/python/utils/estimator_utils.py index 7801f4000..c761e6cfc 100644 --- a/easy_rec/python/utils/estimator_utils.py +++ b/easy_rec/python/utils/estimator_utils.py @@ -1,6 +1,8 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import json import logging @@ -14,18 +16,23 @@ import tensorflow as tf from tensorflow.core.framework.summary_pb2 import Summary from tensorflow.python.client import device_lib -from tensorflow.python.framework import errors_impl, meta_graph, ops +from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import meta_graph +from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.platform import gfile +from tensorflow.python.training.summary_io import SummaryWriterCache + +from easy_rec.python.utils import constant +from easy_rec.python.utils import embedding_utils +from easy_rec.python.utils import shape_utils + from tensorflow.python.training import basic_session_run_hooks, session_run_hook # NOQA from tensorflow.python.training.basic_session_run_hooks import SecondOrStepTimer # NOQA -from tensorflow.python.training.summary_io import SummaryWriterCache from easy_rec.python.ops.incr_record import ( # NOQA - get_sparse_indices, - kv_resource_incr_gather, + get_sparse_indices, kv_resource_incr_gather, ) -from easy_rec.python.utils import constant, embedding_utils, shape_utils try: import horovod.tensorflow as hvd @@ -85,26 +92,28 @@ def __init__(self, num_worker, is_chief, model_dir): def begin(self): """Count the number of workers and masters, and setup barrier queue.""" tf.logging.info('number workers(including master) = %d' % self._num_worker) - with tf.device(tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0)): + with tf.device( + tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0)): self._queue = tf.FIFOQueue( - capacity=self._num_worker, - dtypes=[tf.float32], - shapes=[()], - name='exit_counter', - shared_name='exit_counter', + capacity=self._num_worker, + dtypes=[tf.float32], + shapes=[()], + name='exit_counter', + shared_name='exit_counter', ) self._signal_que = tf.FIFOQueue( - capacity=self._num_worker, - dtypes=[tf.string], - shapes=[()], - name='exit_counter_signal', - shared_name='exit_counter_signal', + capacity=self._num_worker, + dtypes=[tf.string], + shapes=[()], + name='exit_counter_signal', + shared_name='exit_counter_signal', ) self._enque = self._queue.enqueue(1.0) self._que_size = self._queue.size() self._deque = self._queue.dequeue() if self._is_chief: - self._flag_file = os.path.join(self._model_dir, 'atexit_sync_' + str(int(time.time()))) + self._flag_file = os.path.join(self._model_dir, + 'atexit_sync_' + str(int(time.time()))) self._send = self._signal_que.enqueue([self._flag_file]) else: self._recv = self._signal_que.dequeue() @@ -130,7 +139,9 @@ def end(self, session): while que_size < self._num_worker: que_size = session.run(self._que_size) time.sleep(5) - tf.logging.info('waiting for other worker to exit, finished %d, total %d' % (que_size, self._num_worker)) + tf.logging.info( + 'waiting for other worker to exit, finished %d, total %d' % + (que_size, self._num_worker)) # prepare on_exit synchronize base on self._flag_file if self._is_chief: for i in range(self._num_worker - 1): @@ -139,7 +150,8 @@ def end(self, session): self._flag_file = session.run(self._recv) def _check_flag_file(is_chief, flag_file): - logging.info('_check_flag_file: is_chief = %d flag_file=%s' % (is_chief, flag_file)) + logging.info('_check_flag_file: is_chief = %d flag_file=%s' % + (is_chief, flag_file)) if is_chief: with gfile.GFile(flag_file, 'w') as fout: fout.write('atexit time: %d' % int(time.time())) @@ -149,7 +161,8 @@ def _check_flag_file(is_chief, flag_file): from atexit import register - register(_check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file) + register( + _check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file) logging.info('ExitBarrier passed') @@ -178,26 +191,28 @@ def __init__(self, num_worker, is_chief, model_dir, metric_ops=None): def begin(self): """Count the number of workers and masters, and setup barrier queue.""" tf.logging.info('number workers(including master) = %d' % self._num_worker) - with tf.device(tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0)): + with tf.device( + tf.DeviceSpec(job='ps', task=0, device_type='CPU', device_index=0)): self._queue = tf.FIFOQueue( - capacity=self._num_worker, - dtypes=[tf.float32], - shapes=[()], - name='exit_counter', - shared_name='exit_counter', + capacity=self._num_worker, + dtypes=[tf.float32], + shapes=[()], + name='exit_counter', + shared_name='exit_counter', ) self._signal_que = tf.FIFOQueue( - capacity=self._num_worker, - dtypes=[tf.string], - shapes=[()], - name='exit_counter_signal', - shared_name='exit_counter_signal', + capacity=self._num_worker, + dtypes=[tf.string], + shapes=[()], + name='exit_counter_signal', + shared_name='exit_counter_signal', ) self._enque = self._queue.enqueue(1.0) self._que_size = self._queue.size() self._deque = self._queue.dequeue() if self._is_chief: - self._flag_file = os.path.join(self._model_dir, 'atexit_sync_' + str(int(time.time()))) + self._flag_file = os.path.join(self._model_dir, + 'atexit_sync_' + str(int(time.time()))) self._send = self._signal_que.enqueue([self._flag_file]) else: self._recv = self._signal_que.dequeue() @@ -223,7 +238,9 @@ def end(self, session): while que_size < self._num_worker: que_size = session.run(self._que_size) time.sleep(5) - tf.logging.info('waiting for other worker to exit, finished %d, total %d' % (que_size, self._num_worker)) + tf.logging.info( + 'waiting for other worker to exit, finished %d, total %d' % + (que_size, self._num_worker)) # prepare on_exit synchronize base on self._flag_file if self._is_chief: self.eval_result = session.run(self.metric_ops) @@ -233,7 +250,8 @@ def end(self, session): self._flag_file = session.run(self._recv) def _check_flag_file(is_chief, flag_file): - logging.info('_check_flag_file: is_chief = %d flag_file=%s' % (is_chief, flag_file)) + logging.info('_check_flag_file: is_chief = %d flag_file=%s' % + (is_chief, flag_file)) if is_chief: with gfile.GFile(flag_file, 'w') as fout: fout.write('atexit time: %d' % int(time.time())) @@ -243,13 +261,15 @@ def _check_flag_file(is_chief, flag_file): from atexit import register - register(_check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file) + register( + _check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file) session.run(self.metric_ops) logging.info('ExitBarrier passed') class ProgressHook(SessionRunHook): + def __init__(self, num_steps, filename, is_chief): """Initializes a `ProgressHook`. @@ -292,17 +312,17 @@ class CheckpointSaverHook(CheckpointSaverHook): """Saves checkpoints every N steps or seconds.""" def __init__( - self, - checkpoint_dir, - save_secs=None, - save_steps=None, - saver=None, - checkpoint_basename='model.ckpt', - scaffold=None, - listeners=None, - write_graph=True, - data_offset_var=None, - increment_save_config=None, + self, + checkpoint_dir, + save_secs=None, + save_steps=None, + saver=None, + checkpoint_basename='model.ckpt', + scaffold=None, + listeners=None, + write_graph=True, + data_offset_var=None, + increment_save_config=None, ): """Initializes a `CheckpointSaverHook`. @@ -325,13 +345,13 @@ def __init__( ValueError: At most one of saver or scaffold should be set. """ super(CheckpointSaverHook, self).__init__( - checkpoint_dir, - save_secs=save_secs, - save_steps=save_steps, - saver=saver, - checkpoint_basename=checkpoint_basename, - scaffold=scaffold, - listeners=listeners, + checkpoint_dir, + save_secs=save_secs, + save_steps=save_steps, + saver=saver, + checkpoint_basename=checkpoint_basename, + scaffold=scaffold, + listeners=listeners, ) self._cuda_profile_start = 0 self._cuda_profile_stop = 0 @@ -344,28 +364,32 @@ def __init__( if increment_save_config is not None: self._kafka_timeout_ms = os.environ.get('KAFKA_TIMEOUT', 600) * 1000 logging.info('KAFKA_TIMEOUT: %dms' % self._kafka_timeout_ms) - self._kafka_max_req_size = os.environ.get('KAFKA_MAX_REQ_SIZE', 1024 * 1024 * 64) + self._kafka_max_req_size = os.environ.get('KAFKA_MAX_REQ_SIZE', + 1024 * 1024 * 64) logging.info('KAFKA_MAX_REQ_SIZE: %d' % self._kafka_max_req_size) - self._kafka_max_msg_size = os.environ.get('KAFKA_MAX_MSG_SIZE', 1024 * 1024 * 1024) + self._kafka_max_msg_size = os.environ.get('KAFKA_MAX_MSG_SIZE', + 1024 * 1024 * 1024) logging.info('KAFKA_MAX_MSG_SIZE: %d' % self._kafka_max_msg_size) self._dense_name_to_ids = embedding_utils.get_dense_name_to_ids() self._sparse_name_to_ids = embedding_utils.get_sparse_name_to_ids() - with gfile.GFile(os.path.join(checkpoint_dir, constant.DENSE_UPDATE_VARIABLES), 'w') as fout: + with gfile.GFile( + os.path.join(checkpoint_dir, constant.DENSE_UPDATE_VARIABLES), + 'w') as fout: json.dump(self._dense_name_to_ids, fout, indent=2) save_secs = increment_save_config.dense_save_secs save_steps = increment_save_config.dense_save_steps self._dense_timer = SecondOrStepTimer( - every_secs=save_secs if save_secs > 0 else None, - every_steps=save_steps if save_steps > 0 else None, + every_secs=save_secs if save_secs > 0 else None, + every_steps=save_steps if save_steps > 0 else None, ) save_secs = increment_save_config.sparse_save_secs save_steps = increment_save_config.sparse_save_steps self._sparse_timer = SecondOrStepTimer( - every_secs=save_secs if save_secs > 0 else None, - every_steps=save_steps if save_steps > 0 else None, + every_secs=save_secs if save_secs > 0 else None, + every_steps=save_steps if save_steps > 0 else None, ) self._dense_timer.update_last_triggered_step(0) @@ -377,20 +401,21 @@ def __init__( for sparse_var, indice_dtype in sparse_train_vars: with ops.control_dependencies([tf.train.get_global_step()]): with ops.colocate_with(sparse_var): - sparse_indice = get_sparse_indices(var_name=sparse_var.op.name, ktype=indice_dtype) + sparse_indice = get_sparse_indices( + var_name=sparse_var.op.name, ktype=indice_dtype) # sparse_indice = sparse_indice.global_indices self._sparse_indices.append(sparse_indice) if 'EmbeddingVariable' in str(type(sparse_var)): self._sparse_values.append( - kv_resource_incr_gather( - sparse_var._handle, - sparse_indice, - np.zeros(sparse_var.shape.as_list(), dtype=np.float32), - ) - ) + kv_resource_incr_gather( + sparse_var._handle, + sparse_indice, + np.zeros(sparse_var.shape.as_list(), dtype=np.float32), + )) # sparse_var.sparse_read(sparse_indice)) else: - self._sparse_values.append(array_ops.gather(sparse_var, sparse_indice)) + self._sparse_values.append( + array_ops.gather(sparse_var, sparse_indice)) self._kafka_producer = None self._incr_save_dir = None @@ -399,31 +424,33 @@ def __init__( logging.info('increment save topic: %s' % self._topic) admin_clt = KafkaAdminClient( - bootstrap_servers=increment_save_config.kafka.server, - request_timeout_ms=self._kafka_timeout_ms, - api_version_auto_timeout_ms=self._kafka_timeout_ms, + bootstrap_servers=increment_save_config.kafka.server, + request_timeout_ms=self._kafka_timeout_ms, + api_version_auto_timeout_ms=self._kafka_timeout_ms, ) if self._topic not in admin_clt.list_topics(): admin_clt.create_topics( - new_topics=[ - NewTopic( - name=self._topic, - num_partitions=1, - replication_factor=1, - topic_configs={'max.message.bytes': self._kafka_max_msg_size}, - ) - ], - validate_only=False, + new_topics=[ + NewTopic( + name=self._topic, + num_partitions=1, + replication_factor=1, + topic_configs={ + 'max.message.bytes': self._kafka_max_msg_size + }, + ) + ], + validate_only=False, ) logging.info('create increment save topic: %s' % self._topic) admin_clt.close() servers = increment_save_config.kafka.server.split(',') self._kafka_producer = KafkaProducer( - bootstrap_servers=servers, - max_request_size=self._kafka_max_req_size, - api_version_auto_timeout_ms=self._kafka_timeout_ms, - request_timeout_ms=self._kafka_timeout_ms, + bootstrap_servers=servers, + max_request_size=self._kafka_max_req_size, + api_version_auto_timeout_ms=self._kafka_timeout_ms, + request_timeout_ms=self._kafka_timeout_ms, ) elif increment_save_config.HasField('fs'): fs = increment_save_config.fs @@ -438,7 +465,8 @@ def __init__( elif increment_save_config.HasField('datahub'): raise NotImplementedError('datahub increment saving is in development.') else: - raise ValueError('incr_update not specified correctly, must be oneof: kafka,fs') + raise ValueError( + 'incr_update not specified correctly, must be oneof: kafka,fs') self._debug_save_update = increment_save_config.debug_save_update else: @@ -452,15 +480,14 @@ def after_create_session(self, session, coord): # We cannot do this in begin, since we let other hooks to change graph and # add variables at begin. Graph is finalized after all begin calls. tf.train.write_graph( - tf.get_default_graph().as_graph_def(add_shapes=True), - self._checkpoint_dir, - 'graph.pbtxt', + tf.get_default_graph().as_graph_def(add_shapes=True), + self._checkpoint_dir, + 'graph.pbtxt', ) saver_def = self._get_saver().saver_def if self._get_saver() else None graph = tf.get_default_graph() meta_graph_def = meta_graph.create_meta_graph_def( - graph_def=graph.as_graph_def(add_shapes=True), saver_def=saver_def - ) + graph_def=graph.as_graph_def(add_shapes=True), saver_def=saver_def) self._summary_writer.add_graph(graph) self._summary_writer.add_meta_graph(meta_graph_def) @@ -493,11 +520,14 @@ def _send_dense(self, global_step, session): if self._kafka_producer is not None: msg_key = 'dense_update_%d' % global_step - send_res = self._kafka_producer.send(self._topic, bytes_buf, key=msg_key.encode('utf-8')) - logging.info('kafka send dense: %d exception: %s' % (global_step, send_res.exception)) + send_res = self._kafka_producer.send( + self._topic, bytes_buf, key=msg_key.encode('utf-8')) + logging.info('kafka send dense: %d exception: %s' % + (global_step, send_res.exception)) if self._incr_save_dir is not None: - save_path = os.path.join(self._incr_save_dir, 'dense_update_%d' % global_step) + save_path = os.path.join(self._incr_save_dir, + 'dense_update_%d' % global_step) with gfile.GFile(save_path, 'wb') as fout: fout.write(bytes_buf) save_flag = save_path + '.done' @@ -513,7 +543,9 @@ def _send_dense(self, global_step, session): with gfile.GFile(save_path, 'wb') as fout: fout.write(bytes_buf) - logging.info('global_step=%d, increment update dense variables, msg_num=%d' % (global_step, msg_num)) + logging.info( + 'global_step=%d, increment update dense variables, msg_num=%d' % + (global_step, msg_num)) def _send_sparse(self, global_step, session): sparse_train_vars = ops.get_collection(constant.SPARSE_UPDATE_VARIABLES) @@ -525,12 +557,15 @@ def _send_sparse(self, global_step, session): sparse_val_res = [sparse_res[i + msg_num] for i in sel_ids] sparse_train_vars = [sparse_train_vars[i][0] for i in sel_ids] - sel_embed_ids = [self._sparse_name_to_ids[x.name] for x in sparse_train_vars] + sel_embed_ids = [ + self._sparse_name_to_ids[x.name] for x in sparse_train_vars + ] msg_num = len(sel_ids) if msg_num == 0: - logging.warning('there are no sparse updates, will skip this send: %d' % global_step) + logging.warning('there are no sparse updates, will skip this send: %d' % + global_step) return # build msg header @@ -542,7 +577,9 @@ def _send_sparse(self, global_step, session): bytes_buf = np.array(msg_header, dtype=np.int32).tobytes() # build msg body - for tmp_id, tmp_key, tmp_val, tmp_var in zip(sel_embed_ids, sparse_key_res, sparse_val_res, sparse_train_vars): + for tmp_id, tmp_key, tmp_val, tmp_var in zip(sel_embed_ids, sparse_key_res, + sparse_val_res, + sparse_train_vars): # for non kv embedding variables, add partition offset to tmp_key if 'EmbeddingVariable' not in str(type(tmp_var)): if tmp_var._save_slice_info is not None: @@ -551,11 +588,14 @@ def _send_sparse(self, global_step, session): bytes_buf += tmp_val.tobytes() if self._kafka_producer is not None: msg_key = 'sparse_update_%d' % global_step - send_res = self._kafka_producer.send(self._topic, bytes_buf, key=msg_key.encode('utf-8')) - logging.info('kafka send sparse: %d %s' % (global_step, send_res.exception)) + send_res = self._kafka_producer.send( + self._topic, bytes_buf, key=msg_key.encode('utf-8')) + logging.info('kafka send sparse: %d %s' % + (global_step, send_res.exception)) if self._incr_save_dir is not None: - save_path = os.path.join(self._incr_save_dir, 'sparse_update_%d' % global_step) + save_path = os.path.join(self._incr_save_dir, + 'sparse_update_%d' % global_step) with gfile.GFile(save_path, 'wb') as fout: fout.write(bytes_buf) save_flag = save_path + '.done' @@ -572,24 +612,21 @@ def _send_sparse(self, global_step, session): fout.write(bytes_buf) logging.info( - 'global_step=%d, increment update sparse variables, msg_num=%d, msg_size=%d' - % (global_step, msg_num, len(bytes_buf)) - ) + 'global_step=%d, increment update sparse variables, msg_num=%d, msg_size=%d' + % (global_step, msg_num, len(bytes_buf))) def after_run(self, run_context, run_values): super(CheckpointSaverHook, self).after_run(run_context, run_values) stale_global_step = run_values.results global_step = -1 if self._dense_timer is not None and self._dense_timer.should_trigger_for_step( - stale_global_step + self._steps_per_run - ): + stale_global_step + self._steps_per_run): global_step = run_context.session.run(self._global_step_tensor) self._dense_timer.update_last_triggered_step(global_step) self._send_dense(global_step, run_context.session) if self._sparse_timer is not None and self._sparse_timer.should_trigger_for_step( - stale_global_step + self._steps_per_run - ): + stale_global_step + self._steps_per_run): if global_step < 0: global_step = run_context.session.run(self._global_step_tensor) @@ -609,36 +646,42 @@ def _save(self, session, step): for x in save_data_offset: if x: data_offset_json.update(json.loads(x)) - save_offset_path = os.path.join(self._checkpoint_dir, 'model.ckpt-%d.offset' % step) + save_offset_path = os.path.join(self._checkpoint_dir, + 'model.ckpt-%d.offset' % step) with gfile.GFile(save_offset_path, 'w') as fout: json.dump(data_offset_json, fout) self._get_saver().save( - session, - self._save_path, - global_step=step, - write_meta_graph=self._write_graph, + session, + self._save_path, + global_step=step, + write_meta_graph=self._write_graph, ) self._summary_writer.add_session_log( - tf.SessionLog(status=tf.SessionLog.CHECKPOINT, checkpoint_path=self._save_path), - step, + tf.SessionLog( + status=tf.SessionLog.CHECKPOINT, checkpoint_path=self._save_path), + step, ) should_stop = False for l in self._listeners: # noqa: E741 if l.after_save(session, step): - logging.info('A CheckpointSaverListener requested that training be stopped. ' 'listener: {}'.format(l)) + logging.info( + 'A CheckpointSaverListener requested that training be stopped. ' + 'listener: {}'.format(l)) should_stop = True return should_stop def end(self, session): global_step = session.run(self._global_step_tensor) super(CheckpointSaverHook, self).end(session) - if self._dense_timer is not None and global_step != self._dense_timer.last_triggered_step(): + if self._dense_timer is not None and global_step != self._dense_timer.last_triggered_step( + ): self._dense_timer.update_last_triggered_step(global_step) self._send_dense(global_step, session) - if self._sparse_timer is not None and global_step != self._sparse_timer.last_triggered_step(): + if self._sparse_timer is not None and global_step != self._sparse_timer.last_triggered_step( + ): self._sparse_timer.update_last_triggered_step(global_step) self._send_sparse(global_step, session) @@ -672,8 +715,8 @@ def begin(self): assign_ops.append(var.assign(var_data)) else: logging.error( - 'variable [%s] shape not match %r vs %r' % (var.name.split(':')[0], var_shape, list(var_data.shape)) - ) + 'variable [%s] shape not match %r vs %r' % + (var.name.split(':')[0], var_shape, list(var_data.shape))) has_shape_unmatch = True elif 'Momentum' not in var_name and 'global_step' not in var_name: logging.error('variable [%s] not found in ckpt' % var_name) @@ -684,7 +727,8 @@ def begin(self): for var_name in sorted(vars_not_inited.keys()): f.write('%s:%s\n' % (var_name, vars_not_inited[var_name])) assert not has_shape_unmatch, 'exist variable shape not match, restore failed' - assert len(vars_not_inited.keys()) == 0, 'exist variable shape not inited, restore failed' + assert len(vars_not_inited.keys() + ) == 0, 'exist variable shape not inited, restore failed' def after_create_session(self, session, coord): assert self._restore_op is not None @@ -709,11 +753,13 @@ def __init__(self, incompatible_shape_var_map): def begin(self): assign_ops = [] for var, var_tmp in six.iteritems(self._incompatible_shape_var_map): - assign_ops.append(var.assign(shape_utils.pad_or_clip_nd(var_tmp, var.get_shape().as_list()))) + assign_ops.append( + var.assign( + shape_utils.pad_or_clip_nd(var_tmp, + var.get_shape().as_list()))) logging.info( - 'Assign variable[%s] from shape%s to shape%s' - % (var.name, var_tmp.get_shape().as_list(), var.get_shape().as_list()) - ) + 'Assign variable[%s] from shape%s to shape%s' % + (var.name, var_tmp.get_shape().as_list(), var.get_shape().as_list())) self._restore_op = tf.group(assign_ops) def after_create_session(self, session, coord): @@ -751,7 +797,9 @@ def begin(self): var_name = re.sub(':[0-9]$', '', var.name) if var_name in ckpt_var2shape_map: if restore_status[var_name]: - logging.warning('variable %s find in more than one checkpoint, skipped %s' % (var_name, ckpt_path)) + logging.warning( + 'variable %s find in more than one checkpoint, skipped %s' % + (var_name, ckpt_path)) continue name2var[var_name] = var restore_status[var_name] = True @@ -774,6 +822,7 @@ def after_create_session(self, session, coord): class OnlineEvaluationHook(SessionRunHook): + def __init__(self, metric_dict, output_dir): self._metric_dict = metric_dict self._output_dir = output_dir @@ -793,7 +842,8 @@ def end(self, session): self._summary_writer.add_summary(summary, global_step=global_step) self._summary_writer.flush() - eval_result_file = os.path.join(self._output_dir, 'online_eval_result.txt-%s' % global_step) + eval_result_file = os.path.join(self._output_dir, + 'online_eval_result.txt-%s' % global_step) logging.info('Saving online eval result to file %s' % eval_result_file) with gfile.GFile(eval_result_file, 'w') as ofile: result_to_write = {} @@ -852,7 +902,8 @@ def get_ckpt_version(ckpt_path): return int(toks[-1]) -def get_latest_checkpoint_from_checkpoint_path(checkpoint_path, ignore_ckpt_error): +def get_latest_checkpoint_from_checkpoint_path(checkpoint_path, + ignore_ckpt_error): ckpt_path = None if checkpoint_path.endswith('/') or gfile.IsDirectory(checkpoint_path + '/'): checkpoint_dir = checkpoint_path @@ -861,7 +912,9 @@ def get_latest_checkpoint_from_checkpoint_path(checkpoint_path, ignore_ckpt_erro if gfile.Exists(checkpoint_dir): ckpt_path = latest_checkpoint(checkpoint_dir) if ckpt_path: - logging.info('fine_tune_checkpoint is directory, will use the latest checkpoint: %s' % ckpt_path) + logging.info( + 'fine_tune_checkpoint is directory, will use the latest checkpoint: %s' + % ckpt_path) else: assert ignore_ckpt_error, 'fine_tune_checkpoint(%s) is not exists.' % checkpoint_path else: @@ -980,7 +1033,9 @@ def has_sok(): def init_hvd(): if hvd is None: - logging.error('horovod is not installed: HOROVOD_WITH_TENSORFLOW=1 pip install horovod') + logging.error( + 'horovod is not installed: HOROVOD_WITH_TENSORFLOW=1 pip install horovod' + ) sys.exit(1) hvd.init() diff --git a/easy_rec/python/utils/export_big_model.py b/easy_rec/python/utils/export_big_model.py index 1270a0920..243847a5f 100644 --- a/easy_rec/python/utils/export_big_model.py +++ b/easy_rec/python/utils/export_big_model.py @@ -12,30 +12,26 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.framework import ops from tensorflow.python.ops.variables import global_variables -from tensorflow.python.platform.gfile import ( # NOQA - DeleteRecursively, - Exists, - GFile, - Remove, -) +from tensorflow.python.platform.gfile import DeleteRecursively +from tensorflow.python.platform.gfile import Exists +from tensorflow.python.platform.gfile import GFile +from tensorflow.python.platform.gfile import Remove from tensorflow.python.saved_model import signature_constants from tensorflow.python.training.device_setter import replica_device_setter -from tensorflow.python.training.monitored_session import ( # NOQA - ChiefSessionCreator, - Scaffold, -) +from tensorflow.python.training.monitored_session import ChiefSessionCreator +from tensorflow.python.training.monitored_session import Scaffold from tensorflow.python.training.saver import export_meta_graph import easy_rec -from easy_rec.python.utils import constant, estimator_utils, io_util, proto_util # NOQA -from easy_rec.python.utils.meta_graph_editor import ( # NOQA - EMBEDDING_INITIALIZERS, - MetaGraphEditor, -) +from easy_rec.python.utils import constant +from easy_rec.python.utils import estimator_utils +from easy_rec.python.utils import io_util +from easy_rec.python.utils import proto_util +from easy_rec.python.utils.meta_graph_editor import EMBEDDING_INITIALIZERS +from easy_rec.python.utils.meta_graph_editor import MetaGraphEditor if tf.__version__ >= '2.0': from tensorflow.python.framework.ops import disable_eager_execution - disable_eager_execution() ConfigProto = config_pb2.ConfigProto @@ -44,15 +40,8 @@ INCR_UPDATE_SIGNATURE_KEY = 'incr_update_sig' -def export_big_model( - export_dir, - pipeline_config, - redis_params, - serving_input_fn, - estimator, - checkpoint_path, - verbose, -): +def export_big_model(export_dir, pipeline_config, redis_params, + serving_input_fn, estimator, checkpoint_path, verbose): for key in redis_params: logging.info('%s: %s' % (key, redis_params[key])) @@ -75,7 +64,8 @@ def export_big_model( logging.warning('load libwrite_sparse_kv.so failed: %s' % str(ex)) sparse_kv_module = None if not checkpoint_path: - checkpoint_path = estimator_utils.latest_checkpoint(pipeline_config.model_dir) + checkpoint_path = estimator_utils.latest_checkpoint( + pipeline_config.model_dir) logging.info('checkpoint_path = %s' % checkpoint_path) server = None @@ -85,7 +75,8 @@ def export_big_model( tf_config = estimator_utils.chief_to_master() if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = tf.train.Server(cluster, job_name='ps', task_index=tf_config['task']['index']) + server = tf.train.Server( + cluster, job_name='ps', task_index=tf_config['task']['index']) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: @@ -100,17 +91,22 @@ def export_big_model( if cluster: logging.info('cluster = ' + str(cluster)) - with tf.device(replica_device_setter(worker_device='/job:master/task:0', cluster=cluster)): - outputs = estimator._export_model_fn(features, None, None, estimator.params).predictions + with tf.device( + replica_device_setter( + worker_device='/job:master/task:0', cluster=cluster)): + outputs = estimator._export_model_fn(features, None, None, + estimator.params).predictions meta_graph_def = export_meta_graph() redis_embedding_version = redis_params.get('redis_embedding_version', '') if not redis_embedding_version: - meta_graph_def.meta_info_def.meta_graph_version = str(int(time.time())) + meta_graph_def.meta_info_def.meta_graph_version =\ + str(int(time.time())) else: meta_graph_def.meta_info_def.meta_graph_version = redis_embedding_version - logging.info('meta_graph_version = %s' % meta_graph_def.meta_info_def.meta_graph_version) + logging.info('meta_graph_version = %s' % + meta_graph_def.meta_info_def.meta_graph_version) embed_var_parts = {} embed_norm_name = {} @@ -124,9 +120,11 @@ def export_big_model( norm_name_to_ids[norm_name] = 1 tmp_export = x.export() if x.device not in embedding_vars: - embedding_vars[x.device] = [(norm_name, tmp_export.keys, tmp_export.values)] + embedding_vars[x.device] = [(norm_name, tmp_export.keys, + tmp_export.values)] else: - embedding_vars[x.device].append((norm_name, tmp_export.keys, tmp_export.values)) + embedding_vars[x.device].append( + (norm_name, tmp_export.keys, tmp_export.values)) elif '/embedding_weights:' in x.name or '/embedding_weights/part_' in x.name: norm_name, part_id = proto_util.get_norm_embed_name(x.name) norm_name_to_ids[norm_name] = 1 @@ -139,7 +137,10 @@ def export_big_model( for tid, t in enumerate(norm_name_to_ids.keys()): norm_name_to_ids[t] = str(tid) - is_cache_from_redis = [proto_util.is_cache_from_redis(x, redis_cache_names) for x in norm_name_to_ids] # noqa: F841 + is_cache_from_redis = [ # noqa: F841 + proto_util.is_cache_from_redis(x, redis_cache_names) + for x in norm_name_to_ids + ] for x in embed_norm_name: embed_norm_name[x] = norm_name_to_ids[embed_norm_name[x]] @@ -175,18 +176,17 @@ def export_big_model( tmp_names = [embed_norm_name[v] for v in tmp_vars] tmp_spos = [np.array(embed_spos[v], dtype=np.int64) for v in tmp_vars] write_kv_res = kv_module.write_kv( - tmp_names, - tmp_vars, - tmp_spos, - url=redis_url, - password=redis_passwd, - timeout=redis_params.get('redis_timeout', 1500), - version=meta_graph_def.meta_info_def.meta_graph_version, - threads=redis_params.get('redis_threads', 5), - batch_size=redis_params.get('redis_batch_size', 32), - expire=redis_params.get('redis_expire', 24), - verbose=verbose, - ) + tmp_names, + tmp_vars, + tmp_spos, + url=redis_url, + password=redis_passwd, + timeout=redis_params.get('redis_timeout', 1500), + version=meta_graph_def.meta_info_def.meta_graph_version, + threads=redis_params.get('redis_threads', 5), + batch_size=redis_params.get('redis_batch_size', 32), + expire=redis_params.get('redis_expire', 24), + verbose=verbose) all_write_res.append(write_kv_res) for tmp_dev in embedding_vars: @@ -196,52 +196,55 @@ def export_big_model( tmp_sparse_keys = [x[1] for x in tmp_vs] tmp_sparse_vals = [x[2] for x in tmp_vs] write_sparse_kv_res = sparse_kv_module.write_sparse_kv( - tmp_sparse_names, - tmp_sparse_vals, - tmp_sparse_keys, - url=redis_url, - password=redis_passwd, - timeout=redis_params.get('redis_timeout', 1500), - version=meta_graph_def.meta_info_def.meta_graph_version, - threads=redis_params.get('redis_threads', 5), - batch_size=redis_params.get('redis_batch_size', 32), - expire=redis_params.get('redis_expire', 24), - verbose=verbose, - ) + tmp_sparse_names, + tmp_sparse_vals, + tmp_sparse_keys, + url=redis_url, + password=redis_passwd, + timeout=redis_params.get('redis_timeout', 1500), + version=meta_graph_def.meta_info_def.meta_graph_version, + threads=redis_params.get('redis_threads', 5), + batch_size=redis_params.get('redis_batch_size', 32), + expire=redis_params.get('redis_expire', 24), + verbose=verbose) all_write_res.append(write_sparse_kv_res) - session_config = ConfigProto(allow_soft_placement=True, log_device_placement=False) + session_config = ConfigProto( + allow_soft_placement=True, log_device_placement=False) chief_sess_creator = ChiefSessionCreator( - master=server.target if server else '', - checkpoint_filename_with_path=checkpoint_path, - config=session_config, - ) - with tf.train.MonitoredSession(session_creator=chief_sess_creator, hooks=None, stop_grace_period_secs=120) as sess: + master=server.target if server else '', + checkpoint_filename_with_path=checkpoint_path, + config=session_config) + with tf.train.MonitoredSession( + session_creator=chief_sess_creator, + hooks=None, + stop_grace_period_secs=120) as sess: dump_flags = sess.run(all_write_res) logging.info('write embedding to redis succeed: %s' % str(dump_flags)) else: - logging.info('will skip write embedding to redis because ' 'redis_write_kv is set to 0.') + logging.info('will skip write embedding to redis because ' + 'redis_write_kv is set to 0.') # delete embedding_weights collections so that it could be re imported tmp_drop = [] for k in meta_graph_def.collection_def: v = meta_graph_def.collection_def[k] - if len(v.node_list.value) > 0 and 'embedding_weights' in v.node_list.value[0]: + if len( + v.node_list.value) > 0 and 'embedding_weights' in v.node_list.value[0]: tmp_drop.append(k) for k in tmp_drop: meta_graph_def.collection_def.pop(k) meta_graph_editor = MetaGraphEditor( - os.path.join(easy_rec.ops_dir, 'libembed_op.so'), - None, - redis_url, - redis_passwd, - redis_timeout=redis_params.get('redis_timeout', 600), - redis_cache_names=redis_cache_names, - meta_graph_def=meta_graph_def, - norm_name_to_ids=norm_name_to_ids, - debug_dir=export_dir if verbose else '', - ) + os.path.join(easy_rec.ops_dir, 'libembed_op.so'), + None, + redis_url, + redis_passwd, + redis_timeout=redis_params.get('redis_timeout', 600), + redis_cache_names=redis_cache_names, + meta_graph_def=meta_graph_def, + norm_name_to_ids=norm_name_to_ids, + debug_dir=export_dir if verbose else '') meta_graph_editor.edit_graph() tf.reset_default_graph() @@ -253,11 +256,12 @@ def export_big_model( for tmp_norm_name in norm_name_to_ids: fout.write('%s\t%s\n' % (tmp_norm_name, norm_name_to_ids[tmp_norm_name])) ops.add_to_collection( - tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt'), - ) + tf.GraphKeys.ASSET_FILEPATHS, + tf.constant( + embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt')) - export_dir = os.path.join(export_dir, meta_graph_def.meta_info_def.meta_graph_version) + export_dir = os.path.join(export_dir, + meta_graph_def.meta_info_def.meta_graph_version) export_dir = io_util.fix_oss_dir(export_dir) logging.info('export_dir=%s' % export_dir) if Exists(export_dir): @@ -268,34 +272,35 @@ def export_big_model( tensor_info_inputs = {} for tmp_key in inputs: tmp = graph.get_tensor_by_name(inputs[tmp_key].name) - tensor_info_inputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) + tensor_info_inputs[tmp_key] = \ + tf.saved_model.utils.build_tensor_info(tmp) tensor_info_outputs = {} for tmp_key in outputs: tmp = graph.get_tensor_by_name(outputs[tmp_key].name) - tensor_info_outputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) - signature = tf.saved_model.signature_def_utils.build_signature_def( - inputs=tensor_info_inputs, - outputs=tensor_info_outputs, - method_name=signature_constants.PREDICT_METHOD_NAME, - ) + tensor_info_outputs[tmp_key] = \ + tf.saved_model.utils.build_tensor_info(tmp) + signature = ( + tf.saved_model.signature_def_utils.build_signature_def( + inputs=tensor_info_inputs, + outputs=tensor_info_outputs, + method_name=signature_constants.PREDICT_METHOD_NAME)) - session_config = ConfigProto(allow_soft_placement=True, log_device_placement=True) + session_config = ConfigProto( + allow_soft_placement=True, log_device_placement=True) saver = tf.train.Saver() with tf.Session(target=server.target if server else '') as sess: saver.restore(sess, checkpoint_path) builder.add_meta_graph_and_variables( - sess, - [tf.saved_model.tag_constants.SERVING], - signature_def_map={ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature, - }, - assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), - saver=saver, - strip_default_attrs=True, - clear_devices=True, - ) + sess, [tf.saved_model.tag_constants.SERVING], + signature_def_map={ + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature, + }, + assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), + saver=saver, + strip_default_attrs=True, + clear_devices=True) builder.save() # remove temporary files @@ -303,15 +308,9 @@ def export_big_model( return export_dir -def export_big_model_to_oss( - export_dir, - pipeline_config, - oss_params, - serving_input_fn, - estimator, - checkpoint_path, - verbose, -): +def export_big_model_to_oss(export_dir, pipeline_config, oss_params, + serving_input_fn, estimator, checkpoint_path, + verbose): for key in oss_params: logging.info('%s: %s' % (key, oss_params[key])) @@ -319,7 +318,8 @@ def export_big_model_to_oss( kv_module = tf.load_op_library(write_kv_lib_path) if not checkpoint_path: - checkpoint_path = estimator_utils.latest_checkpoint(pipeline_config.model_dir) + checkpoint_path = estimator_utils.latest_checkpoint( + pipeline_config.model_dir) logging.info('checkpoint_path = %s' % checkpoint_path) server = None @@ -329,7 +329,8 @@ def export_big_model_to_oss( tf_config = estimator_utils.chief_to_master() if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) - server = tf.train.Server(cluster, job_name='ps', task_index=tf_config['task']['index']) + server = tf.train.Server( + cluster, job_name='ps', task_index=tf_config['task']['index']) server.join() elif tf_config['task']['type'] == 'master': if 'ps' in tf_config['cluster']: @@ -344,18 +345,23 @@ def export_big_model_to_oss( if cluster: logging.info('cluster = ' + str(cluster)) - with tf.device(replica_device_setter(worker_device='/job:master/task:0', cluster=cluster)): - outputs = estimator._export_model_fn(features, None, None, estimator.params).predictions + with tf.device( + replica_device_setter( + worker_device='/job:master/task:0', cluster=cluster)): + outputs = estimator._export_model_fn(features, None, None, + estimator.params).predictions meta_graph_def = export_meta_graph() meta_graph_def.meta_info_def.meta_graph_version = str(int(time.time())) oss_embedding_version = oss_params.get('oss_embedding_version', '') if not oss_embedding_version: - meta_graph_def.meta_info_def.meta_graph_version = str(int(time.time())) + meta_graph_def.meta_info_def.meta_graph_version =\ + str(int(time.time())) else: meta_graph_def.meta_info_def.meta_graph_version = oss_embedding_version - logging.info('meta_graph_version = %s' % meta_graph_def.meta_info_def.meta_graph_version) + logging.info('meta_graph_version = %s' % + meta_graph_def.meta_info_def.meta_graph_version) embed_var_parts = {} embed_norm_name = {} @@ -370,9 +376,11 @@ def export_big_model_to_oss( norm_name_to_ids[norm_name] = 1 tmp_export = x.export() if x.device not in embedding_vars: - embedding_vars[x.device] = [(norm_name, tmp_export.keys, tmp_export.values, part_id)] + embedding_vars[x.device] = [(norm_name, tmp_export.keys, + tmp_export.values, part_id)] else: - embedding_vars[x.device].append((norm_name, tmp_export.keys, tmp_export.values, part_id)) + embedding_vars[x.device].append( + (norm_name, tmp_export.keys, tmp_export.values, part_id)) elif '/embedding_weights:' in x.name or '/embedding_weights/part_' in x.name: norm_name, part_id = proto_util.get_norm_embed_name(x.name) norm_name_to_ids[norm_name] = 1 @@ -403,7 +411,8 @@ def export_big_model_to_oss( oss_endpoint = oss_params.get('oss_endpoint', '') oss_ak = oss_params.get('oss_ak', '') oss_sk = oss_params.get('oss_sk', '') - logging.info('will export to oss: %s %s %s %s', oss_path, oss_endpoint, oss_ak, oss_sk) + logging.info('will export to oss: %s %s %s %s', oss_path, oss_endpoint, + oss_ak, oss_sk) if oss_params.get('oss_write_kv', ''): # group embed by devices @@ -421,18 +430,17 @@ def export_big_model_to_oss( tmp_names = [embed_norm_name[v] for v in tmp_vars] tmp_spos = [np.array(embed_spos[v], dtype=np.int64) for v in tmp_vars] write_kv_res = kv_module.oss_write_kv( - tmp_names, - tmp_vars, - tmp_spos, - osspath=oss_path, - endpoint=oss_endpoint, - ak=oss_ak, - sk=oss_sk, - threads=oss_params.get('oss_threads', 5), - timeout=5, - expire=5, - verbose=verbose, - ) + tmp_names, + tmp_vars, + tmp_spos, + osspath=oss_path, + endpoint=oss_endpoint, + ak=oss_ak, + sk=oss_sk, + threads=oss_params.get('oss_threads', 5), + timeout=5, + expire=5, + verbose=verbose) all_write_res.append(write_kv_res) for tmp_dev in embedding_vars: @@ -443,54 +451,57 @@ def export_big_model_to_oss( tmp_sparse_vals = [x[2] for x in tmp_vs] tmp_part_ids = [x[3] for x in tmp_vs] write_sparse_kv_res = kv_module.oss_write_sparse_kv( - tmp_sparse_names, - tmp_sparse_vals, - tmp_sparse_keys, - tmp_part_ids, - osspath=oss_path, - endpoint=oss_endpoint, - ak=oss_ak, - sk=oss_sk, - version=meta_graph_def.meta_info_def.meta_graph_version, - threads=oss_params.get('oss_threads', 5), - verbose=verbose, - ) + tmp_sparse_names, + tmp_sparse_vals, + tmp_sparse_keys, + tmp_part_ids, + osspath=oss_path, + endpoint=oss_endpoint, + ak=oss_ak, + sk=oss_sk, + version=meta_graph_def.meta_info_def.meta_graph_version, + threads=oss_params.get('oss_threads', 5), + verbose=verbose) all_write_res.append(write_sparse_kv_res) - session_config = ConfigProto(allow_soft_placement=True, log_device_placement=False) + session_config = ConfigProto( + allow_soft_placement=True, log_device_placement=False) chief_sess_creator = ChiefSessionCreator( - master=server.target if server else '', - checkpoint_filename_with_path=checkpoint_path, - config=session_config, - ) - with tf.train.MonitoredSession(session_creator=chief_sess_creator, hooks=None, stop_grace_period_secs=120) as sess: + master=server.target if server else '', + checkpoint_filename_with_path=checkpoint_path, + config=session_config) + with tf.train.MonitoredSession( + session_creator=chief_sess_creator, + hooks=None, + stop_grace_period_secs=120) as sess: dump_flags = sess.run(all_write_res) logging.info('write embedding to oss succeed: %s' % str(dump_flags)) else: - logging.info('will skip write embedding to oss because ' 'oss_write_kv is set to 0.') + logging.info('will skip write embedding to oss because ' + 'oss_write_kv is set to 0.') # delete embedding_weights collections so that it could be re imported tmp_drop = [] for k in meta_graph_def.collection_def: v = meta_graph_def.collection_def[k] - if len(v.node_list.value) > 0 and 'embedding_weights' in v.node_list.value[0]: + if len( + v.node_list.value) > 0 and 'embedding_weights' in v.node_list.value[0]: tmp_drop.append(k) for k in tmp_drop: meta_graph_def.collection_def.pop(k) meta_graph_editor = MetaGraphEditor( - os.path.join(easy_rec.ops_dir, 'libembed_op.so'), - None, - oss_path=oss_path, - oss_endpoint=oss_endpoint, - oss_ak=oss_ak, - oss_sk=oss_sk, - oss_timeout=oss_params.get('oss_timeout', 1500), - meta_graph_def=meta_graph_def, - norm_name_to_ids=norm_name_to_ids, - incr_update_params=oss_params.get('incr_update', None), - debug_dir=export_dir if verbose else '', - ) + os.path.join(easy_rec.ops_dir, 'libembed_op.so'), + None, + oss_path=oss_path, + oss_endpoint=oss_endpoint, + oss_ak=oss_ak, + oss_sk=oss_sk, + oss_timeout=oss_params.get('oss_timeout', 1500), + meta_graph_def=meta_graph_def, + norm_name_to_ids=norm_name_to_ids, + incr_update_params=oss_params.get('incr_update', None), + debug_dir=export_dir if verbose else '') meta_graph_editor.edit_graph_for_oss() tf.reset_default_graph() @@ -502,20 +513,19 @@ def export_big_model_to_oss( for tmp_norm_name in norm_name_to_ids: fout.write('%s\t%s\n' % (tmp_norm_name, norm_name_to_ids[tmp_norm_name])) ops.add_to_collection( - ops.GraphKeys.ASSET_FILEPATHS, - tf.constant(embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt'), - ) + ops.GraphKeys.ASSET_FILEPATHS, + tf.constant( + embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt')) if 'incr_update' in oss_params: - dense_train_vars_path = os.path.join(os.path.dirname(checkpoint_path), constant.DENSE_UPDATE_VARIABLES) + dense_train_vars_path = os.path.join( + os.path.dirname(checkpoint_path), constant.DENSE_UPDATE_VARIABLES) ops.add_to_collection( - ops.GraphKeys.ASSET_FILEPATHS, - tf.constant( - dense_train_vars_path, - dtype=tf.string, - name=constant.DENSE_UPDATE_VARIABLES, - ), - ) + ops.GraphKeys.ASSET_FILEPATHS, + tf.constant( + dense_train_vars_path, + dtype=tf.string, + name=constant.DENSE_UPDATE_VARIABLES)) asset_file = 'incr_update.txt' asset_file_path = os.path.join(export_dir, asset_file) @@ -525,24 +535,24 @@ def export_big_model_to_oss( if 'kafka' in incr_update: incr_update_json['storage'] = 'kafka' incr_update_json['kafka'] = json.loads( - json_format.MessageToJson(incr_update['kafka'], preserving_proto_field_name=True) - ) + json_format.MessageToJson( + incr_update['kafka'], preserving_proto_field_name=True)) elif 'datahub' in incr_update: incr_update_json['storage'] = 'datahub' incr_update_json['datahub'] = json.loads( - json_format.MessageToJson(incr_update['datahub'], preserving_proto_field_name=True) - ) + json_format.MessageToJson( + incr_update['datahub'], preserving_proto_field_name=True)) elif 'fs' in incr_update: incr_update_json['storage'] = 'fs' incr_update_json['fs'] = {'incr_save_dir': incr_update['fs'].mount_path} json.dump(incr_update_json, fout, indent=2) ops.add_to_collection( - ops.GraphKeys.ASSET_FILEPATHS, - tf.constant(asset_file_path, dtype=tf.string, name=asset_file), - ) + ops.GraphKeys.ASSET_FILEPATHS, + tf.constant(asset_file_path, dtype=tf.string, name=asset_file)) - export_dir = os.path.join(export_dir, meta_graph_def.meta_info_def.meta_graph_version) + export_dir = os.path.join(export_dir, + meta_graph_def.meta_info_def.meta_graph_version) export_dir = io_util.fix_oss_dir(export_dir) logging.info('export_dir=%s' % export_dir) if Exists(export_dir): @@ -553,17 +563,19 @@ def export_big_model_to_oss( tensor_info_inputs = {} for tmp_key in inputs: tmp = graph.get_tensor_by_name(inputs[tmp_key].name) - tensor_info_inputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) + tensor_info_inputs[tmp_key] = \ + tf.saved_model.utils.build_tensor_info(tmp) tensor_info_outputs = {} for tmp_key in outputs: tmp = graph.get_tensor_by_name(outputs[tmp_key].name) - tensor_info_outputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) - signature = tf.saved_model.signature_def_utils.build_signature_def( - inputs=tensor_info_inputs, - outputs=tensor_info_outputs, - method_name=signature_constants.PREDICT_METHOD_NAME, - ) + tensor_info_outputs[tmp_key] = \ + tf.saved_model.utils.build_tensor_info(tmp) + signature = ( + tf.saved_model.signature_def_utils.build_signature_def( + inputs=tensor_info_inputs, + outputs=tensor_info_outputs, + method_name=signature_constants.PREDICT_METHOD_NAME)) if 'incr_update' in oss_params: incr_update_inputs = meta_graph_editor.sparse_update_inputs @@ -574,42 +586,43 @@ def export_big_model_to_oss( tensor_info_incr_update_outputs = {} for tmp_key in incr_update_inputs: tmp = graph.get_tensor_by_name(incr_update_inputs[tmp_key].name) - tensor_info_incr_update_inputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) + tensor_info_incr_update_inputs[tmp_key] = \ + tf.saved_model.utils.build_tensor_info(tmp) for tmp_key in incr_update_outputs: tmp = graph.get_tensor_by_name(incr_update_outputs[tmp_key].name) - tensor_info_incr_update_outputs[tmp_key] = tf.saved_model.utils.build_tensor_info(tmp) - incr_update_signature = tf.saved_model.signature_def_utils.build_signature_def( - inputs=tensor_info_incr_update_inputs, - outputs=tensor_info_incr_update_outputs, - method_name=signature_constants.PREDICT_METHOD_NAME, - ) + tensor_info_incr_update_outputs[tmp_key] = \ + tf.saved_model.utils.build_tensor_info(tmp) + incr_update_signature = ( + tf.saved_model.signature_def_utils.build_signature_def( + inputs=tensor_info_incr_update_inputs, + outputs=tensor_info_incr_update_outputs, + method_name=signature_constants.PREDICT_METHOD_NAME)) else: incr_update_signature = None - session_config = ConfigProto(allow_soft_placement=True, log_device_placement=True) + session_config = ConfigProto( + allow_soft_placement=True, log_device_placement=True) saver = tf.train.Saver() with tf.Session(target=server.target if server else '') as sess: saver.restore(sess, checkpoint_path) - main_op = tf.group( - [ + main_op = tf.group([ Scaffold.default_local_init_op(), - ops.get_collection(EMBEDDING_INITIALIZERS), - ] - ) - incr_update_sig_map = {signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature} + ops.get_collection(EMBEDDING_INITIALIZERS) + ]) + incr_update_sig_map = { + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature + } if incr_update_signature is not None: incr_update_sig_map[INCR_UPDATE_SIGNATURE_KEY] = incr_update_signature builder.add_meta_graph_and_variables( - sess, - [tf.saved_model.tag_constants.SERVING], - signature_def_map=incr_update_sig_map, - assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), - saver=saver, - main_op=main_op, - strip_default_attrs=True, - clear_devices=True, - ) + sess, [tf.saved_model.tag_constants.SERVING], + signature_def_map=incr_update_sig_map, + assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), + saver=saver, + main_op=main_op, + strip_default_attrs=True, + clear_devices=True) builder.save() # remove temporary files diff --git a/easy_rec/python/utils/expr_util.py b/easy_rec/python/utils/expr_util.py index ffd970ab7..c2178b566 100644 --- a/easy_rec/python/utils/expr_util.py +++ b/easy_rec/python/utils/expr_util.py @@ -8,8 +8,8 @@ def _process_multi_expr(expr): two_expr = ['>=', '<=', '=='] expr_list = [] while idx < size: - if idx + 2 <= size and expr[idx : idx + 2] in two_expr: - expr_list.append(expr[idx : idx + 2]) + if idx + 2 <= size and expr[idx:idx + 2] in two_expr: + expr_list.append(expr[idx:idx + 2]) idx += 2 else: expr_list.append(expr[idx]) @@ -25,7 +25,9 @@ def _process_enum(enum, input_names, prefix=''): def _get_expression_list(expression, input_names, prefix=''): - ops = ['+', '-', '*', '/', '(', ')', '>', '>=', '<', '<=', '==', '=', '&', '|'] + ops = [ + '+', '-', '*', '/', '(', ')', '>', '>=', '<', '<=', '==', '=', '&', '|' + ] expression_list = [] eunm = '' pre_expr = '' diff --git a/easy_rec/python/utils/fg_util.py b/easy_rec/python/utils/fg_util.py index 35036d01a..b4aa978dc 100644 --- a/easy_rec/python/utils/fg_util.py +++ b/easy_rec/python/utils/fg_util.py @@ -6,9 +6,9 @@ from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.protos.feature_config_pb2 import FeatureConfig from easy_rec.python.utils.config_util import get_compatible_feature_configs + from easy_rec.python.utils.convert_rtp_fg import ( # NOQA - load_input_field_and_feature_config, -) + load_input_field_and_feature_config,) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -27,7 +27,8 @@ def load_fg_json_to_config(pipeline_config): with tf.gfile.GFile(fg_json_path, 'r') as fin: rtp_fg = json.load(fin) - fg_config = load_input_field_and_feature_config(rtp_fg, label_fields=label_fields) + fg_config = load_input_field_and_feature_config( + rtp_fg, label_fields=label_fields) pipeline_config.data_config.ClearField('input_fields') pipeline_config.ClearField('feature_configs') diff --git a/easy_rec/python/utils/hit_rate_utils.py b/easy_rec/python/utils/hit_rate_utils.py index 258f7a5b3..a55cfa53c 100644 --- a/easy_rec/python/utils/hit_rate_utils.py +++ b/easy_rec/python/utils/hit_rate_utils.py @@ -27,15 +27,20 @@ def load_graph(i_emb_table, emb_dim, knn_metric, timeout, knn_strict): option.nlist = 5 option.nprobe = 2 g = gl.Graph().node( - i_emb_table, - node_type='i', - decoder=gl.Decoder(attr_types=['float'] * emb_dim, attr_delimiter=','), - option=option, + i_emb_table, + node_type='i', + decoder=gl.Decoder(attr_types=['float'] * emb_dim, attr_delimiter=','), + option=option, ) return g -def batch_hitrate(src_ids, recall_ids, recall_distances, gt_items, num_interests, mask=None): +def batch_hitrate(src_ids, + recall_ids, + recall_distances, + gt_items, + num_interests, + mask=None): """Compute hitrate of a batch of src ids. Args: @@ -104,32 +109,41 @@ def reduce_hitrate(cluster, hits, count, task_index): var_worker_count: variable used to mark the number of worker that have completed the calculation of hitrate. """ - with tf.device(tf.train.replica_device_setter(worker_device='/job:worker/task:%d' % task_index, cluster=cluster)): + with tf.device( + tf.train.replica_device_setter( + worker_device='/job:worker/task:%d' % task_index, cluster=cluster)): with tf.variable_scope('hitrate_var', reuse=tf.AUTO_REUSE): var_worker_count = tf.get_variable( - 'worker_count', - shape=(), - dtype=tf.int32, - initializer=tf.zeros_initializer(), + 'worker_count', + shape=(), + dtype=tf.int32, + initializer=tf.zeros_initializer(), ) - var_hits = tf.get_variable('hits', shape=(), dtype=tf.float32, initializer=tf.zeros_initializer()) + var_hits = tf.get_variable( + 'hits', + shape=(), + dtype=tf.float32, + initializer=tf.zeros_initializer()) var_gt_count = tf.get_variable( - 'gt_count', - shape=(), - dtype=tf.float32, - initializer=tf.zeros_initializer(), + 'gt_count', + shape=(), + dtype=tf.float32, + initializer=tf.zeros_initializer(), ) var_total_hitrate = tf.get_variable( - 'total_hitate', - shape=(), - dtype=tf.float32, - initializer=tf.zeros_initializer(), + 'total_hitate', + shape=(), + dtype=tf.float32, + initializer=tf.zeros_initializer(), ) var_hits = tf.assign_add(var_hits, hits, use_locking=True) var_gt_count = tf.assign_add(var_gt_count, count, use_locking=True) - var_gt_count = tf.Print(var_gt_count, [var_gt_count, var_hits], message='var_gt_count/var_hits') - var_total_hitrate = tf.assign(var_total_hitrate, var_hits / var_gt_count, use_locking=True) + var_gt_count = tf.Print( + var_gt_count, [var_gt_count, var_hits], + message='var_gt_count/var_hits') + var_total_hitrate = tf.assign( + var_total_hitrate, var_hits / var_gt_count, use_locking=True) with tf.control_dependencies([var_total_hitrate]): var_worker_count = tf.assign_add(var_worker_count, 1, use_locking=True) return var_total_hitrate, var_worker_count @@ -170,46 +184,53 @@ def _to_multi_float_attrs(x, userid): else: arr = [_to_float_attrs(sub_x) for sub_x in x.split('|')] assert len(arr) == num_interests, 'invalid arr len=%d, x=%s, userid=%s' % ( - len(arr), - x, - userid, + len(arr), + x, + userid, ) return arr src_ids = np.array([src_items[0] for src_items in gt_record]) - user_embedding = np.array([_to_multi_float_attrs(src_items[2], src_items[0]) for src_items in gt_record]) + user_embedding = np.array([ + _to_multi_float_attrs(src_items[2], src_items[0]) + for src_items in gt_record + ]) user_emb_num = [src_items[3] for src_items in gt_record] - print('max(user_emb_num) = %d len(src_ids) = %d' % (np.max(user_emb_num), len(src_ids))) + print('max(user_emb_num) = %d len(src_ids) = %d' % + (np.max(user_emb_num), len(src_ids))) # a list of list. - gt_items = [list(map(int, src_items[1].split(','))) for src_items in gt_record] + gt_items = [ + list(map(int, src_items[1].split(','))) for src_items in gt_record + ] logging.info('src_nodes.float_attrs.shape=%s' % str(user_embedding.shape)) user_embedding = user_embedding.reshape([-1, user_embedding.shape[-1]]) # numpy array - recall_ids, recall_distances = g.search('i', user_embedding, gl.KnnOption(k=top_k)) + recall_ids, recall_distances = g.search('i', user_embedding, + gl.KnnOption(k=top_k)) logging.info('recall_ids.shape=%s' % str(recall_ids.shape)) def _make_mask(lens): mask = np.ones([len(lens), num_interests], dtype=np.float32) for tmp_id, tmp_len in enumerate(lens): - mask[tmp_id, int(tmp_len) :] = 0 + mask[tmp_id, int(tmp_len):] = 0 return mask mask = _make_mask(user_emb_num) recall_ids = recall_ids.reshape([-1, num_interests, recall_ids.shape[-1]]) - recall_distances = recall_distances.reshape([-1, num_interests, recall_distances.shape[-1]]) + recall_distances = recall_distances.reshape( + [-1, num_interests, recall_distances.shape[-1]]) hitrates, bad_cases, bad_dists, hits, gt_count = batch_hitrate( - src_ids, recall_ids, recall_distances, gt_items, num_interests, mask - ) + src_ids, recall_ids, recall_distances, gt_items, num_interests, mask) return ( - hits, - gt_count, - src_ids, - recall_ids, - recall_distances, - hitrates, - bad_cases, - bad_dists, + hits, + gt_count, + src_ids, + recall_ids, + recall_distances, + hitrates, + bad_cases, + bad_dists, ) diff --git a/easy_rec/python/utils/hive_utils.py b/easy_rec/python/utils/hive_utils.py index 700ff3d03..9b348c9fc 100644 --- a/easy_rec/python/utils/hive_utils.py +++ b/easy_rec/python/utils/hive_utils.py @@ -9,6 +9,7 @@ class TableInfo(object): + def __init__(self, tablename, selected_cols, partition_kv, limit_num): self.tablename = tablename self.selected_cols = selected_cols @@ -38,13 +39,13 @@ class HiveUtils(object): """Common IO based interface, could run at local or on data science.""" def __init__( - self, - data_config, - hive_config, - selected_cols='', - record_defaults=[], - task_index=0, - task_num=1, + self, + data_config, + hive_config, + selected_cols='', + record_defaults=[], + task_index=0, + task_num=1, ): self._data_config = data_config self._hive_config = hive_config @@ -65,15 +66,16 @@ def _construct_table_info(self, table_name, limit_num): else: partition_kv = None - table_info = TableInfo(table_name, self._selected_cols, partition_kv, limit_num) + table_info = TableInfo(table_name, self._selected_cols, partition_kv, + limit_num) return table_info def _construct_hive_connect(self): conn = hive.Connection( - host=self._hive_config.host, - port=self._hive_config.port, - username=self._hive_config.username, - database=self._hive_config.database, + host=self._hive_config.host, + port=self._hive_config.port, + username=self._hive_config.username, + database=self._hive_config.database, ) return conn @@ -119,12 +121,15 @@ def run_sql(self, sql): data = [] return data - def is_table_or_partition_exist(self, table_name, partition_name=None, partition_val=None): + def is_table_or_partition_exist(self, + table_name, + partition_name=None, + partition_val=None): if partition_name and partition_val: sql = 'show partitions %s partition(%s=%s)' % ( - table_name, - partition_name, - partition_val, + table_name, + partition_name, + partition_val, ) try: res = self.run_sql(sql) @@ -174,7 +179,8 @@ def get_all_cols(self, input_path): for col in data: col_name = col[0].strip() - if col_name and (not col_name.startswith('#')) and (col_name not in col_names): + if col_name and (not col_name.startswith('#')) and (col_name + not in col_names): if col_name != pt_name: col_names.append(col_name) cols_types.append(col[1].strip()) diff --git a/easy_rec/python/utils/hpo_util.py b/easy_rec/python/utils/hpo_util.py index af684d910..ac446bf28 100644 --- a/easy_rec/python/utils/hpo_util.py +++ b/easy_rec/python/utils/hpo_util.py @@ -122,7 +122,9 @@ def kill_old_proc(tmp_dir, platform='pai'): if platform == 'emr': # clear easy_rec_hpo yarn jobs yarn_job_file = os.path.join(tmp_dir, 'yarn_job.txt') - os.system('yarn application -list | awk \'{ if ($2 == "easy_rec_hpo") print $1 }\' > %s' % yarn_job_file) + os.system( + 'yarn application -list | awk \'{ if ($2 == "easy_rec_hpo") print $1 }\' > %s' + % yarn_job_file) yarn_job_arr = [] with open(yarn_job_file, 'r') as fin: for line_str in fin: @@ -130,5 +132,6 @@ def kill_old_proc(tmp_dir, platform='pai'): yarn_job_arr.append(line_str) yarn_job_arr = list(set(yarn_job_arr)) if len(yarn_job_arr) > 0: - logging.info('will kill the easy_rec_hpo yarn jobs: %s' % ','.join(yarn_job_arr)) + logging.info('will kill the easy_rec_hpo yarn jobs: %s' % + ','.join(yarn_job_arr)) os.system('yarn application -kill %s' % ' '.join(yarn_job_arr)) diff --git a/easy_rec/python/utils/hvd_utils.py b/easy_rec/python/utils/hvd_utils.py index e0efd8c4c..223283486 100644 --- a/easy_rec/python/utils/hvd_utils.py +++ b/easy_rec/python/utils/hvd_utils.py @@ -46,7 +46,8 @@ def begin(self): # if '/embedding' not in x.name and 'DynamicVariable' not in str(type(x)): if x.name not in embed_para_vars: bcast_vars.append(x) - logging.info('will broadcast variable: name=%s shape=%s' % (x.name, x.get_shape())) + logging.info('will broadcast variable: name=%s shape=%s' % + (x.name, x.get_shape())) if not self.bcast_op or self.bcast_op.graph != tf.get_default_graph(): with tf.device(self.device): self.bcast_op = broadcast_variables(bcast_vars, self.root_rank) diff --git a/easy_rec/python/utils/input_utils.py b/easy_rec/python/utils/input_utils.py index b986d9d2b..427ff9280 100644 --- a/easy_rec/python/utils/input_utils.py +++ b/easy_rec/python/utils/input_utils.py @@ -12,12 +12,12 @@ def get_type_defaults(field_type, default_val=''): type_defaults = { - DatasetConfig.INT32: 0, - DatasetConfig.INT64: 0, - DatasetConfig.STRING: '', - DatasetConfig.BOOL: False, - DatasetConfig.FLOAT: 0.0, - DatasetConfig.DOUBLE: 0.0, + DatasetConfig.INT32: 0, + DatasetConfig.INT64: 0, + DatasetConfig.STRING: '', + DatasetConfig.BOOL: False, + DatasetConfig.FLOAT: 0.0, + DatasetConfig.DOUBLE: 0.0, } assert field_type in type_defaults, 'invalid type: %s' % field_type if default_val == '': @@ -54,15 +54,18 @@ def string_to_number(field, ftype, default_value, name=''): if ftype in [DatasetConfig.INT32, DatasetConfig.INT64]: # Int type is not supported in fg. # If you specify INT32, INT64 in DatasetConfig, you need to perform a cast at here. - tmp_field = tf.string_to_number(field, tf.double, name='field_as_flt_%s' % name) + tmp_field = tf.string_to_number( + field, tf.double, name='field_as_flt_%s' % name) if ftype in [DatasetConfig.INT64]: tmp_field = tf.cast(tmp_field, tf.int64) else: tmp_field = tf.cast(tmp_field, tf.int32) elif ftype in [DatasetConfig.FLOAT]: - tmp_field = tf.string_to_number(field, tf.float32, name='field_as_flt_%s' % name) + tmp_field = tf.string_to_number( + field, tf.float32, name='field_as_flt_%s' % name) elif ftype in [DatasetConfig.DOUBLE]: - tmp_field = tf.string_to_number(field, tf.float64, name='field_as_flt_%s' % name) + tmp_field = tf.string_to_number( + field, tf.float64, name='field_as_flt_%s' % name) elif ftype in [DatasetConfig.BOOL]: tmp_field = tf.logical_or(tf.equal(field, 'True'), tf.equal(field, 'true')) elif ftype in [DatasetConfig.STRING]: @@ -74,14 +77,14 @@ def string_to_number(field, ftype, default_value, name=''): def np_to_tf_type(np_type): _types_map = { - int: tf.int32, - np.int32: tf.int32, - np.int64: tf.int64, - str: tf.string, - np.float: tf.float32, - np.float32: tf.float32, - float: tf.float32, - np.double: tf.float64, + int: tf.int32, + np.int32: tf.int32, + np.int64: tf.int64, + str: tf.string, + np.float: tf.float32, + np.float32: tf.float32, + float: tf.float32, + np.double: tf.float64, } if np_type in _types_map: return _types_map[np_type] diff --git a/easy_rec/python/utils/io_util.py b/easy_rec/python/utils/io_util.py index 45eee96b5..9a5ede73c 100644 --- a/easy_rec/python/utils/io_util.py +++ b/easy_rec/python/utils/io_util.py @@ -81,9 +81,11 @@ def download(oss_or_url, dst_dir=''): response = urllib.request.urlopen(oss_or_url, timeout=HTTP_MAX_TIMEOUT) file_content = response.read() except Exception as e: - raise RuntimeError('Download %s failed: %s\n %s' % (oss_or_url, str(e), traceback.format_exc())) + raise RuntimeError('Download %s failed: %s\n %s' % + (oss_or_url, str(e), traceback.format_exc())) else: - tf.logging.warning('skip downloading %s, seems to be a local file' % oss_or_url) + tf.logging.warning('skip downloading %s, seems to be a local file' % + oss_or_url) return oss_or_url if dst_dir != '' and not os.path.exists(dst_dir): @@ -129,8 +131,10 @@ def download_and_uncompress_resource(resource_path, dst_dir=EASY_REC_RES_DIR): create_module_dir(dst_dir) _, basename = os.path.split(resource_path) - if not basename.endswith('.tar.gz') and not basename.endswith('.zip') and not basename.endswith('.py'): - raise ValueError('resource %s should be tar.gz or zip or py' % resource_path) + if not basename.endswith('.tar.gz') and not basename.endswith( + '.zip') and not basename.endswith('.py'): + raise ValueError('resource %s should be tar.gz or zip or py' % + resource_path) download(resource_path, dst_dir) @@ -141,7 +145,8 @@ def download_and_uncompress_resource(resource_path, dst_dir=EASY_REC_RES_DIR): stat, output = getstatusoutput('cd %s && unzip %s' % (dst_dir, basename)) if stat != 0: - raise ValueError('uncompress resoruce %s failed: %s' % resource_path, output) + raise ValueError('uncompress resoruce %s failed: %s' % resource_path, + output) return dst_dir @@ -207,11 +212,11 @@ def convert_tf_flags_to_argparse(flags): flag_type = type(default) help_str = flag.help or '' args[flag_name] = [ - False, - flag_type, - default, - help_str, - flag.choices if hasattr(flag, 'choices') else None, + False, + flag_type, + default, + help_str, + flag.choices if hasattr(flag, 'choices') else None, ] def str2bool(v): @@ -227,43 +232,46 @@ def str2bool(v): for flag_name, (multi, flag_type, default, help_str, choices) in args.items(): if flag_type == bool: parser.add_argument( - '--' + flag_name, - type=str2bool, - nargs='?', - const=True, - default=False, - help=help_str, + '--' + flag_name, + type=str2bool, + nargs='?', + const=True, + default=False, + help=help_str, ) elif flag_type == str: if choices: parser.add_argument( - '--' + flag_name, - type=str, - choices=choices, - default=default, - help=help_str, + '--' + flag_name, + type=str, + choices=choices, + default=default, + help=help_str, ) elif multi: parser.add_argument( - '--' + flag_name, - type=str, - action='append', - default=default, - help=help_str, + '--' + flag_name, + type=str, + action='append', + default=default, + help=help_str, ) else: - parser.add_argument('--' + flag_name, type=str, default=default, help=help_str) + parser.add_argument( + '--' + flag_name, type=str, default=default, help=help_str) elif flag_type in (list, dict): parser.add_argument( - '--' + flag_name, - type=lambda s: ast.literal_eval(s), - default=default, - help=help_str, + '--' + flag_name, + type=lambda s: ast.literal_eval(s), + default=default, + help=help_str, ) elif flag_type in (int, float): - parser.add_argument('--' + flag_name, type=flag_type, default=default, help=help_str) + parser.add_argument( + '--' + flag_name, type=flag_type, default=default, help=help_str) else: - parser.add_argument('--' + flag_name, type=str, default=default, help=help_str) + parser.add_argument( + '--' + flag_name, type=str, default=default, help=help_str) return parser diff --git a/easy_rec/python/utils/load_class.py b/easy_rec/python/utils/load_class.py index 1f89f3cc7..517af9bcc 100644 --- a/easy_rec/python/utils/load_class.py +++ b/easy_rec/python/utils/load_class.py @@ -51,6 +51,7 @@ def load_by_path(path): def _get_methods(aClass): + def should_track(func_name): return func_name == '__init__' or func_name[0] != '_' @@ -72,7 +73,8 @@ def _get_method_declare(aMethod): return sig_str else: spec = inspect.getargspec(aMethod) - args = inspect.formatargspec(spec.args, spec.varargs, spec.keywords, spec.defaults) + args = inspect.formatargspec(spec.args, spec.varargs, spec.keywords, + spec.defaults) return '%s%s' % (name, args) except TypeError: return '%s(cls, ...)' % name @@ -106,7 +108,8 @@ def check_class(cls, impl_cls, function_names=None): missing[name + '()'] = 'method signature differs' if len(missing) > 0: - raise Exception('incompatible Implementation-implementation %s: %s' % (impl_cls.__class__.__name__, missing)) + raise Exception('incompatible Implementation-implementation %s: %s' % + (impl_cls.__class__.__name__, missing)) def import_pkg(pkg_info, prefix_to_remove=None): @@ -156,8 +159,8 @@ def auto_import(user_path=None): """ # True False indicates import recursively or not pre_defined_dirs = [ - ('easy_rec/python/model', False), - ('easy_rec/python/input', False), + ('easy_rec/python/model', False), + ('easy_rec/python/input', False), ] parent_dir = easy_rec.parent_dir @@ -167,8 +170,8 @@ def auto_import(user_path=None): if parent_dir != '': for idx in range(len(pre_defined_dirs)): pre_defined_dirs[idx] = ( - os.path.join(parent_dir, pre_defined_dirs[idx][0]), - pre_defined_dirs[idx][1], + os.path.join(parent_dir, pre_defined_dirs[idx][0]), + pre_defined_dirs[idx][1], ) prefix_to_remove = parent_dir + '/' @@ -193,19 +196,19 @@ def auto_import(user_path=None): def register_class(class_map, class_name, cls): assert class_name not in class_map or class_map[class_name] == cls, ( - 'confilict class %s , %s is already register to be %s' - % ( - cls, - class_name, - str(class_map[class_name]), - ) - ) + 'confilict class %s , %s is already register to be %s' % ( + cls, + class_name, + str(class_map[class_name]), + )) logging.debug('register class %s' % class_name) class_map[class_name] = cls def get_register_class_meta(class_map, have_abstract_class=True): + class RegisterABCMeta(ABCMeta): + def __new__(mcs, name, bases, attrs): newclass = super(RegisterABCMeta, mcs).__new__(mcs, name, bases, attrs) register_class(class_map, name, newclass) @@ -215,7 +218,8 @@ def create_class(cls, name): if name in class_map: return class_map[name] else: - raise Exception('Class %s is not registered. Available ones are %s' % (name, list(class_map.keys()))) + raise Exception('Class %s is not registered. Available ones are %s' % + (name, list(class_map.keys()))) setattr(newclass, 'create_class', create_class) return newclass @@ -245,5 +249,6 @@ def load_keras_layer(name): return pydoc.locate(path), False except pydoc.ErrorDuringImport: print('load keras layer %s failed' % name) - logging.error('load keras layer %s failed: %s' % (name, traceback.format_exc())) + logging.error('load keras layer %s failed: %s' % + (name, traceback.format_exc())) return None, False diff --git a/easy_rec/python/utils/meta_graph_editor.py b/easy_rec/python/utils/meta_graph_editor.py index e4131bf73..2df87f299 100644 --- a/easy_rec/python/utils/meta_graph_editor.py +++ b/easy_rec/python/utils/meta_graph_editor.py @@ -7,39 +7,36 @@ from google.protobuf import text_format from tensorflow.python.framework import ops from tensorflow.python.platform.gfile import GFile - # from tensorflow.python.saved_model import constants from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model.loader_impl import SavedModelLoader from easy_rec.python.utils import ( # NOQA - conditional, - constant, - embedding_utils, - proto_util, + conditional, constant, embedding_utils, proto_util, ) EMBEDDING_INITIALIZERS = 'embedding_initializers' class MetaGraphEditor: + def __init__( - self, - lookup_lib_path, - saved_model_dir, - redis_url=None, - redis_passwd=None, - redis_timeout=0, - redis_cache_names=[], - oss_path=None, - oss_endpoint=None, - oss_ak=None, - oss_sk=None, - oss_timeout=0, - meta_graph_def=None, - norm_name_to_ids=None, - incr_update_params=None, - debug_dir='', + self, + lookup_lib_path, + saved_model_dir, + redis_url=None, + redis_passwd=None, + redis_timeout=0, + redis_cache_names=[], + oss_path=None, + oss_endpoint=None, + oss_ak=None, + oss_sk=None, + oss_timeout=0, + meta_graph_def=None, + norm_name_to_ids=None, + incr_update_params=None, + debug_dir='', ): self._lookup_op = tf.load_op_library(lookup_lib_path) self._debug_dir = debug_dir @@ -54,10 +51,12 @@ def __init__( tf.reset_default_graph() from tensorflow.python.framework import meta_graph - meta_graph.import_scoped_meta_graph_with_return_elements(meta_graph_def, clear_devices=True) + meta_graph.import_scoped_meta_graph_with_return_elements( + meta_graph_def, clear_devices=True) # tf.train.import_meta_graph(meta_graph_def) self._meta_graph_version = meta_graph_def.meta_info_def.meta_graph_version - self._signature_def = meta_graph_def.signature_def[signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] + self._signature_def = meta_graph_def.signature_def[ + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] if self._verbose: debug_out_path = os.path.join(self._debug_dir, 'meta_graph_raw.txt') @@ -166,13 +165,13 @@ def _get_share_embed_name(self, x, embed_names): embed_name_sub = '/'.join(tmp_toks) if tmp_name == embed_name_sub: assert not sel_embed_name, 'confusions encountered: %s %s' % ( - x, - ','.join(embed_names), + x, + ','.join(embed_names), ) sel_embed_name = embed_name assert sel_embed_name, '%s not find in shared_embeddings: %s' % ( - tmp_name, - ','.join(embed_names), + tmp_name, + ','.join(embed_names), ) return sel_embed_name @@ -187,27 +186,30 @@ def _find_embed_combiners(self, norm_embed_names): embed_combiners = {} embed_combine_node_cts = {} combiner_map = { - 'SparseSegmentSum': 'sum', - 'SparseSegmentMean': 'mean', - 'SparseSegmentSqrtN': 'sqrtn', + 'SparseSegmentSum': 'sum', + 'SparseSegmentMean': 'mean', + 'SparseSegmentSqrtN': 'sqrtn', } for node in self._meta_graph_def.graph_def.node: if node.op in combiner_map: norm_name, _ = proto_util.get_norm_embed_name(node.name) embed_combiners[norm_name] = combiner_map[node.op] - embed_combine_node_cts[norm_name] = embed_combine_node_cts.get(norm_name, 0) + 1 + embed_combine_node_cts[norm_name] = embed_combine_node_cts.get( + norm_name, 0) + 1 elif node.op == 'RealDiv' and len(node.input) == 2: # for tag feature with weights, and combiner == mean if 'SegmentSum' in node.input[0] and 'SegmentSum' in node.input[1]: norm_name, _ = proto_util.get_norm_embed_name(node.name) embed_combiners[norm_name] = 'mean' - embed_combine_node_cts[norm_name] = embed_combine_node_cts.get(norm_name, 0) + 1 + embed_combine_node_cts[norm_name] = embed_combine_node_cts.get( + norm_name, 0) + 1 elif node.op == 'SegmentSum': norm_name, _ = proto_util.get_norm_embed_name(node.name) # avoid overwrite RealDiv results if norm_name not in embed_combiners: embed_combiners[norm_name] = 'sum' - embed_combine_node_cts[norm_name] = embed_combine_node_cts.get(norm_name, 0) + 1 + embed_combine_node_cts[norm_name] = embed_combine_node_cts.get( + norm_name, 0) + 1 return [embed_combiners[x] for x in norm_embed_names] def _find_lookup_indices_values_shapes(self): @@ -235,7 +237,8 @@ def _get_output_shape(graph_def, input_name): # embed_name, _ = proto_util.get_norm_embed_name(node.name, self._verbose) fea_name, _ = proto_util.get_norm_embed_name(node.name, self._verbose) for tmp_input in node.input: - tmp_shape = _get_output_shape(self._meta_graph_def.graph_def, tmp_input) + tmp_shape = _get_output_shape(self._meta_graph_def.graph_def, + tmp_input) if '_embedding_weights/Cast' in tmp_input: continue elif len(tmp_shape.dim) == 2: @@ -278,8 +281,8 @@ def _find_embed_names_and_dims(self, norm_embed_names): embed_is_kv = {} for node in self._meta_graph_def.graph_def.node: if 'embedding_weights' in node.name and node.op in [ - 'VariableV2', - 'KvVarHandleOp', + 'VariableV2', + 'KvVarHandleOp', ]: tmp = node.attr['shape'].shape.dim[-1].size tmp2 = 1 @@ -319,7 +322,8 @@ def find_lookup_inputs(self): weights = self._find_lookup_weights() for fea in shapes.keys(): - logging.info('Lookup Input[%s]: indices=%s values=%s shapes=%s' % (fea, indices[fea], values[fea], shapes[fea])) + logging.info('Lookup Input[%s]: indices=%s values=%s shapes=%s' % + (fea, indices[fea], values[fea], shapes[fea])) graph = tf.get_default_graph() @@ -348,35 +352,42 @@ def _get_tensor_by_name(tensor_name): # get embedding dimensions ( - self._embed_names, - self._embed_dims, - self._embed_sizes, - self._embed_is_kv, + self._embed_names, + self._embed_dims, + self._embed_sizes, + self._embed_is_kv, ) = self._find_embed_names_and_dims(values.keys()) if not self._embed_name_to_ids: embed_name_uniq = list(set(self._embed_names)) - self._embed_name_to_ids = {t: tid for tid, t in enumerate(embed_name_uniq)} - self._embed_ids = [int(self._embed_name_to_ids[x]) for x in self._embed_names] + self._embed_name_to_ids = { + t: tid for tid, t in enumerate(embed_name_uniq) + } + self._embed_ids = [ + int(self._embed_name_to_ids[x]) for x in self._embed_names + ] - self._is_cache_from_redis = [proto_util.is_cache_from_redis(x, self._redis_cache_names) for x in self._embed_names] + self._is_cache_from_redis = [ + proto_util.is_cache_from_redis(x, self._redis_cache_names) + for x in self._embed_names + ] # normalized feature names self._feature_names = list(values.keys()) return ( + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + ) + + def add_lookup_op( + self, lookup_input_indices, lookup_input_values, lookup_input_shapes, lookup_input_weights, - ) - - def add_lookup_op( - self, - lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights, ): logging.info('add custom lookup operation to lookup embeddings from redis') self._lookup_outs = [None for i in range(len(lookup_input_values))] @@ -386,18 +397,18 @@ def add_lookup_op( for i in range(len(self._lookup_outs)): i_1 = i + 1 self._lookup_outs[i] = self._lookup_op.kv_lookup( - lookup_input_indices[i:i_1], - lookup_input_values[i:i_1], - lookup_input_shapes[i:i_1], - lookup_input_weights[i:i_1], - url=self._redis_url, - password=self._redis_passwd, - timeout=self._redis_timeout, - combiners=self._embed_combiners[i:i_1], - embedding_dims=self._embed_dims[i:i_1], - embedding_names=self._embed_ids[i:i_1], - cache=self._is_cache_from_redis, - version=self._meta_graph_version, + lookup_input_indices[i:i_1], + lookup_input_values[i:i_1], + lookup_input_shapes[i:i_1], + lookup_input_weights[i:i_1], + url=self._redis_url, + password=self._redis_passwd, + timeout=self._redis_timeout, + combiners=self._embed_combiners[i:i_1], + embedding_dims=self._embed_dims[i:i_1], + embedding_names=self._embed_ids[i:i_1], + cache=self._is_cache_from_redis, + version=self._meta_graph_version, )[0] meta_graph_def = tf.train.export_meta_graph() @@ -405,15 +416,17 @@ def add_lookup_op( if self._verbose: debug_path = os.path.join(self._debug_dir, 'graph_raw.txt') with GFile(debug_path, 'w') as fout: - fout.write(text_format.MessageToString(self._meta_graph_def.graph_def, as_utf8=True)) + fout.write( + text_format.MessageToString( + self._meta_graph_def.graph_def, as_utf8=True)) return meta_graph_def def add_oss_lookup_op( - self, - lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights, + self, + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, ): logging.info('add custom lookup operation to lookup embeddings from oss') place_on_cpu = os.getenv('place_embedding_on_cpu') @@ -443,21 +456,21 @@ def add_oss_lookup_op( # shared_name='embedding_lookup_res', # name='embedding_lookup_fused/lookup')[0] self._lookup_outs = self._lookup_op.oss_read_kv( - lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights, - osspath=self._oss_path, - endpoint=self._oss_endpoint, - ak=self._oss_ak, - sk=self._oss_sk, - timeout=self._oss_timeout, - combiners=self._embed_combiners, - embedding_dims=self._embed_dims, - embedding_ids=self._embed_ids, - embedding_is_kv=self._embed_is_kv, - shared_name='embedding_lookup_res', - name='embedding_lookup_fused/lookup', + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + osspath=self._oss_path, + endpoint=self._oss_endpoint, + ak=self._oss_ak, + sk=self._oss_sk, + timeout=self._oss_timeout, + combiners=self._embed_combiners, + embedding_dims=self._embed_dims, + embedding_ids=self._embed_ids, + embedding_is_kv=self._embed_is_kv, + shared_name='embedding_lookup_res', + name='embedding_lookup_fused/lookup', ) N = np.max([int(x) for x in self._embed_ids]) + 1 @@ -466,24 +479,24 @@ def add_oss_lookup_op( uniq_embed_combiners = ['mean' for x in range(N)] uniq_embed_is_kvs = [0 for x in range(N)] for embed_id, embed_combiner, embed_is_kv, embed_dim in zip( - self._embed_ids, self._embed_combiners, self._embed_is_kv, self._embed_dims - ): + self._embed_ids, self._embed_combiners, self._embed_is_kv, + self._embed_dims): uniq_embed_combiners[embed_id] = embed_combiner uniq_embed_is_kvs[embed_id] = embed_is_kv uniq_embed_dims[embed_id] = embed_dim lookup_init_op = self._lookup_op.oss_init( - osspath=self._oss_path, - endpoint=self._oss_endpoint, - ak=self._oss_ak, - sk=self._oss_sk, - combiners=uniq_embed_combiners, - embedding_dims=uniq_embed_dims, - embedding_ids=uniq_embed_ids, - embedding_is_kv=uniq_embed_is_kvs, - N=N, - shared_name='embedding_lookup_res', - name='embedding_lookup_fused/init', + osspath=self._oss_path, + endpoint=self._oss_endpoint, + ak=self._oss_ak, + sk=self._oss_sk, + combiners=uniq_embed_combiners, + embedding_dims=uniq_embed_dims, + embedding_ids=uniq_embed_ids, + embedding_is_kv=uniq_embed_is_kvs, + N=N, + shared_name='embedding_lookup_res', + name='embedding_lookup_fused/init', ) ops.add_to_collection(EMBEDDING_INITIALIZERS, lookup_init_op) @@ -492,12 +505,13 @@ def add_oss_lookup_op( # all sparse variables are updated by a single custom operation message_ph = tf.placeholder(tf.int8, [None], name='incr_update/message') embedding_update = self._lookup_op.embedding_update( - message=message_ph, - shared_name='embedding_lookup_res', - name='embedding_lookup_fused/embedding_update', + message=message_ph, + shared_name='embedding_lookup_res', + name='embedding_lookup_fused/embedding_update', ) self._embedding_update_inputs['incr_update/sparse/message'] = message_ph - self._embedding_update_outputs['incr_update/sparse/embedding_update'] = embedding_update + self._embedding_update_outputs[ + 'incr_update/sparse/embedding_update'] = embedding_update # dense variables are updated one by one dense_name_to_ids = embedding_utils.get_dense_name_to_ids() @@ -505,7 +519,8 @@ def add_oss_lookup_op( dense_var_id = dense_name_to_ids[x.op.name] dense_input_name = 'incr_update/dense/%d/input' % dense_var_id dense_output_name = 'incr_update/dense/%d/output' % dense_var_id - dense_update_input = tf.placeholder(tf.float32, x.get_shape(), name=dense_input_name) + dense_update_input = tf.placeholder( + tf.float32, x.get_shape(), name=dense_input_name) self._dense_update_inputs[dense_input_name] = dense_update_input dense_assign_op = tf.assign(x, dense_update_input) self._dense_update_outputs[dense_output_name] = dense_assign_op @@ -515,7 +530,9 @@ def add_oss_lookup_op( if self._verbose: debug_path = os.path.join(self._debug_dir, 'graph_raw.txt') with GFile(debug_path, 'w') as fout: - fout.write(text_format.MessageToString(self._meta_graph_def.graph_def, as_utf8=True)) + fout.write( + text_format.MessageToString( + self._meta_graph_def.graph_def, as_utf8=True)) return meta_graph_def def bytes2str(self, x): @@ -533,13 +550,14 @@ def clear_meta_graph_embeding(self, meta_graph_def): def _clear_embedding_in_meta_collect(meta_graph_def, collect_name): tmp_vals = [ - x - for x in meta_graph_def.collection_def[collect_name].bytes_list.value - if 'embedding_weights' not in self.bytes2str(x) + x + for x in meta_graph_def.collection_def[collect_name].bytes_list.value + if 'embedding_weights' not in self.bytes2str(x) ] meta_graph_def.collection_def[collect_name].bytes_list.ClearField('value') for tmp_v in tmp_vals: - meta_graph_def.collection_def[collect_name].bytes_list.value.append(tmp_v) + meta_graph_def.collection_def[collect_name].bytes_list.value.append( + tmp_v) _clear_embedding_in_meta_collect(meta_graph_def, 'model_variables') _clear_embedding_in_meta_collect(meta_graph_def, 'trainable_variables') @@ -547,17 +565,15 @@ def _clear_embedding_in_meta_collect(meta_graph_def, collect_name): # clear Kv(pai embedding variable) ops in meta_info_def.stripped_op_list.op kept_ops = [ - x - for x in meta_graph_def.meta_info_def.stripped_op_list.op - if x.name - not in [ - 'InitializeKvVariableOp', - 'KvResourceGather', - 'KvResourceImportV2', - 'KvVarHandleOp', - 'KvVarIsInitializedOp', - 'ReadKvVariableOp', - ] + x for x in meta_graph_def.meta_info_def.stripped_op_list.op + if x.name not in [ + 'InitializeKvVariableOp', + 'KvResourceGather', + 'KvResourceImportV2', + 'KvVarHandleOp', + 'KvVarIsInitializedOp', + 'ReadKvVariableOp', + ] ] meta_graph_def.meta_info_def.stripped_op_list.ClearField('op') meta_graph_def.meta_info_def.stripped_op_list.op.extend(kept_ops) @@ -573,7 +589,8 @@ def clear_meta_collect(self, meta_graph_def): for key in meta_graph_def.collection_def: val = meta_graph_def.collection_def[key] if val.HasField('node_list'): - if 'embedding_weights' in val.node_list.value[0] and 'easy_rec' not in val.node_list.value[0]: + if 'embedding_weights' in val.node_list.value[ + 0] and 'easy_rec' not in val.node_list.value[0]: drop_meta_collects.append(key) elif key == 'saved_model_assets': drop_meta_collects.append(key) @@ -581,6 +598,7 @@ def clear_meta_collect(self, meta_graph_def): meta_graph_def.collection_def.pop(key) def remove_embedding_weights_and_update_lookup_outputs(self): + def _should_drop(name): if '_embedding_weights' in name: if self._verbose: @@ -588,7 +606,9 @@ def _should_drop(name): return True logging.info('remove embedding_weights node in graph_def.node') - logging.info('and replace the old embedding_lookup outputs with new lookup_op outputs') + logging.info( + 'and replace the old embedding_lookup outputs with new lookup_op outputs' + ) for tid, node in enumerate(self._all_graph_nodes): # drop the nodes @@ -597,16 +617,20 @@ def _should_drop(name): else: for i in range(len(node.input)): if _should_drop(node.input[i]): - input_name, _ = proto_util.get_norm_embed_name(node.input[i], self._verbose) + input_name, _ = proto_util.get_norm_embed_name( + node.input[i], self._verbose) print('REPLACE:' + node.input[i] + '=>' + input_name) - input_name = self._lookup_outs[self._feature_names.index(input_name)].name + input_name = self._lookup_outs[self._feature_names.index( + input_name)].name if input_name.endswith(':0'): input_name = input_name.replace(':0', '') node.input[i] = input_name # drop by ids def _drop_by_ids(self, tmp_obj, key, drop_ids): - keep_vals = [x for i, x in enumerate(getattr(tmp_obj, key)) if i not in drop_ids] + keep_vals = [ + x for i, x in enumerate(getattr(tmp_obj, key)) if i not in drop_ids + ] tmp_obj.ClearField(key) getattr(tmp_obj, key).extend(keep_vals) @@ -629,19 +653,26 @@ def clear_save_restore(self): if self._restore_tensor_node: drop_ids = [] - for tmp_id, tmp_name in enumerate(self._restore_tensor_node.attr['value'].tensor.string_val): + for tmp_id, tmp_name in enumerate( + self._restore_tensor_node.attr['value'].tensor.string_val): if 'embedding_weights' in self.bytes2str(tmp_name): drop_ids.append(tmp_id) - self._drop_by_ids(self._restore_tensor_node.attr['value'].tensor, 'string_val', drop_ids) - keep_node_num = len(self._restore_tensor_node.attr['value'].tensor.string_val) + self._drop_by_ids(self._restore_tensor_node.attr['value'].tensor, + 'string_val', drop_ids) + keep_node_num = len( + self._restore_tensor_node.attr['value'].tensor.string_val) logging.info( - 'update self._restore_tensor_node: string_val keep_num = %d drop_num = %d' % (keep_node_num, len(drop_ids)) - ) - self._restore_tensor_node.attr['value'].tensor.tensor_shape.dim[0].size = keep_node_num - self._restore_tensor_node.attr['_output_shapes'].list.shape[0].dim[0].size = keep_node_num - - logging.info('update save/RestoreV2, drop tensor_shapes, _output_shapes, related to embedding_weights') + 'update self._restore_tensor_node: string_val keep_num = %d drop_num = %d' + % (keep_node_num, len(drop_ids))) + self._restore_tensor_node.attr['value'].tensor.tensor_shape.dim[ + 0].size = keep_node_num + self._restore_tensor_node.attr['_output_shapes'].list.shape[0].dim[ + 0].size = keep_node_num + + logging.info( + 'update save/RestoreV2, drop tensor_shapes, _output_shapes, related to embedding_weights' + ) self._restore_shard_node = None for node_id, node in enumerate(self._all_graph_nodes): if not self._all_graph_node_flags[tid]: @@ -659,14 +690,17 @@ def clear_save_restore(self): self._restore_all_node.append(node) def clear_save_assign(self): - logging.info('update save/Assign, drop tensor_shapes, _output_shapes, related to embedding_weights') + logging.info( + 'update save/Assign, drop tensor_shapes, _output_shapes, related to embedding_weights' + ) # edit save/Assign drop_save_assigns = [] all_kv_drop = [] for tid, node in enumerate(self._all_graph_nodes): if not self._all_graph_node_flags[tid]: continue - if node.op == 'Assign' and 'save/Assign' in node.name and 'embedding_weights' in node.input[0]: + if node.op == 'Assign' and 'save/Assign' in node.name and 'embedding_weights' in node.input[ + 0]: drop_save_assigns.append('^' + node.name) self._all_graph_node_flags[tid] = False elif 'embedding_weights/ConcatPartitions/concat' in node.name: @@ -692,16 +726,18 @@ def clear_save_assign(self): # update node(save/Assign_[0-N])'s input[1] by the position of # node.input[0] in save/RestoreV2/tensor_names # the outputs of save/RestoreV2 is connected to save/Assign - tmp_id = [self.bytes2str(x) for x in self._restore_tensor_node.attr['value'].tensor.string_val].index( - node.input[0] - ) + tmp_id = [ + self.bytes2str(x) + for x in self._restore_tensor_node.attr['value'].tensor.string_val + ].index(node.input[0]) if tmp_id != 0: tmp_input2 = 'save/RestoreV2:%d' % tmp_id else: tmp_input2 = 'save/RestoreV2' if tmp_input2 != node.input[1]: if self._verbose: - logging.info("update save/Assign[%s]'s input from %s to %s" % (node.name, node.input[1], tmp_input2)) + logging.info("update save/Assign[%s]'s input from %s to %s" % + (node.name, node.input[1], tmp_input2)) node.input[1] = tmp_input2 # save/restore_all need save/restore_shard as input @@ -749,19 +785,22 @@ def clear_save_v2(self): for node in self._all_graph_nodes: if node.name == 'save/SaveV2/shape_and_slices' and node.op == 'Const': # _output_shapes # size # string_val - node.attr['_output_shapes'].list.shape[0].dim[0].size -= len(save_drop_ids) + node.attr['_output_shapes'].list.shape[0].dim[0].size -= len( + save_drop_ids) node.attr['value'].tensor.tensor_shape.dim[0].size -= len(save_drop_ids) - self._drop_by_ids(node.attr['value'].tensor, 'string_val', save_drop_ids) + self._drop_by_ids(node.attr['value'].tensor, 'string_val', + save_drop_ids) elif node.name == 'save/SaveV2/tensor_names': # tensor_names may not have the same order as save/SaveV2/shape_and_slices tmp_drop_ids = [ - tmp_id - for tmp_id, tmp_val in enumerate(node.attr['value'].tensor.string_val) - if 'embedding_weights' in self.bytes2str(tmp_val) + tmp_id for tmp_id, tmp_val in enumerate( + node.attr['value'].tensor.string_val) + if 'embedding_weights' in self.bytes2str(tmp_val) ] # attr['value'].tensor.string_val # tensor_shape # size assert len(save_drop_ids) == len(save_drop_ids) - node.attr['_output_shapes'].list.shape[0].dim[0].size -= len(tmp_drop_ids) + node.attr['_output_shapes'].list.shape[0].dim[0].size -= len( + tmp_drop_ids) node.attr['value'].tensor.tensor_shape.dim[0].size -= len(tmp_drop_ids) self._drop_by_ids(node.attr['value'].tensor, 'string_val', tmp_drop_ids) @@ -784,13 +823,14 @@ def clear_initialize(self): self._all_graph_node_flags[tid] = False elif 'embedding_weights' in node.name and node.op == 'VariableV2': self._all_graph_node_flags[tid] = False - elif 'embedding_weights' in node.name and node.name.endswith('/read') and node.op == 'Identity': + elif 'embedding_weights' in node.name and node.name.endswith( + '/read') and node.op == 'Identity': self._all_graph_node_flags[tid] = False elif 'embedding_weights' in node.name and node.op == 'Identity': node_toks = node.name.split('/') node_tok = node_toks[-1] if 'embedding_weights_' in node_tok: - node_tok = node_tok[len('embedding_weights_') :] + node_tok = node_tok[len('embedding_weights_'):] try: int(node_tok) self._all_graph_node_flags[tid] = False @@ -802,13 +842,17 @@ def clear_embedding_variable(self): for tid, node in enumerate(self._all_graph_nodes): if not self._all_graph_node_flags[tid]: continue - if node.op in ['ReadKvVariableOp', 'KvVarIsInitializedOp', 'KvVarHandleOp']: + if node.op in [ + 'ReadKvVariableOp', 'KvVarIsInitializedOp', 'KvVarHandleOp' + ]: self._all_graph_node_flags[tid] = False # there maybe some nodes depend on the dropped nodes, they are dropped as well def drop_dependent_nodes(self): drop_names = [ - tmp_node.name for tid, tmp_node in enumerate(self._all_graph_nodes) if not self._all_graph_node_flags[tid] + tmp_node.name + for tid, tmp_node in enumerate(self._all_graph_nodes) + if not self._all_graph_node_flags[tid] ] while True: more_drop_names = [] @@ -816,7 +860,8 @@ def drop_dependent_nodes(self): if not self._all_graph_node_flags[tid]: continue if len(tmp_node.input) > 0 and tmp_node.input[0] in drop_names: - logging.info('drop dependent node: %s depend on %s' % (tmp_node.name, tmp_node.input[0])) + logging.info('drop dependent node: %s depend on %s' % + (tmp_node.name, tmp_node.input[0])) self._all_graph_node_flags[tid] = False more_drop_names.append(tmp_node.name) drop_names = more_drop_names @@ -826,18 +871,18 @@ def drop_dependent_nodes(self): def edit_graph(self): # the main entrance ( - lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights, + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, ) = self.find_lookup_inputs() # add lookup op to the graph self._meta_graph_def = self.add_lookup_op( - lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights, + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, ) self.clear_meta_graph_embeding(self._meta_graph_def) @@ -864,9 +909,10 @@ def edit_graph(self): self.drop_dependent_nodes() self._meta_graph_def.graph_def.ClearField('node') - self._meta_graph_def.graph_def.node.extend( - [x for tid, x in enumerate(self._all_graph_nodes) if self._all_graph_node_flags[tid]] - ) + self._meta_graph_def.graph_def.node.extend([ + x for tid, x in enumerate(self._all_graph_nodes) + if self._all_graph_node_flags[tid] + ]) logging.info('old node number = %d' % self._old_node_num) logging.info('node number = %d' % len(self._meta_graph_def.graph_def.node)) @@ -877,23 +923,24 @@ def edit_graph(self): fout.write(text_format.MessageToString(self.graph_def, as_utf8=True)) debug_dump_path = os.path.join(self._debug_dir, 'meta_graph.txt') with GFile(debug_dump_path, 'w') as fout: - fout.write(text_format.MessageToString(self._meta_graph_def, as_utf8=True)) + fout.write( + text_format.MessageToString(self._meta_graph_def, as_utf8=True)) def edit_graph_for_oss(self): # the main entrance ( - lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights, + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, ) = self.find_lookup_inputs() # add lookup op to the graph self._meta_graph_def = self.add_oss_lookup_op( - lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights, + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, ) self.clear_meta_graph_embeding(self._meta_graph_def) @@ -920,9 +967,10 @@ def edit_graph_for_oss(self): self.drop_dependent_nodes() self._meta_graph_def.graph_def.ClearField('node') - self._meta_graph_def.graph_def.node.extend( - [x for tid, x in enumerate(self._all_graph_nodes) if self._all_graph_node_flags[tid]] - ) + self._meta_graph_def.graph_def.node.extend([ + x for tid, x in enumerate(self._all_graph_nodes) + if self._all_graph_node_flags[tid] + ]) logging.info('old node number = %d' % self._old_node_num) logging.info('node number = %d' % len(self._meta_graph_def.graph_def.node)) @@ -933,4 +981,5 @@ def edit_graph_for_oss(self): fout.write(text_format.MessageToString(self.graph_def, as_utf8=True)) debug_dump_path = os.path.join(self._debug_dir, 'meta_graph.txt') with GFile(debug_dump_path, 'w') as fout: - fout.write(text_format.MessageToString(self._meta_graph_def, as_utf8=True)) + fout.write( + text_format.MessageToString(self._meta_graph_def, as_utf8=True)) diff --git a/easy_rec/python/utils/multi_optimizer.py b/easy_rec/python/utils/multi_optimizer.py index 0c2b5ad28..c34c4abe0 100644 --- a/easy_rec/python/utils/multi_optimizer.py +++ b/easy_rec/python/utils/multi_optimizer.py @@ -6,6 +6,7 @@ class MultiOptimizer(optimizer.Optimizer): + def __init__(self, opts, grouped_vars, use_locking=False): """Combine multiple optimizers for optimization, such as WideAndDeep. @@ -22,7 +23,8 @@ def __init__(self, opts, grouped_vars, use_locking=False): def compute_gradients(self, loss, variables, **kwargs): grad_and_vars = [] for gid, opt in enumerate(self._opts): - grad_and_vars.extend(opt.compute_gradients(loss, self._grouped_vars[gid], **kwargs)) + grad_and_vars.extend( + opt.compute_gradients(loss, self._grouped_vars[gid], **kwargs)) return grad_and_vars def apply_gradients(self, grads_and_vars, global_step=None, name=None): diff --git a/easy_rec/python/utils/odps_util.py b/easy_rec/python/utils/odps_util.py index c6ef3cb30..de2d25d00 100644 --- a/easy_rec/python/utils/odps_util.py +++ b/easy_rec/python/utils/odps_util.py @@ -10,9 +10,9 @@ def is_type_compatiable(odps_type, input_type): """Check that odps_type are compatiable with input_type.""" type_map = { - 'bigint': DatasetConfig.INT64, - 'string': DatasetConfig.STRING, - 'double': DatasetConfig.DOUBLE, + 'bigint': DatasetConfig.INT64, + 'string': DatasetConfig.STRING, + 'double': DatasetConfig.DOUBLE, } tmp_type = type_map[odps_type] if tmp_type == input_type: @@ -31,9 +31,9 @@ def is_type_compatiable(odps_type, input_type): def odps_type_to_input_type(odps_type): """Check that odps_type are compatiable with input_type.""" odps_type_map = { - 'bigint': DatasetConfig.INT64, - 'string': DatasetConfig.STRING, - 'double': DatasetConfig.DOUBLE, + 'bigint': DatasetConfig.INT64, + 'string': DatasetConfig.STRING, + 'double': DatasetConfig.DOUBLE, } assert odps_type in odps_type_map, 'only support [bigint, string, double]' input_type = odps_type_map[odps_type] @@ -65,13 +65,12 @@ def check_input_field_and_types(data_config): for x, y in zip(input_fields, input_field_types): tmp_type = type_map[x] assert is_type_compatiable(tmp_type, y), ( - 'feature[%s] type error: odps %s is not compatible with input_type %s' - % ( - x, - tmp_type, - DatasetConfig.FieldType.Name(y), - ) - ) + 'feature[%s] type error: odps %s is not compatible with input_type %s' + % ( + x, + tmp_type, + DatasetConfig.FieldType.Name(y), + )) def odps_type_2_tf_type(odps_type): diff --git a/easy_rec/python/utils/pai_util.py b/easy_rec/python/utils/pai_util.py index 26c5a4ce5..538d280e3 100644 --- a/easy_rec/python/utils/pai_util.py +++ b/easy_rec/python/utils/pai_util.py @@ -55,8 +55,8 @@ def process_config(configs, task_index=0, worker_num=1): configs = configs.split(',') if len(configs) > 1: assert len(configs) == worker_num, ( - 'number of configs must be equal to number of workers,' + ' when number of configs > 1' - ) + 'number of configs must be equal to number of workers,' + + ' when number of configs > 1') config = configs[task_index] else: config = configs[0] @@ -76,7 +76,9 @@ def process_config(configs, task_index=0, worker_num=1): def test(): - f = download('https://easy-rec.oss-cn-hangzhou.aliyuncs.com/config/MultiTower/dwd_avazu_ctr_deepmodel.config') + f = download( + 'https://easy-rec.oss-cn-hangzhou.aliyuncs.com/config/MultiTower/dwd_avazu_ctr_deepmodel.config' + ) assert f == 'dwd_avazu_ctr_deepmodel.config' diff --git a/easy_rec/python/utils/proto_util.py b/easy_rec/python/utils/proto_util.py index f3b6efa08..c7b3275b3 100644 --- a/easy_rec/python/utils/proto_util.py +++ b/easy_rec/python/utils/proto_util.py @@ -31,13 +31,14 @@ def get_norm_embed_name(name, verbose=False): for i in range(0, len(name_toks) - 1): if name_toks[i + 1].startswith('embedding_weights:'): var_id = name_toks[i + 1].replace('embedding_weights:', '') - tmp_name = '/'.join(name_toks[: i + 1]) + tmp_name = '/'.join(name_toks[:i + 1]) if var_id != '0': tmp_name = tmp_name + '_' + var_id if verbose: logging.info('norm %s to %s' % (name, tmp_name)) return tmp_name, 0 - if i > 1 and name_toks[i + 1].startswith('part_') and name_toks[i] == 'embedding_weights': + if i > 1 and name_toks[i + 1].startswith( + 'part_') and name_toks[i] == 'embedding_weights': tmp_name = '/'.join(name_toks[:i]) part_id = name_toks[i + 1].replace('part_', '') part_toks = part_id.split(':') @@ -50,8 +51,9 @@ def get_norm_embed_name(name, verbose=False): # input_layer/app_category_embedding/app_category_embedding_weights/SparseReshape # => input_layer/app_category_embedding for i in range(0, len(name_toks) - 1): - if name_toks[i + 1].endswith('_embedding_weights') or '_embedding_weights_' in name_toks[i + 1]: - tmp_name = '/'.join(name_toks[: i + 1]) + if name_toks[i + 1].endswith( + '_embedding_weights') or '_embedding_weights_' in name_toks[i + 1]: + tmp_name = '/'.join(name_toks[:i + 1]) if verbose: logging.info('norm %s to %s' % (name, tmp_name)) return tmp_name, 0 @@ -59,7 +61,7 @@ def get_norm_embed_name(name, verbose=False): # => input_layer/app_category_embedding for i in range(0, len(name_toks) - 1): if name_toks[i + 1] == 'embedding_weights': - tmp_name = '/'.join(name_toks[: i + 1]) + tmp_name = '/'.join(name_toks[:i + 1]) if verbose: logging.info('norm %s to %s' % (name, tmp_name)) return tmp_name, 0 diff --git a/easy_rec/python/utils/restore_filter.py b/easy_rec/python/utils/restore_filter.py index 734b76bff..b11934725 100644 --- a/easy_rec/python/utils/restore_filter.py +++ b/easy_rec/python/utils/restore_filter.py @@ -2,7 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Define filters for restore.""" -from abc import ABCMeta, abstractmethod +from abc import ABCMeta +from abc import abstractmethod from enum import Enum @@ -31,6 +32,7 @@ def keep(self, var_name): class KeywordFilter(Filter): + def __init__(self, pattern, exclusive=False): """Init KeywordFilter. @@ -50,6 +52,7 @@ def keep(self, var_name): class CombineFilter(Filter): + def __init__(self, filters, logical=Logical.AND): """Init CombineFilter. diff --git a/easy_rec/python/utils/shape_utils.py b/easy_rec/python/utils/shape_utils.py index 24a202215..d26b434a9 100644 --- a/easy_rec/python/utils/shape_utils.py +++ b/easy_rec/python/utils/shape_utils.py @@ -15,7 +15,9 @@ # ============================================================================== """Utils used to manipulate tensor shapes.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import six import tensorflow as tf @@ -62,7 +64,8 @@ def merge_shape(t, shape_list): the tensor t with shape updated """ t_shape = t.get_shape().as_list() - assert len(shape_list) == len(t_shape), 'input shape size should be the same of the tensor' + assert len(shape_list) == len( + t_shape), 'input shape size should be the same of the tensor' for idx, size in enumerate(shape_list): if size is not None: t_shape[idx] = size @@ -88,9 +91,9 @@ def pad_tensor(t, length): t_d0 = t_shape[0] pad_d0 = tf.expand_dims(length - t_d0, 0) pad_shape = tf.cond( - tf.greater(t_rank, 1), - lambda: tf.concat([pad_d0, t_shape[1:]], 0), - lambda: tf.expand_dims(length - t_d0, 0), + tf.greater(t_rank, 1), + lambda: tf.concat([pad_d0, t_shape[1:]], 0), + lambda: tf.expand_dims(length - t_d0, 0), ) padded_t = tf.concat([t, tf.zeros(pad_shape, dtype=t.dtype)], 0) if not _is_tensor(length): @@ -146,10 +149,17 @@ def pad_nd(tensor, output_shape): """ tensor_shape = tf.shape(tensor) - trailing_paddings = [shape - tensor_shape[i] if shape is not None else 0 for i, shape in enumerate(output_shape)] - paddings = tf.stack([tf.zeros(len(trailing_paddings), dtype=tf.int32), trailing_paddings], axis=1) + trailing_paddings = [ + shape - tensor_shape[i] if shape is not None else 0 + for i, shape in enumerate(output_shape) + ] + paddings = tf.stack( + [tf.zeros(len(trailing_paddings), dtype=tf.int32), trailing_paddings], + axis=1) padded_tensor = tf.pad(tensor, paddings=paddings) - output_static_shape = [dim if not isinstance(dim, tf.Tensor) else None for dim in output_shape] + output_static_shape = [ + dim if not isinstance(dim, tf.Tensor) else None for dim in output_shape + ] padded_tensor.set_shape(output_static_shape) return padded_tensor @@ -167,10 +177,11 @@ def pad_or_clip_nd(tensor, output_shape): """ tensor_shape = tf.shape(tensor) clip_size = [ - tf.where(tensor_shape[i] - shape > 0, shape, -1) if shape is not None else -1 - for i, shape in enumerate(output_shape) + tf.where(tensor_shape[i] - shape > 0, shape, -1) + if shape is not None else -1 for i, shape in enumerate(output_shape) ] - clipped_tensor = tf.slice(tensor, begin=tf.zeros(len(clip_size), dtype=tf.int32), size=clip_size) + clipped_tensor = tf.slice( + tensor, begin=tf.zeros(len(clip_size), dtype=tf.int32), size=clip_size) # Pad tensor if the shape of clipped tensor is smaller than the expected # shape. @@ -225,19 +236,19 @@ def check_min_image_dim(min_dim, image_tensor): image_width = static_shape.get_width(image_shape) if image_height is None or image_width is None: shape_assert = tf.Assert( - tf.logical_and( - tf.greater_equal(tf.shape(image_tensor)[1], min_dim), - tf.greater_equal(tf.shape(image_tensor)[2], min_dim), - ), - ['image size must be >= {} in both height and width.'.format(min_dim)], + tf.logical_and( + tf.greater_equal(tf.shape(image_tensor)[1], min_dim), + tf.greater_equal(tf.shape(image_tensor)[2], min_dim), + ), + ['image size must be >= {} in both height and width.'.format(min_dim)], ) with tf.control_dependencies([shape_assert]): return tf.identity(image_tensor) if image_height < min_dim or image_width < min_dim: raise ValueError( - 'image size must be >= %d in both height and width; image dim = %d,%d' % (min_dim, image_height, image_width) - ) + 'image size must be >= %d in both height and width; image dim = %d,%d' % + (min_dim, image_height, image_width)) return image_tensor @@ -262,7 +273,8 @@ def assert_shape_equal(shape_a, shape_b): Raises: ValueError: When shapes are both static and unequal. """ - if all(isinstance(dim, int) for dim in shape_a) and all(isinstance(dim, int) for dim in shape_b): + if all(isinstance(dim, int) for dim in shape_a) and all( + isinstance(dim, int) for dim in shape_b): if shape_a != shape_b: raise ValueError('Unequal shapes {}, {}'.format(shape_a, shape_b)) else: @@ -293,7 +305,8 @@ def assert_shape_equal_along_first_dimension(shape_a, shape_b): """ if isinstance(shape_a[0], int) and isinstance(shape_b[0], int): if shape_a[0] != shape_b[0]: - raise ValueError('Unequal first dimension {}, {}'.format(shape_a[0], shape_b[0])) + raise ValueError('Unequal first dimension {}, {}'.format( + shape_a[0], shape_b[0])) else: return tf.no_op() else: @@ -317,11 +330,11 @@ def assert_box_normalized(boxes, maximum_normalized_coordinate=1.1): box_minimum = tf.reduce_min(boxes) box_maximum = tf.reduce_max(boxes) return tf.Assert( - tf.logical_and( - tf.less_equal(box_maximum, maximum_normalized_coordinate), - tf.greater_equal(box_minimum, 0), - ), - [boxes], + tf.logical_and( + tf.less_equal(box_maximum, maximum_normalized_coordinate), + tf.greater_equal(box_minimum, 0), + ), + [boxes], ) @@ -380,19 +393,20 @@ def assert_rank(tensor, expected_rank, name=None): if actual_rank not in expected_rank_dict: scope_name = tf.get_variable_scope().name raise ValueError( - 'For the tensor `%s` in scope `%s`, the actual rank ' - '`%d` (shape = %s) is not equal to the expected rank `%s`' - % (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank)) - ) + 'For the tensor `%s` in scope `%s`, the actual rank ' + '`%d` (shape = %s) is not equal to the expected rank `%s`' % + (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank))) def truncate_sequence(seq_emb, seq_len, limited_len): + def truncate(seq_embed, seq_length): - seq_embed = tf.slice(seq_embed, [0, 0, 0], [shape[0], limited_len, shape[2]]) + seq_embed = tf.slice(seq_embed, [0, 0, 0], + [shape[0], limited_len, shape[2]]) seq_length = tf.where( - tf.greater(seq_length, limited_len), - tf.ones_like(seq_length) * limited_len, - seq_length, + tf.greater(seq_length, limited_len), + tf.ones_like(seq_length) * limited_len, + seq_length, ) return seq_embed, seq_length @@ -403,9 +417,9 @@ def keep(seq_embed, seq_length): max_seq_len = shape[1] return tf.cond( - max_seq_len > limited_len, - lambda: truncate(seq_emb, seq_len), - lambda: keep(seq_emb, seq_len), + max_seq_len > limited_len, + lambda: truncate(seq_emb, seq_len), + lambda: keep(seq_emb, seq_len), ) @@ -419,10 +433,13 @@ def padding(): def truncate(): sliced = tf.slice(seq_emb, [0, 0, 0], [-1, fixed_len, -1]) - length = tf.where(seq_len < fixed_len, seq_len, tf.ones_like(seq_len) * fixed_len) if seq_len is not None else None + length = tf.where(seq_len < fixed_len, seq_len, + tf.ones_like(seq_len) * + fixed_len) if seq_len is not None else None return sliced, length def keep(): return seq_emb, seq_len - return tf.cond(padding_length > 0, padding, lambda: tf.cond(padding_length < 0, truncate, keep)) + return tf.cond(padding_length > 0, padding, + lambda: tf.cond(padding_length < 0, truncate, keep)) diff --git a/easy_rec/python/utils/test_utils.py b/easy_rec/python/utils/test_utils.py index 069844354..42349f90d 100644 --- a/easy_rec/python/utils/test_utils.py +++ b/easy_rec/python/utils/test_utils.py @@ -37,7 +37,8 @@ def get_hdfs_tmp_dir(test_dir): """Create a randomly of directory in HDFS.""" - tmp_name = ''.join([random.choice(string.ascii_letters + string.digits) for i in range(8)]) + tmp_name = ''.join( + [random.choice(string.ascii_letters + string.digits) for i in range(8)]) assert isinstance(test_dir, str) test_rand_dir = os.path.join(test_dir, tmp_name) gfile.MkDir(test_rand_dir) @@ -49,7 +50,8 @@ def proc_wait(proc, timeout=1200): while proc.poll() is None and time.time() - t0 < timeout: time.sleep(1) if proc.poll() is None: - logging.warning('proc[pid=%d] timeout[%d], will kill the proc' % (proc.pid, timeout)) + logging.warning('proc[pid=%d] timeout[%d], will kill the proc' % + (proc.pid, timeout)) proc.terminate() while proc.poll() is None: time.sleep(1) @@ -58,7 +60,9 @@ def proc_wait(proc, timeout=1200): def get_tmp_dir(): max_retry = 5 while max_retry > 0: - tmp_name = ''.join([random.choice(string.ascii_letters + string.digits) for i in range(12)]) + tmp_name = ''.join([ + random.choice(string.ascii_letters + string.digits) for i in range(12) + ]) if os.environ.get('TEST_DIR', '') != '': global TEST_DIR TEST_DIR = os.environ['TEST_DIR'] @@ -98,7 +102,8 @@ def run_cmd(cmd_str, log_file, env=None): cmd_str = cmd_str.replace('\r', ' ').replace('\n', ' ') logging.info('RUNCMD: %s > %s 2>&1 ' % (cmd_str, log_file)) with open(log_file, 'w') as lfile: - proc = subprocess.Popen(cmd_str, stdout=lfile, stderr=subprocess.STDOUT, shell=True, env=env) + proc = subprocess.Popen( + cmd_str, stdout=lfile, stderr=subprocess.STDOUT, shell=True, env=env) if six.PY2: # for debug purpose proc.args = cmd_str @@ -157,8 +162,12 @@ def _replace_data_for_test(data_path): return data_path -def _load_config_for_test(pipeline_config_path, test_dir, total_steps=50, num_epochs=0): - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) +def _load_config_for_test(pipeline_config_path, + test_dir, + total_steps=50, + num_epochs=0): + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path) train_config = pipeline_config.train_config eval_config = pipeline_config.eval_config data_config = pipeline_config.data_config @@ -173,19 +182,20 @@ def _load_config_for_test(pipeline_config_path, test_dir, total_steps=50, num_ep def _load_config_for_distribute_eval(pipeline_config_path, test_dir): - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path) pipeline_config.model_dir = test_dir logging.info('test_model_dir %s' % pipeline_config.model_dir) return pipeline_config def test_datahub_train_eval( - pipeline_config_path, - odps_oss_config, - test_dir, - process_pipeline_func=None, - total_steps=50, - post_check_func=None, + pipeline_config_path, + odps_oss_config, + test_dir, + process_pipeline_func=None, + total_steps=50, + post_check_func=None, ): gpus = get_available_gpus() if len(gpus) > 0: @@ -201,7 +211,8 @@ def test_datahub_train_eval( if isinstance(pipeline_config_path, EasyRecConfig): pipeline_config = pipeline_config_path else: - pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, total_steps) + pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, + total_steps) pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -229,8 +240,8 @@ def test_datahub_train_eval( proc_wait(proc, timeout=TEST_TIME_OUT) if proc.returncode != 0: logging.warning( - 'train %s failed[pid=%d][code=%d][args=%s]' % (test_pipeline_config_path, proc.pid, proc.returncode, proc.args) - ) + 'train %s failed[pid=%d][code=%d][args=%s]' % + (test_pipeline_config_path, proc.pid, proc.returncode, proc.args)) return False if post_check_func: return post_check_func(pipeline_config) @@ -238,21 +249,22 @@ def test_datahub_train_eval( def _Load_config_for_test_eval(pipeline_config_path): - pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config_path) + pipeline_config = config_util.get_configs_from_pipeline_file( + pipeline_config_path) return pipeline_config def test_single_train_eval( - pipeline_config_path, - test_dir, - process_pipeline_func=None, - hyperparam_str='', - total_steps=50, - post_check_func=None, - check_mode=False, - fine_tune_checkpoint=None, - extra_cmd_args=None, - timeout=-1, + pipeline_config_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='', + total_steps=50, + post_check_func=None, + check_mode=False, + fine_tune_checkpoint=None, + extra_cmd_args=None, + timeout=-1, ): gpus = get_available_gpus() if len(gpus) > 0: @@ -268,7 +280,8 @@ def test_single_train_eval( if isinstance(pipeline_config_path, EasyRecConfig): pipeline_config = pipeline_config_path else: - pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, total_steps) + pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, + total_steps) pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -321,7 +334,8 @@ def test_single_pre_check(pipeline_config_path, test_dir): config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') - train_cmd = 'python -m easy_rec.python.tools.pre_check --pipeline_config_path %s ' % (test_pipeline_config_path) + train_cmd = 'python -m easy_rec.python.tools.pre_check --pipeline_config_path %s ' % ( + test_pipeline_config_path) proc = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, 'master')) proc_wait(proc, timeout=TEST_TIME_OUT) @@ -339,9 +353,9 @@ def test_single_predict(test_dir, input_path, output_path, saved_model_dir): set_gpu_id(None) predict_cmd = 'python -m easy_rec.python.predict --input_path %s --output_path %s --saved_model_dir %s' % ( - input_path, - output_path, - saved_model_dir, + input_path, + output_path, + saved_model_dir, ) proc = run_cmd(predict_cmd, '%s/log_%s.txt' % (test_dir, 'master')) @@ -356,10 +370,9 @@ def test_feature_selection(pipeline_config): model_dir = pipeline_config.model_dir pipeline_config_path = os.path.join(model_dir, 'pipeline.config') output_dir = os.path.join(model_dir, 'feature_selection') - cmd = ( - 'python -m easy_rec.python.tools.feature_selection --config_path %s ' - '--output_dir %s --topk 5 --visualize true' % (pipeline_config_path, output_dir) - ) + cmd = ('python -m easy_rec.python.tools.feature_selection --config_path %s ' + '--output_dir %s --topk 5 --visualize true' % + (pipeline_config_path, output_dir)) proc = run_cmd(cmd, os.path.join(model_dir, 'log_feature_selection.txt')) proc_wait(proc, timeout=TEST_TIME_OUT) if proc.returncode != 0: @@ -369,21 +382,22 @@ def test_feature_selection(pipeline_config): def yaml_replace( - train_yaml_path, - pipline_config_path, - test_pipeline_config_path, - test_export_dir=None, + train_yaml_path, + pipline_config_path, + test_pipeline_config_path, + test_export_dir=None, ): with open(train_yaml_path, 'r', encoding='utf-8') as _file: sample = _file.read() x = yaml.load(sample) _command = x['app']['command'] if test_export_dir is not None: - _command = _command.replace(pipline_config_path, test_pipeline_config_path).replace( - '{EXPOERT_DIR}', test_export_dir - ) + _command = _command.replace(pipline_config_path, + test_pipeline_config_path).replace( + '{EXPOERT_DIR}', test_export_dir) else: - _command = _command.replace(pipline_config_path, test_pipeline_config_path) + _command = _command.replace(pipline_config_path, + test_pipeline_config_path) x['app']['command'] = _command with open(train_yaml_path, 'w', encoding='utf-8') as _file: @@ -391,12 +405,12 @@ def yaml_replace( def test_hdfs_train_eval( - pipeline_config_path, - train_yaml_path, - test_dir, - process_pipeline_func=None, - hyperparam_str='', - total_steps=2000, + pipeline_config_path, + train_yaml_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='', + total_steps=2000, ): gpus = get_available_gpus() if len(gpus) > 0: @@ -407,7 +421,8 @@ def test_hdfs_train_eval( logging.info('train_yaml_path %s' % train_yaml_path) if 'TF_CONFIG' in os.environ: del os.environ['TF_CONFIG'] - pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, total_steps) + pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, + total_steps) logging.info('model_dir in pipeline_config has been modified') pipeline_config.train_config.train_distribute = 0 pipeline_config.train_config.num_gpus_per_worker = 1 @@ -429,11 +444,11 @@ def test_hdfs_train_eval( def test_hdfs_eval( - pipeline_config_path, - eval_yaml_path, - test_dir, - process_pipeline_func=None, - hyperparam_str='', + pipeline_config_path, + eval_yaml_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='', ): gpus = get_available_gpus() if len(gpus) > 0: @@ -462,11 +477,11 @@ def test_hdfs_eval( def test_hdfs_export( - pipeline_config_path, - export_yaml_path, - test_dir, - process_pipeline_func=None, - hyperparam_str='', + pipeline_config_path, + export_yaml_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='', ): gpus = get_available_gpus() if len(gpus) > 0: @@ -485,10 +500,10 @@ def test_hdfs_export( test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') test_export_path = os.path.join(test_dir, 'export_dir') yaml_replace( - export_yaml_path, - pipeline_config_path, - test_pipeline_config_path, - test_export_path, + export_yaml_path, + pipeline_config_path, + test_pipeline_config_path, + test_export_path, ) logging.info('test_pipeline_config_path is %s' % test_pipeline_config_path) eval_cmd = 'el_submit -yaml %s' % export_yaml_path @@ -527,8 +542,8 @@ def _get_ports(num_worker): ports = os.environ['ports'] port_arr = [int(x) for x in ports.split(',')] assert len(port_arr) >= num_worker, 'not enough ports: %s, required: %d' % ( - ports, - num_worker, + ports, + num_worker, ) return port_arr[:num_worker] else: @@ -536,12 +551,12 @@ def _get_ports(num_worker): def _ps_worker_train( - pipeline_config_path, - test_dir, - num_worker, - num_evaluator=0, - fit_on_eval=False, - fit_on_eval_steps=None, + pipeline_config_path, + test_dir, + num_worker, + num_evaluator=0, + fit_on_eval=False, + fit_on_eval_steps=None, ): gpus = get_available_gpus() # not enough gpus, run on cpu only @@ -550,9 +565,9 @@ def _ps_worker_train( ports = _get_ports(num_worker + 1) chief_or_master = 'master' if num_evaluator == 0 else 'chief' cluster = { - chief_or_master: ['localhost:%d' % ports[0]], - 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], - 'ps': ['localhost:%d' % ports[-1]], + chief_or_master: ['localhost:%d' % ports[0]], + 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], + 'ps': ['localhost:%d' % ports[-1]], } tf_config = {'cluster': cluster} procs = {} @@ -564,7 +579,8 @@ def _ps_worker_train( train_cmd += ' --fit_on_eval' if fit_on_eval_steps is not None: train_cmd += ' --fit_on_eval_steps ' + str(int(fit_on_eval_steps)) - procs[chief_or_master] = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, chief_or_master)) + procs[chief_or_master] = run_cmd( + train_cmd, '%s/log_%s.txt' % (test_dir, chief_or_master)) tf_config['task'] = {'type': 'ps', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') @@ -575,17 +591,23 @@ def _ps_worker_train( os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id(gpus[idx + 1]) worker_name = 'worker_%d' % idx - procs[worker_name] = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, worker_name)) + procs[worker_name] = run_cmd(train_cmd, + '%s/log_%s.txt' % (test_dir, worker_name)) if num_evaluator > 0: tf_config['task'] = {'type': 'evaluator', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') - procs['evaluator'] = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, 'evaluator')) + procs['evaluator'] = run_cmd(train_cmd, + '%s/log_%s.txt' % (test_dir, 'evaluator')) return procs -def _ps_worker_distribute_eval(pipeline_config_path, checkpoint_path, test_dir, num_worker, num_evaluator=0): +def _ps_worker_distribute_eval(pipeline_config_path, + checkpoint_path, + test_dir, + num_worker, + num_evaluator=0): gpus = get_available_gpus() # not enough gpus, run on cpu only if len(gpus) < num_worker: @@ -593,9 +615,9 @@ def _ps_worker_distribute_eval(pipeline_config_path, checkpoint_path, test_dir, ports = _get_ports(num_worker + 1) chief_or_master = 'master' if num_evaluator == 0 else 'chief' cluster = { - chief_or_master: ['localhost:%d' % ports[0]], - 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], - 'ps': ['localhost:%d' % ports[-1]], + chief_or_master: ['localhost:%d' % ports[0]], + 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], + 'ps': ['localhost:%d' % ports[-1]], } tf_config = {'cluster': cluster} procs = {} @@ -604,24 +626,29 @@ def _ps_worker_distribute_eval(pipeline_config_path, checkpoint_path, test_dir, os.environ[constant.SORT_COL_BY_NAME] = '1' set_gpu_id(gpus[0]) train_cmd = 'python -m easy_rec.python.eval --pipeline_config_path {} --checkpoint_path {} \ - --distribute_eval True --eval_result_path distribute_eval_result.txt'.format(pipeline_config_path, checkpoint_path) - procs[chief_or_master] = run_cmd(train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, chief_or_master)) + --distribute_eval True --eval_result_path distribute_eval_result.txt'.format( + pipeline_config_path, checkpoint_path) + procs[chief_or_master] = run_cmd( + train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, chief_or_master)) tf_config['task'] = {'type': 'ps', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') - procs['ps'] = run_cmd(train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, 'ps')) + procs['ps'] = run_cmd(train_cmd, + '%s/distribute_eval_log_%s.txt' % (test_dir, 'ps')) for idx in range(num_worker - 1): tf_config['task'] = {'type': 'worker', 'index': idx} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id(gpus[idx + 1]) worker_name = 'worker_%d' % idx - procs[worker_name] = run_cmd(train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, worker_name)) + procs[worker_name] = run_cmd( + train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, worker_name)) if num_evaluator > 0: tf_config['task'] = {'type': 'evaluator', 'index': 0} os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id('') - procs['evaluator'] = run_cmd(train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, 'evaluator')) + procs['evaluator'] = run_cmd( + train_cmd, '%s/distribute_eval_log_%s.txt' % (test_dir, 'evaluator')) return procs @@ -632,7 +659,11 @@ def _multi_worker_mirror_train(pipeline_config_path, test_dir, num_worker): if len(gpus) < num_worker: gpus = [None] * num_worker ports = _get_ports(num_worker) - tf_config = {'cluster': {'worker': ['localhost:%d' % ports[i] for i in range(num_worker)]}} + tf_config = { + 'cluster': { + 'worker': ['localhost:%d' % ports[i] for i in range(num_worker)] + } + } procs = {} train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path %s' % pipeline_config_path for idx in range(num_worker): @@ -640,7 +671,8 @@ def _multi_worker_mirror_train(pipeline_config_path, test_dir, num_worker): os.environ['TF_CONFIG'] = json.dumps(tf_config) set_gpu_id(gpus[idx]) worker_name = 'worker_%d' % idx - procs[worker_name] = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, worker_name)) + procs[worker_name] = run_cmd(train_cmd, + '%s/log_%s.txt' % (test_dir, worker_name)) return procs @@ -655,9 +687,9 @@ def _multi_worker_hvd_train(pipeline_config_path, test_dir, num_worker): ports = _get_ports(num_worker) hosts = ','.join(['localhost:%d' % ports[i] for i in range(num_worker)]) train_cmd = 'horovodrun -np %d --hosts %s python -m easy_rec.python.train_eval --pipeline_config_path %s' % ( - num_worker, - hosts, - pipeline_config_path, + num_worker, + hosts, + pipeline_config_path, ) proc = run_cmd(train_cmd, '%s/log_hvd.txt' % test_dir) proc_wait(proc, timeout=1200) @@ -665,25 +697,26 @@ def _multi_worker_hvd_train(pipeline_config_path, test_dir, num_worker): def test_distributed_train_eval( - pipeline_config_path, - test_dir, - total_steps=50, - num_evaluator=0, - edit_config_json=None, - use_hvd=False, - fit_on_eval=False, - num_epoch=0, + pipeline_config_path, + test_dir, + total_steps=50, + num_evaluator=0, + edit_config_json=None, + use_hvd=False, + fit_on_eval=False, + num_epoch=0, ): logging.info('testing pipeline config %s' % pipeline_config_path) - pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, total_steps, num_epoch) + pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, + total_steps, num_epoch) if edit_config_json is not None: config_util.edit_config(pipeline_config, edit_config_json) if use_hvd: pipeline_config.train_config.sync_replicas = False if pipeline_config.train_config.train_distribute not in [ - DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy, + DistributionStrategy.EmbeddingParallelStrategy, + DistributionStrategy.SokStrategy, ]: pipeline_config.train_config.train_distribute = DistributionStrategy.HorovodStrategy @@ -699,16 +732,17 @@ def test_distributed_train_eval( if train_config.train_distribute == DistributionStrategy.NoStrategy: num_worker = 2 procs = _ps_worker_train( - test_pipeline_config_path, - test_dir, - num_worker, - num_evaluator, - fit_on_eval, - fit_on_eval_steps=int(total_steps // 2), + test_pipeline_config_path, + test_dir, + num_worker, + num_evaluator, + fit_on_eval, + fit_on_eval_steps=int(total_steps // 2), ) elif train_config.train_distribute == DistributionStrategy.MultiWorkerMirroredStrategy: num_worker = 2 - procs = _multi_worker_mirror_train(test_pipeline_config_path, test_dir, num_worker) + procs = _multi_worker_mirror_train(test_pipeline_config_path, test_dir, + num_worker) else: raise NotImplementedError @@ -767,22 +801,32 @@ def test_distribute_eval_test(cur_eval_path, test_dir): return False single_data = read_data_from_json_path(single_work_eval_path) distribute_data = read_data_from_json_path(distribute_eval_path) - single_ret = {k: single_data[k] for k in single_data.keys() if 'loss' not in k and 'step' not in k} - distribute_ret = {k: distribute_data[k] for k in distribute_data.keys() if 'loss' not in k} + single_ret = { + k: single_data[k] + for k in single_data.keys() + if 'loss' not in k and 'step' not in k + } + distribute_ret = { + k: distribute_data[k] for k in distribute_data.keys() if 'loss' not in k + } difference_num = 0.00001 for k in single_ret.keys(): if abs(single_ret[k] - distribute_ret[k]) > difference_num: logging.error( - 'distribute_eval difference[%.8f] large than threshold[%.8f]' - % (abs(single_ret[k] - distribute_ret[k]), difference_num) - ) + 'distribute_eval difference[%.8f] large than threshold[%.8f]' % + (abs(single_ret[k] - distribute_ret[k]), difference_num)) return False return True -def test_distributed_eval(pipeline_config_path, checkpoint_path, test_dir, total_steps=50, num_evaluator=0): +def test_distributed_eval(pipeline_config_path, + checkpoint_path, + test_dir, + total_steps=50, + num_evaluator=0): logging.info('testing pipeline config %s' % pipeline_config_path) - pipeline_config = _load_config_for_distribute_eval(pipeline_config_path, test_dir) + pipeline_config = _load_config_for_distribute_eval(pipeline_config_path, + test_dir) train_config = pipeline_config.train_config config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') @@ -794,11 +838,11 @@ def test_distributed_eval(pipeline_config_path, checkpoint_path, test_dir, total if train_config.train_distribute == DistributionStrategy.NoStrategy: num_worker = 2 procs = _ps_worker_distribute_eval( - test_pipeline_config_path, - checkpoint_path, - test_dir, - num_worker, - num_evaluator, + test_pipeline_config_path, + checkpoint_path, + test_dir, + num_worker, + num_evaluator, ) else: raise NotImplementedError diff --git a/easy_rec/python/utils/tf_utils.py b/easy_rec/python/utils/tf_utils.py index 5ed647404..389b8ded7 100644 --- a/easy_rec/python/utils/tf_utils.py +++ b/easy_rec/python/utils/tf_utils.py @@ -25,12 +25,12 @@ def get_ps_num_from_tf_config(): def get_tf_type(field_type): type_map = { - DatasetConfig.INT32: tf.int32, - DatasetConfig.INT64: tf.int64, - DatasetConfig.STRING: tf.string, - DatasetConfig.BOOL: tf.bool, - DatasetConfig.FLOAT: tf.float32, - DatasetConfig.DOUBLE: tf.double, + DatasetConfig.INT32: tf.int32, + DatasetConfig.INT64: tf.int64, + DatasetConfig.STRING: tf.string, + DatasetConfig.BOOL: tf.bool, + DatasetConfig.FLOAT: tf.float32, + DatasetConfig.DOUBLE: tf.double, } assert field_type in type_map, 'invalid type: %s' % field_type return type_map[field_type] @@ -38,12 +38,12 @@ def get_tf_type(field_type): def get_col_type(tf_type): type_map = { - tf.int32: 'BIGINT', - tf.int64: 'BIGINT', - tf.string: 'STRING', - tf.float32: 'FLOAT', - tf.double: 'DOUBLE', - tf.bool: 'BOOLEAN', + tf.int32: 'BIGINT', + tf.int64: 'BIGINT', + tf.string: 'STRING', + tf.float32: 'FLOAT', + tf.double: 'DOUBLE', + tf.bool: 'BOOLEAN', } assert tf_type in type_map, 'invalid type: %s' % tf_type return type_map[tf_type] diff --git a/examples/data/amazon_books_data/process_amazon.py b/examples/data/amazon_books_data/process_amazon.py index 0da6c044e..7fc05f526 100644 --- a/examples/data/amazon_books_data/process_amazon.py +++ b/examples/data/amazon_books_data/process_amazon.py @@ -6,21 +6,21 @@ title = ['UserID', 'BookID', 'Time'] print('Reading train data...') train = pd.read_table( - 'AmazonBooksData/book_train.txt', - sep=',', - header=None, - names=title, - engine='python', - encoding='ISO-8859-1', + 'AmazonBooksData/book_train.txt', + sep=',', + header=None, + names=title, + engine='python', + encoding='ISO-8859-1', ) print('Reading test data...') test = pd.read_table( - 'AmazonBooksData/book_test.txt', - sep=',', - header=None, - names=title, - engine='python', - encoding='ISO-8859-1', + 'AmazonBooksData/book_test.txt', + sep=',', + header=None, + names=title, + engine='python', + encoding='ISO-8859-1', ) print('Start processing train data...') @@ -89,16 +89,20 @@ def gen_neg(): test_set_df = pd.DataFrame(test_set) print('Start writing amazon_train_data...') -train_set_df.to_csv(r'amazon_train_data', index=False, sep='\t', mode='a', header=False) +train_set_df.to_csv( + r'amazon_train_data', index=False, sep='\t', mode='a', header=False) print('Start writing amazon_test_data...') -test_set_df.to_csv(r'amazon_test_data', index=False, sep='\t', mode='a', header=False) +test_set_df.to_csv( + r'amazon_test_data', index=False, sep='\t', mode='a', header=False) print('Negative Sampling') train_book = train[['BookID']].drop_duplicates() test_book = test[['BookID']].drop_duplicates() negative_book = pd.concat([train_book, test_book]).drop_duplicates() -df_ones = pd.DataFrame(1, index=negative_book.index, columns=negative_book.columns) +df_ones = pd.DataFrame( + 1, index=negative_book.index, columns=negative_book.columns) negative_book_data = pd.concat([negative_book, df_ones, negative_book], axis=1) new_header = ['id:int64', 'weight:float', 'feature:string'] -negative_book_data.to_csv(r'negative_book_data', index=False, sep='\t', mode='a', header=new_header) +negative_book_data.to_csv( + r'negative_book_data', index=False, sep='\t', mode='a', header=new_header) print('Done.') diff --git a/examples/data/criteo/process_criteo_kaggle.py b/examples/data/criteo/process_criteo_kaggle.py index f7671abdd..5b9cb4f34 100644 --- a/examples/data/criteo/process_criteo_kaggle.py +++ b/examples/data/criteo/process_criteo_kaggle.py @@ -5,12 +5,15 @@ target_columns = ['label'] columns = target_columns + dense_features + category_features -data_train = pd.read_csv('criteo_kaggle_display/train.txt', sep='\t', names=columns) +data_train = pd.read_csv( + 'criteo_kaggle_display/train.txt', sep='\t', names=columns) samples_num = data_train.shape[0] print('samples_num:', samples_num, round(samples_num * 0.9)) train_num = int(round(samples_num * 0.9)) -data_train[:train_num].to_csv(r'criteo_train_data', index=False, sep='\t', mode='a', header=False) -data_train[train_num:].to_csv(r'criteo_test_data', index=False, sep='\t', mode='a', header=False) +data_train[:train_num].to_csv( + r'criteo_train_data', index=False, sep='\t', mode='a', header=False) +data_train[train_num:].to_csv( + r'criteo_test_data', index=False, sep='\t', mode='a', header=False) print('Done.') diff --git a/examples/data/movielens_1m/process_ml_1m.py b/examples/data/movielens_1m/process_ml_1m.py index bc3d64abd..be4dfd441 100644 --- a/examples/data/movielens_1m/process_ml_1m.py +++ b/examples/data/movielens_1m/process_ml_1m.py @@ -11,12 +11,12 @@ def process_data(): print('----User Data----') users_title = ['UserID', 'Gender', 'Age', 'JobID', 'ZipCode'] users = pd.read_table( - 'ml-1m/users.dat', - sep='::', - header=None, - names=users_title, - engine='python', - encoding='ISO-8859-1', + 'ml-1m/users.dat', + sep='::', + header=None, + names=users_title, + engine='python', + encoding='ISO-8859-1', ) users = users.filter(regex='UserID|Gender|Age|JobID|ZipCode') # process the gender and age of user @@ -30,19 +30,25 @@ def process_data(): print('----Movie Data----') movies_title = ['MovieID', 'Title', 'Genres'] movies = pd.read_table( - 'ml-1m/movies.dat', - sep='::', - header=None, - names=movies_title, - engine='python', - encoding='ISO-8859-1', + 'ml-1m/movies.dat', + sep='::', + header=None, + names=movies_title, + engine='python', + encoding='ISO-8859-1', ) # split the title and year in Feature:'Title' pattern = re.compile(r'^(.*)\((\d+)\)$') - title_map = {val: pattern.match(val).group(1) for ii, val in enumerate(set(movies['Title']))} - year_map = {val: pattern.match(val).group(2) for ii, val in enumerate(set(movies['Title']))} + title_map = { + val: pattern.match(val).group(1) + for ii, val in enumerate(set(movies['Title'])) + } + year_map = { + val: pattern.match(val).group(2) + for ii, val in enumerate(set(movies['Title'])) + } movies['Year'] = movies['Title'].map(year_map) movies['Title'] = movies['Title'].map(title_map) @@ -50,12 +56,12 @@ def process_data(): print('----Rating Data----') ratings_title = ['UserID', 'MovieID', 'ratings', 'timestamps'] ratings = pd.read_table( - 'ml-1m/ratings.dat', - sep='::', - header=None, - names=ratings_title, - engine='python', - encoding='ISO-8859-1', + 'ml-1m/ratings.dat', + sep='::', + header=None, + names=ratings_title, + engine='python', + encoding='ISO-8859-1', ) ratings = ratings.filter(regex='UserID|MovieID|ratings') # ratings of 4 and 5 are viewed as positive samples [label:1] @@ -83,6 +89,8 @@ def process_data(): # split train set and test set, and write to file print('Start writing to file.') -data_new[:665110].to_csv(r'movies_train_data', index=False, sep='\t', mode='a', header=False) -data_new[665110:].to_csv(r'movies_test_data', index=False, sep='\t', mode='a', header=False) +data_new[:665110].to_csv( + r'movies_train_data', index=False, sep='\t', mode='a', header=False) +data_new[665110:].to_csv( + r'movies_test_data', index=False, sep='\t', mode='a', header=False) print('Done.') diff --git a/examples/match_model/dssm.md b/examples/match_model/dssm.md index a5e7bdbf6..0fa909e94 100644 --- a/examples/match_model/dssm.md +++ b/examples/match_model/dssm.md @@ -61,7 +61,7 @@ model_config:{ - dnn: deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - id: 指定user_id/item_id列 -- simi_func: 向量相似度函数，包括[COSINE, INNER_PRODUCT, EUCLID]，默认COSINE，建议使用INNER_PRODUCT +- simi_func: 向量相似度函数，包括\[COSINE, INNER_PRODUCT, EUCLID\]，默认COSINE，建议使用INNER_PRODUCT - embedding_regularization: 对embedding部分加regularization，防止overfit 支持的metric_set包括: diff --git a/examples/match_model/dssm_negative_sample.md b/examples/match_model/dssm_negative_sample.md index 402915c3e..531081634 100644 --- a/examples/match_model/dssm_negative_sample.md +++ b/examples/match_model/dssm_negative_sample.md @@ -92,7 +92,7 @@ model_config:{ - dnn: deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - id: 指定user_id/item_id列 -- simi_func: 向量相似度函数，包括[COSINE, INNER_PRODUCT, EUCLID]，默认COSINE，建议使用INNER_PRODUCT +- simi_func: 向量相似度函数，包括\[COSINE, INNER_PRODUCT, EUCLID\]，默认COSINE，建议使用INNER_PRODUCT - scale_simi: 是否自动缩放相似度便于loss计算，建议设置成false - loss_type: 目前只支持SOFTMAX_CROSS_ENTROPY - embedding_regularization: 对embedding部分加regularization，防止overfit diff --git a/git-lfs/git_lfs.py b/git-lfs/git_lfs.py index edeaeb55d..2363193ce 100644 --- a/git-lfs/git_lfs.py +++ b/git-lfs/git_lfs.py @@ -12,14 +12,16 @@ blank_split = re.compile('[\t ]') logging.basicConfig( - format='[%(levelname)s] %(asctime)s %(filename)s[%(lineno)d] : %(message)s', - level=logging.INFO, + format='[%(levelname)s] %(asctime)s %(filename)s[%(lineno)d] : %(message)s', + level=logging.INFO, ) try: import oss2 except ImportError: - logging.error('please install python_oss from https://github.com/aliyun/aliyun-oss-python-sdk.git') + logging.error( + 'please install python_oss from https://github.com/aliyun/aliyun-oss-python-sdk.git' + ) sys.exit(1) git_bin_path = '.git_bin_path' @@ -49,8 +51,8 @@ def load_git_url(): line_str = line_str.strip() line_json = json.loads(line_str) git_bin_url_map[line_json['leaf_path']] = ( - line_json['sig'], - line_json['remote_path'], + line_json['sig'], + line_json['remote_path'], ) except Exception as ex: logging.warning('exception: %s' % str(ex)) @@ -64,9 +66,9 @@ def save_git_url(git_bin_url_map): for key in keys: val = git_bin_url_map[key] tmp_str = '{"leaf_path": "%s", "sig": "%s", "remote_path": "%s"}' % ( - key, - val[0], - val[1], + key, + val[0], + val[1], ) fout.write('%s\n' % tmp_str) @@ -107,7 +109,8 @@ def load_git_bin(): line_json = json.loads(line_str) file_arr[line_json['leaf_name']] = line_json['leaf_file'] except Exception as ex: - logging.warning('%s is corrupted : %s' % (git_bin_path, traceback.format_exc(ex))) + logging.warning('%s is corrupted : %s' % + (git_bin_path, traceback.format_exc(ex))) return file_arr @@ -120,8 +123,8 @@ def save_git_bin(git_arr): leaf_files.sort() # make sure that leaf_name is in front of leaf_file tmp_str = '{"leaf_name": "%s", "leaf_file": %s}' % ( - leaf_path, - json.dumps(leaf_files), + leaf_path, + json.dumps(leaf_files), ) fout.write('%s\n' % tmp_str) @@ -224,7 +227,9 @@ def get_yes_no(msg): if __name__ == '__main__': if len(sys.argv) < 2: - logging.error('usage: python git_lfs.py [pull] [push] [add filename] [resolve_conflict]') + logging.error( + 'usage: python git_lfs.py [pull] [push] [add filename] [resolve_conflict]' + ) sys.exit(1) home_directory = os.path.expanduser('~') with open('.git_oss_config_pub', 'r') as fin: @@ -241,7 +246,8 @@ def get_yes_no(msg): if line_str.startswith('#'): continue line_str = line_str.replace('~/', home_directory + '/') - line_str = line_str.replace('${TMPDIR}/', os.environ.get('TMPDIR', '/tmp/')) + line_str = line_str.replace('${TMPDIR}/', + os.environ.get('TMPDIR', '/tmp/')) line_str = line_str.replace('${PROJECT_NAME}', get_proj_name()) line_tok = [x.strip() for x in line_str.split('=') if x != ''] if line_tok[0] == 'host': @@ -253,13 +259,15 @@ def get_yes_no(msg): elif line_tok[0] == 'git_oss_private_config': git_oss_private_path = line_tok[1] if git_oss_private_path.startswith('~/'): - git_oss_private_path = os.path.join(home_directory, git_oss_private_path[2:]) + git_oss_private_path = os.path.join(home_directory, + git_oss_private_path[2:]) elif line_tok[0] == 'git_oss_cache_dir': git_oss_cache_dir = line_tok[1] elif line_tok[0] == 'accl_endpoint': accl_endpoint = line_tok[1] - logging.info('git_oss_data_dir=%s, host=%s, bucket_name=%s' % (git_oss_data_dir, host, bucket_name)) + logging.info('git_oss_data_dir=%s, host=%s, bucket_name=%s' % + (git_oss_data_dir, host, bucket_name)) logging.info('git_oss_cache_dir: %s' % git_oss_cache_dir) @@ -280,7 +288,8 @@ def get_yes_no(msg): oss_auth = oss2.Auth(accessid, accesskey) oss_bucket = oss2.Bucket(oss_auth, host, bucket_name) else: - logging.info('git_oss_private_path[%s] is not found, read-only mode' % git_oss_private_path) + logging.info('git_oss_private_path[%s] is not found, read-only mode' % + git_oss_private_path) # pull only mode oss_auth = None oss_bucket = None @@ -351,13 +360,14 @@ def get_yes_no(msg): local_sig = '' update = False - if len(sys.argv) > 2 and (sys.argv[2] == '-f' or sys.argv[2] == '--force'): + if len(sys.argv) > 2 and (sys.argv[2] == '-f' or + sys.argv[2] == '--force'): update = True else: if has_conflict(leaf_path, leaf_files): update = get_yes_no( - 'update %s using remote file[remote_sig=%s local_sig=%s]?[N/Y]' % (leaf_path, remote_sig, local_sig) - ) + 'update %s using remote file[remote_sig=%s local_sig=%s]?[N/Y]' % + (leaf_path, remote_sig, local_sig)) else: update = True if not update: @@ -393,7 +403,8 @@ def get_yes_no(msg): logging.warning('cache invalid, will download from remote') os.remove(tar_tmp_path) continue - logging.warning('download failed, local_sig(%s) != remote_sig(%s)' % (local_sig, remote_sig)) + logging.warning('download failed, local_sig(%s) != remote_sig(%s)' % + (local_sig, remote_sig)) except subprocess.CalledProcessError as ex: logging.error('exception: %s' % str(ex)) except oss2.exceptions.RequestError as ex: @@ -497,7 +508,8 @@ def get_yes_no(msg): else: git_objs[leaf_name] = leaf_file else: - logging.warning('invalid state: merge_start = %d, line_str = %s' % (merge_start, line_str)) + logging.warning('invalid state: merge_start = %d, line_str = %s' % + (merge_start, line_str)) save_git_bin(git_objs) git_bin_url_map = {} @@ -514,13 +526,16 @@ def get_yes_no(msg): line_json = json.loads(line_str) if line_json['leaf_path'] in git_objs: git_bin_url_map[line_json['leaf_path']] = ( - line_json['sig'], - line_json['remote_path'], + line_json['sig'], + line_json['remote_path'], ) else: - logging.warning('invalid state: merge_start = %d, line_str = %s' % (merge_start, line_str)) + logging.warning('invalid state: merge_start = %d, line_str = %s' % + (merge_start, line_str)) save_git_url(git_bin_url_map) logging.info('all conflicts fixed.') else: logging.warning('invalid cmd: %s' % sys.argv[1]) - logging.warning('choices are: %s' % ','.join(['push', 'pull', 'add', 'remove', 'resolve_conflict'])) + logging.warning( + 'choices are: %s' % + ','.join(['push', 'pull', 'add', 'remove', 'resolve_conflict'])) diff --git a/pai_jobs/deploy_ext.sh b/pai_jobs/deploy_ext.sh index ff17f1760..6dad904b3 100755 --- a/pai_jobs/deploy_ext.sh +++ b/pai_jobs/deploy_ext.sh @@ -60,7 +60,7 @@ then fi ODPSCMD=`which $ODPSCMD` -if [ $? -ne 0 ] && [ $mode -ne 2 ] +if [[ $? -ne 0 ]] && [[ $mode -ne 2 ]] then echo "$ODPSCMD is not in PATH" exit 1 diff --git a/pai_jobs/run.py b/pai_jobs/run.py index a9c920041..9761e98a7 100644 --- a/pai_jobs/run.py +++ b/pai_jobs/run.py @@ -6,7 +6,6 @@ from __future__ import print_function import logging - # use few threads to avoid oss error import os import time @@ -18,20 +17,16 @@ import easy_rec from easy_rec.python.inference.odps_predictor import ODPSPredictor from easy_rec.python.inference.vector_retrieve import VectorRetrieve -from easy_rec.python.main import _train_and_evaluate_impl as train_and_evaluate_impl from easy_rec.python.tools.pre_check import run_check + +from easy_rec.python.main import _train_and_evaluate_impl as train_and_evaluate_impl # NOQA + from easy_rec.python.utils import ( # NOQA - config_util, - constant, - estimator_utils, - fg_util, - hpo_util, - pai_util, + config_util, constant, estimator_utils, fg_util, hpo_util, pai_util, ) from easy_rec.python.utils.distribution_utils import ( # NOQA - DistributionStrategyMap, - set_distribution_config, - set_tf_config_and_get_train_worker_num, + DistributionStrategyMap, set_distribution_config, + set_tf_config_and_get_train_worker_num, ) os.environ['IS_ON_PAI'] = '1' @@ -46,118 +41,164 @@ logging.error('failed to import tfio: %s' % str(ex)) tf.disable_eager_execution() -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') -tf.app.flags.DEFINE_string('worker_hosts', '', 'Comma-separated list of hostname:port pairs') -tf.app.flags.DEFINE_string('ps_hosts', '', 'Comma-separated list of hostname:port pairs') +tf.app.flags.DEFINE_string('worker_hosts', '', + 'Comma-separated list of hostname:port pairs') +tf.app.flags.DEFINE_string('ps_hosts', '', + 'Comma-separated list of hostname:port pairs') tf.app.flags.DEFINE_string('job_name', '', 'task type, ps/worker') tf.app.flags.DEFINE_integer('task_index', 0, 'Index of task within the job') tf.app.flags.DEFINE_string('config', '', 'EasyRec config file path') -tf.app.flags.DEFINE_string('cmd', 'train', 'command type, train/evaluate/export') +tf.app.flags.DEFINE_string('cmd', 'train', + 'command type, train/evaluate/export') tf.app.flags.DEFINE_string('tables', '', 'tables passed by pai command') # flags for train -tf.app.flags.DEFINE_integer('num_gpus_per_worker', 1, 'number of gpu to use in training') -tf.app.flags.DEFINE_boolean('with_evaluator', False, 'whether a evaluator is necessary') +tf.app.flags.DEFINE_integer('num_gpus_per_worker', 1, + 'number of gpu to use in training') +tf.app.flags.DEFINE_boolean('with_evaluator', False, + 'whether a evaluator is necessary') tf.app.flags.DEFINE_string( - 'eval_method', - 'none', - 'default to none, choices are [none: not evaluate,' - + 'master: evaluate on master, separate: evaluate on a separate task]', + 'eval_method', + 'none', + 'default to none, choices are [none: not evaluate,' + + 'master: evaluate on master, separate: evaluate on a separate task]', ) -tf.app.flags.DEFINE_string('distribute_strategy', '', 'training distribute strategy') +tf.app.flags.DEFINE_string('distribute_strategy', '', + 'training distribute strategy') tf.app.flags.DEFINE_string('edit_config_json', '', 'edit config json string') tf.app.flags.DEFINE_string('train_tables', '', 'tables used for train') tf.app.flags.DEFINE_string('eval_tables', '', 'tables used for evaluation') tf.app.flags.DEFINE_string('boundary_table', '', 'tables used for boundary') tf.app.flags.DEFINE_string('sampler_table', '', 'tables used for sampler') -tf.app.flags.DEFINE_string('fine_tune_checkpoint', None, 'finetune checkpoint path') -tf.app.flags.DEFINE_string('query_table', '', 'table used for retrieve vector neighbours') -tf.app.flags.DEFINE_string('doc_table', '', 'table used for be retrieved as indexed vectors') -tf.app.flags.DEFINE_enum('knn_distance', 'inner_product', ['l2', 'inner_product'], 'type of knn distance') -tf.app.flags.DEFINE_integer('knn_num_neighbours', None, 'top n neighbours to be retrieved') -tf.app.flags.DEFINE_integer('knn_feature_dims', None, 'number of feature dimensions') +tf.app.flags.DEFINE_string('fine_tune_checkpoint', None, + 'finetune checkpoint path') +tf.app.flags.DEFINE_string('query_table', '', + 'table used for retrieve vector neighbours') +tf.app.flags.DEFINE_string('doc_table', '', + 'table used for be retrieved as indexed vectors') +tf.app.flags.DEFINE_enum('knn_distance', 'inner_product', + ['l2', 'inner_product'], 'type of knn distance') +tf.app.flags.DEFINE_integer('knn_num_neighbours', None, + 'top n neighbours to be retrieved') +tf.app.flags.DEFINE_integer('knn_feature_dims', None, + 'number of feature dimensions') tf.app.flags.DEFINE_enum( - 'knn_index_type', - 'ivfflat', - ['flat', 'ivfflat', 'ivfpq', 'gpu_flat', 'gpu_ivfflat', 'gpu_ivfpg'], - 'knn index type', + 'knn_index_type', + 'ivfflat', + ['flat', 'ivfflat', 'ivfpq', 'gpu_flat', 'gpu_ivfflat', 'gpu_ivfpg'], + 'knn index type', ) -tf.app.flags.DEFINE_string('knn_feature_delimiter', ',', 'delimiter for feature vectors') -tf.app.flags.DEFINE_integer('knn_nlist', 5, 'number of split part on each worker') -tf.app.flags.DEFINE_integer('knn_nprobe', 2, 'number of probe part on each worker') +tf.app.flags.DEFINE_string('knn_feature_delimiter', ',', + 'delimiter for feature vectors') +tf.app.flags.DEFINE_integer('knn_nlist', 5, + 'number of split part on each worker') +tf.app.flags.DEFINE_integer('knn_nprobe', 2, + 'number of probe part on each worker') tf.app.flags.DEFINE_integer( - 'knn_compress_dim', - 8, - 'number of dimensions after compress for `ivfpq` and `gpu_ivfpq`', + 'knn_compress_dim', + 8, + 'number of dimensions after compress for `ivfpq` and `gpu_ivfpq`', ) # flags used for evaluate & export tf.app.flags.DEFINE_string( - 'checkpoint_path', - '', - 'checkpoint to be evaluated or exported ' 'if not specified, use the latest checkpoint ' 'in train_config.model_dir', + 'checkpoint_path', + '', + 'checkpoint to be evaluated or exported ' + 'if not specified, use the latest checkpoint ' + 'in train_config.model_dir', ) # flags used for evaluate -tf.app.flags.DEFINE_string('eval_result_path', 'eval_result.txt', 'eval result metric file') -tf.app.flags.DEFINE_bool('distribute_eval', False, 'use distribute parameter server for train and eval.') +tf.app.flags.DEFINE_string('eval_result_path', 'eval_result.txt', + 'eval result metric file') +tf.app.flags.DEFINE_bool('distribute_eval', False, + 'use distribute parameter server for train and eval.') # flags used for export -tf.app.flags.DEFINE_string('export_dir', '', 'directory where model should be exported to') +tf.app.flags.DEFINE_string('export_dir', '', + 'directory where model should be exported to') tf.app.flags.DEFINE_bool('clear_export', False, 'remove export_dir if exists') -tf.app.flags.DEFINE_string('export_done_file', '', 'a flag file to signal that export model is done') -tf.app.flags.DEFINE_integer('max_wait_ckpt_ts', 0, 'max wait time in seconds for checkpoints') -tf.app.flags.DEFINE_boolean('continue_train', True, 'use the same model to continue train or not') +tf.app.flags.DEFINE_string('export_done_file', '', + 'a flag file to signal that export model is done') +tf.app.flags.DEFINE_integer('max_wait_ckpt_ts', 0, + 'max wait time in seconds for checkpoints') +tf.app.flags.DEFINE_boolean('continue_train', True, + 'use the same model to continue train or not') # flags used for predict -tf.app.flags.DEFINE_string('saved_model_dir', '', 'directory where saved_model.pb exists') +tf.app.flags.DEFINE_string('saved_model_dir', '', + 'directory where saved_model.pb exists') tf.app.flags.DEFINE_string('outputs', '', 'output tables') -tf.app.flags.DEFINE_string('all_cols', '', 'union of (selected_cols, reserved_cols), separated with , ') tf.app.flags.DEFINE_string( - 'all_col_types', - '', - 'column data types, for build record defaults, separated with ,', + 'all_cols', '', + 'union of (selected_cols, reserved_cols), separated with , ') +tf.app.flags.DEFINE_string( + 'all_col_types', + '', + 'column data types, for build record defaults, separated with ,', ) -tf.app.flags.DEFINE_string('selected_cols', '', 'columns to keep from input table, they are separated with ,') -tf.app.flags.DEFINE_string('reserved_cols', '', 'columns to keep from input table, they are separated with ,') tf.app.flags.DEFINE_string( - 'output_cols', - None, - 'output columns, such as: score float. multiple columns are separated by ,', + 'selected_cols', '', + 'columns to keep from input table, they are separated with ,') +tf.app.flags.DEFINE_string( + 'reserved_cols', '', + 'columns to keep from input table, they are separated with ,') +tf.app.flags.DEFINE_string( + 'output_cols', + None, + 'output columns, such as: score float. multiple columns are separated by ,', ) tf.app.flags.DEFINE_integer('batch_size', 1024, 'predict batch size') -tf.app.flags.DEFINE_string('profiling_file', None, 'time stat file which can be viewed using chrome tracing') +tf.app.flags.DEFINE_string( + 'profiling_file', None, + 'time stat file which can be viewed using chrome tracing') tf.app.flags.DEFINE_string('redis_url', None, 'export to redis url, host:port') tf.app.flags.DEFINE_string('redis_passwd', None, 'export to redis passwd') tf.app.flags.DEFINE_integer('redis_threads', 5, 'export to redis threads') -tf.app.flags.DEFINE_integer('redis_batch_size', 1024, 'export to redis batch_size') -tf.app.flags.DEFINE_integer('redis_timeout', 600, 'export to redis time_out in seconds') -tf.app.flags.DEFINE_integer('redis_expire', 24, 'export to redis expire time in hour') -tf.app.flags.DEFINE_string('redis_embedding_version', '', 'redis embedding version') +tf.app.flags.DEFINE_integer('redis_batch_size', 1024, + 'export to redis batch_size') +tf.app.flags.DEFINE_integer('redis_timeout', 600, + 'export to redis time_out in seconds') +tf.app.flags.DEFINE_integer('redis_expire', 24, + 'export to redis expire time in hour') +tf.app.flags.DEFINE_string('redis_embedding_version', '', + 'redis embedding version') tf.app.flags.DEFINE_integer('redis_write_kv', 1, 'whether write kv ') -tf.app.flags.DEFINE_string('oss_path', None, 'write embed objects to oss folder, oss://bucket/folder') +tf.app.flags.DEFINE_string( + 'oss_path', None, 'write embed objects to oss folder, oss://bucket/folder') tf.app.flags.DEFINE_string('oss_endpoint', None, 'oss endpoint') tf.app.flags.DEFINE_string('oss_ak', None, 'oss ak') tf.app.flags.DEFINE_string('oss_sk', None, 'oss sk') -tf.app.flags.DEFINE_integer('oss_threads', 10, '# threads access oss at the same time') -tf.app.flags.DEFINE_integer('oss_timeout', 10, 'connect to oss, time_out in seconds') +tf.app.flags.DEFINE_integer('oss_threads', 10, + '# threads access oss at the same time') +tf.app.flags.DEFINE_integer('oss_timeout', 10, + 'connect to oss, time_out in seconds') tf.app.flags.DEFINE_integer('oss_expire', 24, 'oss expire time in hours') -tf.app.flags.DEFINE_integer('oss_write_kv', 1, 'whether to write embedding to oss') +tf.app.flags.DEFINE_integer('oss_write_kv', 1, + 'whether to write embedding to oss') tf.app.flags.DEFINE_string('oss_embedding_version', '', 'oss embedding version') tf.app.flags.DEFINE_bool('verbose', False, 'print more debug information') -tf.app.flags.DEFINE_bool('place_embedding_on_cpu', False, 'whether to place embedding variables on cpu') +tf.app.flags.DEFINE_bool('place_embedding_on_cpu', False, + 'whether to place embedding variables on cpu') # for automl hyper parameter tuning tf.app.flags.DEFINE_string('model_dir', None, 'model directory') -tf.app.flags.DEFINE_bool('clear_model', False, 'remove model directory if exists') -tf.app.flags.DEFINE_string('hpo_param_path', None, 'hyperparameter tuning param path') -tf.app.flags.DEFINE_string('hpo_metric_save_path', None, 'hyperparameter save metric path') +tf.app.flags.DEFINE_bool('clear_model', False, + 'remove model directory if exists') +tf.app.flags.DEFINE_string('hpo_param_path', None, + 'hyperparameter tuning param path') +tf.app.flags.DEFINE_string('hpo_metric_save_path', None, + 'hyperparameter save metric path') tf.app.flags.DEFINE_string('asset_files', None, 'extra files to add to export') tf.app.flags.DEFINE_bool('check_mode', False, 'is use check mode') tf.app.flags.DEFINE_string('fg_json_path', None, '') -tf.app.flags.DEFINE_bool('enable_avx_str_split', False, 'enable avx str split to speedup') +tf.app.flags.DEFINE_bool('enable_avx_str_split', False, + 'enable avx str split to speedup') FLAGS = tf.app.flags.FLAGS @@ -174,14 +215,18 @@ def set_selected_cols(pipeline_config, selected_cols, all_cols, all_col_types): if all_cols: all_cols_arr = all_cols.split(',') all_col_types_arr = all_col_types.split(',') - all_col_types_map = {x.strip(): y.strip() for x, y in zip(all_cols_arr, all_col_types_arr)} + all_col_types_map = { + x.strip(): y.strip() for x, y in zip(all_cols_arr, all_col_types_arr) + } selected_cols_arr = [x.strip() for x in selected_cols.split(',')] selected_col_types = [all_col_types_map[x] for x in selected_cols_arr] selected_col_types = ','.join(selected_col_types) pipeline_config.data_config.selected_col_types = selected_col_types - print('[run.py] data_config.selected_cols = "%s"' % pipeline_config.data_config.selected_cols) - print('[run.py] data_config.selected_col_types = "%s"' % pipeline_config.data_config.selected_col_types) + print('[run.py] data_config.selected_cols = "%s"' % + pipeline_config.data_config.selected_cols) + print('[run.py] data_config.selected_col_types = "%s"' % + pipeline_config.data_config.selected_col_types) def _wait_ckpt(ckpt_path, max_wait_ts): @@ -194,7 +239,8 @@ def _wait_ckpt(ckpt_path, max_wait_ts): logging.info('wait for checkpoint in directory[%s]' % ckpt_path) time.sleep(30) else: - logging.info('find checkpoint[%s] in directory[%s]' % (tmp_ckpt, ckpt_path)) + logging.info('find checkpoint[%s] in directory[%s]' % + (tmp_ckpt, ckpt_path)) break else: while time.time() - start_ts < max_wait_ts: @@ -210,7 +256,8 @@ def main(argv): pai_util.set_on_pai() if FLAGS.enable_avx_str_split: constant.enable_avx_str_split() - logging.info('will enable avx str split: %s' % constant.is_avx_str_split_enabled()) + logging.info('will enable avx str split: %s' % + constant.is_avx_str_split_enabled()) if FLAGS.distribute_eval: os.environ['distribute_eval'] = 'True' @@ -226,15 +273,14 @@ def main(argv): worker_hosts = FLAGS.worker_hosts.split(',') num_worker = len(worker_hosts) assert FLAGS.distribute_strategy in DistributionStrategyMap, ( - 'invalid distribute_strategy [%s], available ones are %s' - % ( - FLAGS.distribute_strategy, - ','.join(DistributionStrategyMap.keys()), - ) - ) + 'invalid distribute_strategy [%s], available ones are %s' % ( + FLAGS.distribute_strategy, + ','.join(DistributionStrategyMap.keys()), + )) if FLAGS.config: - config = pai_util.process_config(FLAGS.config, FLAGS.task_index, len(FLAGS.worker_hosts.split(','))) + config = pai_util.process_config(FLAGS.config, FLAGS.task_index, + len(FLAGS.worker_hosts.split(','))) pipeline_config = config_util.get_configs_from_pipeline_file(config, False) # should be in front of edit_config_json step @@ -251,17 +297,20 @@ def main(argv): pipeline_config.model_dir = FLAGS.model_dir pipeline_config.model_dir = pipeline_config.model_dir.strip() print('[run.py] update model_dir to %s' % pipeline_config.model_dir) - assert pipeline_config.model_dir.startswith('oss://'), 'invalid model_dir format: %s' % pipeline_config.model_dir + assert pipeline_config.model_dir.startswith( + 'oss://'), 'invalid model_dir format: %s' % pipeline_config.model_dir if FLAGS.asset_files: - pipeline_config.export_config.asset_files.extend(FLAGS.asset_files.split(',')) + pipeline_config.export_config.asset_files.extend( + FLAGS.asset_files.split(',')) if FLAGS.config: if not pipeline_config.model_dir.endswith('/'): pipeline_config.model_dir += '/' if FLAGS.clear_model: - if gfile.IsDirectory(pipeline_config.model_dir) and estimator_utils.is_chief(): + if gfile.IsDirectory( + pipeline_config.model_dir) and estimator_utils.is_chief(): gfile.DeleteRecursively(pipeline_config.model_dir) if FLAGS.max_wait_ckpt_ts > 0: @@ -275,9 +324,11 @@ def main(argv): if not FLAGS.train_tables and FLAGS.tables: tables = FLAGS.tables.split(',') - assert len(tables) >= 2, 'at least 2 tables must be specified, but only[%d]: %s' % ( - len(tables), - FLAGS.tables, + assert len( + tables + ) >= 2, 'at least 2 tables must be specified, but only[%d]: %s' % ( + len(tables), + FLAGS.tables, ) if FLAGS.train_tables: @@ -298,12 +349,12 @@ def main(argv): if pipeline_config.train_config.HasField('fine_tune_checkpoint'): pipeline_config.train_config.fine_tune_checkpoint = estimator_utils.get_latest_checkpoint_from_checkpoint_path( - pipeline_config.train_config.fine_tune_checkpoint, False - ) + pipeline_config.train_config.fine_tune_checkpoint, False) if FLAGS.boundary_table: logging.info('Load boundary_table: %s' % FLAGS.boundary_table) - config_util.add_boundaries_to_config(pipeline_config, FLAGS.boundary_table) + config_util.add_boundaries_to_config(pipeline_config, + FLAGS.boundary_table) if FLAGS.sampler_table: pipeline_config.data_config.negative_sampler.input_path = FLAGS.sampler_table @@ -311,10 +362,10 @@ def main(argv): if FLAGS.train_tables or FLAGS.tables: # parse selected_cols set_selected_cols( - pipeline_config, - FLAGS.selected_cols, - FLAGS.all_cols, - FLAGS.all_col_types, + pipeline_config, + FLAGS.selected_cols, + FLAGS.all_cols, + FLAGS.all_col_types, ) else: pipeline_config.data_config.selected_cols = '' @@ -333,40 +384,44 @@ def main(argv): print('[run.py] with_evaluator %s' % str(FLAGS.with_evaluator)) print('[run.py] eval_method %s' % FLAGS.eval_method) - assert FLAGS.eval_method in ['none', 'master', 'separate'], 'invalid evalaute_method: %s' % FLAGS.eval_method + assert FLAGS.eval_method in [ + 'none', 'master', 'separate' + ], 'invalid evalaute_method: %s' % FLAGS.eval_method # with_evaluator is depreciated, keeped for compatibility if FLAGS.with_evaluator: FLAGS.eval_method = 'separate' num_worker = set_tf_config_and_get_train_worker_num( - FLAGS.ps_hosts, - FLAGS.worker_hosts, - FLAGS.task_index, - FLAGS.job_name, - distribute_strategy=distribute_strategy, - eval_method=FLAGS.eval_method, + FLAGS.ps_hosts, + FLAGS.worker_hosts, + FLAGS.task_index, + FLAGS.job_name, + distribute_strategy=distribute_strategy, + eval_method=FLAGS.eval_method, ) - set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy) + set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, + distribute_strategy) logging.info('run.py check_mode: %s .' % FLAGS.check_mode) train_and_evaluate_impl( - pipeline_config, - continue_train=FLAGS.continue_train, - check_mode=FLAGS.check_mode, + pipeline_config, + continue_train=FLAGS.continue_train, + check_mode=FLAGS.check_mode, ) if FLAGS.hpo_metric_save_path: hpo_util.save_eval_metrics( - pipeline_config.model_dir, - metric_save_path=FLAGS.hpo_metric_save_path, - has_evaluator=(FLAGS.eval_method == 'separate'), + pipeline_config.model_dir, + metric_save_path=FLAGS.hpo_metric_save_path, + has_evaluator=(FLAGS.eval_method == 'separate'), ) elif FLAGS.cmd == 'evaluate': check_param('config') # TODO: support multi-worker evaluation if not FLAGS.distribute_eval: - assert len(FLAGS.worker_hosts.split(',')) == 1, 'evaluate only need 1 worker' + assert len( + FLAGS.worker_hosts.split(',')) == 1, 'evaluate only need 1 worker' config_util.auto_expand_share_feature_configs(pipeline_config) if FLAGS.eval_tables: @@ -376,21 +431,22 @@ def main(argv): distribute_strategy = DistributionStrategyMap[FLAGS.distribute_strategy] set_tf_config_and_get_train_worker_num( - FLAGS.ps_hosts, - FLAGS.worker_hosts, - FLAGS.task_index, - FLAGS.job_name, - eval_method='none', + FLAGS.ps_hosts, + FLAGS.worker_hosts, + FLAGS.task_index, + FLAGS.job_name, + eval_method='none', ) - set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy) + set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, + distribute_strategy) if FLAGS.eval_tables or FLAGS.tables: # parse selected_cols set_selected_cols( - pipeline_config, - FLAGS.selected_cols, - FLAGS.all_cols, - FLAGS.all_col_types, + pipeline_config, + FLAGS.selected_cols, + FLAGS.all_cols, + FLAGS.all_col_types, ) else: pipeline_config.data_config.selected_cols = '' @@ -401,13 +457,15 @@ def main(argv): logging.info('will_use_distribute_eval') distribute_eval = os.environ.get('distribute_eval') logging.info('distribute_eval = {}'.format(distribute_eval)) - easy_rec.distribute_evaluate(pipeline_config, FLAGS.checkpoint_path, None, FLAGS.eval_result_path) + easy_rec.distribute_evaluate(pipeline_config, FLAGS.checkpoint_path, None, + FLAGS.eval_result_path) else: os.environ['distribute_eval'] = 'False' logging.info('will_use_eval') distribute_eval = os.environ.get('distribute_eval') logging.info('distribute_eval = {}'.format(distribute_eval)) - easy_rec.evaluate(pipeline_config, FLAGS.checkpoint_path, None, FLAGS.eval_result_path) + easy_rec.evaluate(pipeline_config, FLAGS.checkpoint_path, None, + FLAGS.eval_result_path) elif FLAGS.cmd == 'export': check_param('export_dir') check_param('config') @@ -452,11 +510,11 @@ def main(argv): oss_params['oss_write_kv'] = True if FLAGS.oss_write_kv == 1 else False set_tf_config_and_get_train_worker_num( - FLAGS.ps_hosts, - FLAGS.worker_hosts, - FLAGS.task_index, - FLAGS.job_name, - eval_method='none', + FLAGS.ps_hosts, + FLAGS.worker_hosts, + FLAGS.task_index, + FLAGS.job_name, + eval_method='none', ) assert len(FLAGS.worker_hosts.split(',')) == 1, 'export only need 1 worker' @@ -472,12 +530,12 @@ def main(argv): extra_params = redis_params extra_params.update(oss_params) export_out_dir = easy_rec.export( - export_dir, - pipeline_config, - FLAGS.checkpoint_path, - FLAGS.asset_files, - FLAGS.verbose, - **extra_params, + export_dir, + pipeline_config, + FLAGS.checkpoint_path, + FLAGS.asset_files, + FLAGS.verbose, + **extra_params, ) if FLAGS.export_done_file: flag_file = os.path.join(export_out_dir, FLAGS.export_done_file) @@ -487,49 +545,52 @@ def main(argv): elif FLAGS.cmd == 'predict': check_param('tables') check_param('saved_model_dir') - logging.info('will use the following columns as model input: %s' % FLAGS.selected_cols) - logging.info('will copy the following columns to output: %s' % FLAGS.reserved_cols) + logging.info('will use the following columns as model input: %s' % + FLAGS.selected_cols) + logging.info('will copy the following columns to output: %s' % + FLAGS.reserved_cols) profiling_file = FLAGS.profiling_file if FLAGS.task_index == 0 else None if profiling_file is not None: print('profiling_file = %s ' % profiling_file) predictor = ODPSPredictor( - FLAGS.saved_model_dir, - fg_json_path=FLAGS.fg_json_path, - profiling_file=profiling_file, - all_cols=FLAGS.all_cols, - all_col_types=FLAGS.all_col_types, + FLAGS.saved_model_dir, + fg_json_path=FLAGS.fg_json_path, + profiling_file=profiling_file, + all_cols=FLAGS.all_cols, + all_col_types=FLAGS.all_col_types, ) input_table, output_table = FLAGS.tables, FLAGS.outputs - logging.info('input_table = %s, output_table = %s' % (input_table, output_table)) + logging.info('input_table = %s, output_table = %s' % + (input_table, output_table)) worker_num = len(FLAGS.worker_hosts.split(',')) predictor.predict_impl( - input_table, - output_table, - reserved_cols=FLAGS.reserved_cols, - output_cols=FLAGS.output_cols, - batch_size=FLAGS.batch_size, - slice_id=FLAGS.task_index, - slice_num=worker_num, + input_table, + output_table, + reserved_cols=FLAGS.reserved_cols, + output_cols=FLAGS.output_cols, + batch_size=FLAGS.batch_size, + slice_id=FLAGS.task_index, + slice_num=worker_num, ) elif FLAGS.cmd == 'export_checkpoint': check_param('export_dir') check_param('config') set_tf_config_and_get_train_worker_num( - FLAGS.ps_hosts, - FLAGS.worker_hosts, - FLAGS.task_index, - FLAGS.job_name, - eval_method='none', + FLAGS.ps_hosts, + FLAGS.worker_hosts, + FLAGS.task_index, + FLAGS.job_name, + eval_method='none', ) assert len(FLAGS.worker_hosts.split(',')) == 1, 'export only need 1 woker' config_util.auto_expand_share_feature_configs(pipeline_config) easy_rec.export_checkpoint( - pipeline_config, - export_path=FLAGS.export_dir + '/model', - checkpoint_path=FLAGS.checkpoint_path, - asset_files=FLAGS.asset_files, - verbose=FLAGS.verbose, + pipeline_config, + export_path=FLAGS.export_dir + '/model', + checkpoint_path=FLAGS.checkpoint_path, + asset_files=FLAGS.asset_files, + verbose=FLAGS.verbose, ) elif FLAGS.cmd == 'vector_retrieve': check_param('knn_distance') @@ -537,38 +598,42 @@ def main(argv): assert FLAGS.knn_num_neighbours is not None, '`knn_num_neighbours` should not be None' query_table, doc_table, output_table = ( - FLAGS.query_table, - FLAGS.doc_table, - FLAGS.outputs, + FLAGS.query_table, + FLAGS.doc_table, + FLAGS.outputs, ) if not query_table: tables = FLAGS.tables.split(',') - assert len(tables) >= 1, 'at least 1 tables must be specified, but only[%d]: %s' % ( - len(tables), - FLAGS.tables, + assert len( + tables + ) >= 1, 'at least 1 tables must be specified, but only[%d]: %s' % ( + len(tables), + FLAGS.tables, ) query_table = tables[0] doc_table = tables[1] if len(tables) > 1 else query_table knn = VectorRetrieve( - query_table, - doc_table, - output_table, - ndim=FLAGS.knn_feature_dims, - distance=1 if FLAGS.knn_distance == 'inner_product' else 0, - delimiter=FLAGS.knn_feature_delimiter, - batch_size=FLAGS.batch_size, - index_type=FLAGS.knn_index_type, - nlist=FLAGS.knn_nlist, - nprobe=FLAGS.knn_nprobe, - m=FLAGS.knn_compress_dim, + query_table, + doc_table, + output_table, + ndim=FLAGS.knn_feature_dims, + distance=1 if FLAGS.knn_distance == 'inner_product' else 0, + delimiter=FLAGS.knn_feature_delimiter, + batch_size=FLAGS.batch_size, + index_type=FLAGS.knn_index_type, + nlist=FLAGS.knn_nlist, + nprobe=FLAGS.knn_nprobe, + m=FLAGS.knn_compress_dim, ) worker_hosts = FLAGS.worker_hosts.split(',') knn(FLAGS.knn_num_neighbours, FLAGS.task_index, len(worker_hosts)) elif FLAGS.cmd == 'check': run_check(pipeline_config, FLAGS.tables) else: - raise ValueError('cmd should be one of train/evaluate/export/predict/export_checkpoint/vector_retrieve') + raise ValueError( + 'cmd should be one of train/evaluate/export/predict/export_checkpoint/vector_retrieve' + ) if __name__ == '__main__': diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 0b02db173..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,52 +0,0 @@ -[tool.black] -line-length = 120 -target-version = ['py38', 'py39', 'py310'] -skip-string-normalization = false -# 注意：Black 不支持 2 空格缩进！Black 强制使用 4 空格 - -[tool.ruff] -line-length = 120 -indent-width = 2 # 添加缩进配置 -select = [ - "E", # pycodestyle errors - "F", # pyflakes - "W", # pycodestyle warnings - "I", # isort -] -ignore = [ - "E501", # line too long (handled by black) - "E203", # whitespace before ':' (conflicts with black) - "E111", # indentation is not a multiple of four - "E114", # indentation is not a multiple of four (comment) - "E117", # over-indented -] - -[tool.ruff.format] -indent-style = "space" -quote-style = "single" - -[tool.ruff.lint] -select = [ - "E", # pycodestyle errors - "F", # pyflakes - "W", # pycodestyle warnings - "I", # isort -] -ignore = [ - "E501", # line too long (handled by formatter) - "E203", # whitespace before ':' (conflicts with formatter) - "E111", # indentation is not a multiple of four - "E114", # indentation is not a multiple of four (comment) - "E117", # over-indented -] - -[tool.isort] -profile = "black" -line_length = 120 -indent = 2 # isort 的缩进配置 -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -ensure_newline_before_comments = true -known_first_party = ["easy_rec"] diff --git a/scripts/ci_test_change_files.py b/scripts/ci_test_change_files.py index 8538ad4f1..0502f90c1 100644 --- a/scripts/ci_test_change_files.py +++ b/scripts/ci_test_change_files.py @@ -11,12 +11,15 @@ logging.error(ex) sys.exit(2) -logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') +logging.basicConfig( + level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--pull_request_num', type=int, default=None, help='pull request number') - parser.add_argument('--exclude_dirs', nargs='*', type=str, help='the directory to be ignored') + parser.add_argument( + '--pull_request_num', type=int, default=None, help='pull request number') + parser.add_argument( + '--exclude_dirs', nargs='*', type=str, help='the directory to be ignored') args = parser.parse_args() diff --git a/scripts/pre-commit b/scripts/pre-commit index 93e8a05cc..fbd34dfde 100755 --- a/scripts/pre-commit +++ b/scripts/pre-commit @@ -18,7 +18,9 @@ os.environ.pop('__PYVENV_LAUNCHER__', None) # start templated INSTALL_PYTHON = '/apsarapangu/disk2/yancheng.lgq/miniconda3/envs/tf_1_15/bin/python' -ARGS = ['hook-impl', '--config=.pre-commit-config.yaml', '--hook-type=pre-commit'] +ARGS = [ + 'hook-impl', '--config=.pre-commit-config.yaml', '--hook-type=pre-commit' +] # end templated ARGS.extend(('--hook-dir', os.path.realpath(os.path.dirname(__file__)))) ARGS.append('--') diff --git a/setup.cfg b/setup.cfg index 79d3afea1..f0223c47a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,43 +1,34 @@ +[easy_install] +index_url = http://mirrors.aliyun.com/pypi/simple/ + +[bdist_wheel] +universal = 1 + [isort] -profile = black -line_length = 120 -indent = 2 -src_paths = easy_rec +line_length = 79 +multi_line_output = 7 +force_single_line = true known_standard_library = setuptools known_first_party = easy_rec known_third_party = absl,common_io,distutils,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,scipy,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,tensorflow_probability,yaml -skip = easy_rec/python/protos, venv +no_lines_before = LOCALFOLDER +default_section = THIRDPARTY +skip = easy_rec/python/protos [yapf] -based_on_style = pep8 -COLUMN_LIMIT = 120 -INDENT_WIDTH = 2 -CONTINUATION_INDENT_WIDTH = 2 -ALLOW_SPLIT_BEFORE_DICT_VALUE = False -BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF = True -SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET = False -NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS = True +BASED_ON_STYLE = yapf +ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT = true [flake8] -show-source = true -statistics = true +select = B,C,D,E,F,P,T4,W,B9 max-line-length = 120 -extend-select = - B,C,D,E,F,P,T4,W,B9, - BLK,T10 -extend-ignore = - E111,E114,E121,E124,E125,E126,E129,E203,E251,E721,F401, - W291,W503,W504, +ignore = + E111,E114,E125,E129,W291,W503,W504, + # docstring missing error should be used when all docstrings are completed D100,D101,D102,D103,D104,D105,D106,D107 per-file-ignores = - __init__.py:F401, - easy_rec/python/utils/test_utils.py:E402, - easy_rec/python/utils/io_util.py:E402, + __init__.py: F401 + easy_rec/python/utils/test_utils.py: E402 + easy_rec/python/utils/io_util.py: E402 +exclude = docs/src,scripts,easy_rec/python/protos,*.pyi,.git docstring-convention = google -enable-extensions = G - -[docformatter] -wrap_summaries = 120 -wrap_descriptions = 120 -blank = True -pre_summary_newline = False diff --git a/setup.py b/setup.py index ede80ef6f..674b7282f 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,8 @@ import codecs import os - -from setuptools import find_packages, setup +from setuptools import find_packages +from setuptools import setup def readme(): @@ -50,23 +50,23 @@ def parse_require_file(fpath): setup( - name='easy-rec', - version=get_version(), - description='An easy-to-use framework for Recommendation', - doc=readme(), - author='EasyRec Team', - author_email='easy_rec@alibaba-inc.com', - url='https://github.com/alibaba/EasyRec', - packages=find_packages(), - include_package_data=True, - classifiers=[ - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3', - ], - tests_require=parse_requirements('requirements/tests.txt'), - install_requires=parse_requirements('requirements/runtime.txt'), - extras_require={ - 'all': parse_requirements('requirements.txt'), - 'tests': parse_requirements('requirements/tests.txt'), - }, + name='easy-rec', + version=get_version(), + description='An easy-to-use framework for Recommendation', + doc=readme(), + author='EasyRec Team', + author_email='easy_rec@alibaba-inc.com', + url='https://github.com/alibaba/EasyRec', + packages=find_packages(), + include_package_data=True, + classifiers=[ + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3', + ], + tests_require=parse_requirements('requirements/tests.txt'), + install_requires=parse_requirements('requirements/runtime.txt'), + extras_require={ + 'all': parse_requirements('requirements.txt'), + 'tests': parse_requirements('requirements/tests.txt'), + }, ) From acf61f0e356d2b554cf9314f63821e8b70904b16 Mon Sep 17 00:00:00 2001 From: yangxudong Date: Wed, 17 Dec 2025 15:24:55 +0800 Subject: [PATCH 51/51] fix bug of BST component (default dropout rate) --- easy_rec/python/layers/keras/bst.py | 54 ++++++++++++++----- .../layers/multihead_cross_attention.py | 14 ++--- .../python/loss/zero_inflated_lognormal.py | 4 +- 3 files changed, 50 insertions(+), 22 deletions(-) diff --git a/easy_rec/python/layers/keras/bst.py b/easy_rec/python/layers/keras/bst.py index 123f86e73..ee4dc0561 100644 --- a/easy_rec/python/layers/keras/bst.py +++ b/easy_rec/python/layers/keras/bst.py @@ -19,16 +19,14 @@ def __init__(self, params, name='bst', reuse=None, **kwargs): self.l2_reg = params.l2_regularizer self.config = params.get_pb_config() - def encode(self, seq_input, max_position): + def encode(self, seq_input, seq_mask, max_position, hidden_dropout, + attention_dropout, target_pos): seq_fea = multihead_cross_attention.embedding_postprocessor( seq_input, position_embedding_name=self.name, max_position_embeddings=max_position, reuse_position_embedding=self.reuse, - ) - - n = tf.count_nonzero(seq_input, axis=-1) - seq_mask = tf.cast(n > 0, tf.int32) + dropout_prob=hidden_dropout) attention_mask = multihead_cross_attention.create_attention_mask_from_input_mask( from_tensor=seq_fea, to_mask=seq_mask) @@ -42,8 +40,8 @@ def encode(self, seq_input, max_position): attention_mask=attention_mask, intermediate_size=self.config.intermediate_size, intermediate_act_fn=hidden_act, - hidden_dropout_prob=self.config.hidden_dropout_prob, - attention_probs_dropout_prob=self.config.attention_probs_dropout_prob, + hidden_dropout_prob=hidden_dropout, + attention_probs_dropout_prob=attention_dropout, initializer_range=self.config.initializer_range, name=self.name + '/transformer', reuse=self.reuse, @@ -52,15 +50,19 @@ def encode(self, seq_input, max_position): if self.config.output_all_token_embeddings: out_fea = tf.reshape(attention_fea, [-1, max_position * self.config.hidden_size]) - else: + elif target_pos == 'head': out_fea = attention_fea[:, 0, :] # target feature + else: + out_fea = attention_fea[:, -1, :] # target feature print('bst output shape:', out_fea.shape) return out_fea def call(self, inputs, training=None, **kwargs): - if not training: - self.config.hidden_dropout_prob = 0.0 - self.config.attention_probs_dropout_prob = 0.0 + hidden_dropout = self.config.hidden_dropout_prob if training else 0.0 + attention_dropout = self.config.attention_probs_dropout_prob if training else 0.0 + tf.logging.info( + 'BST config hidden_dropout_prob=%f, attention_probs_dropout_prob=%f', + hidden_dropout, attention_dropout) assert isinstance(inputs, (list, tuple)) assert len(inputs) >= 2 # seq_input: [batch_size, seq_len, embed_size] @@ -78,6 +80,9 @@ def call(self, inputs, training=None, **kwargs): ', you should set `max_seq_len` in sequence feature configs', ) + orig_mask = tf.sequence_mask( + seq_len, maxlen=cur_batch_max_seq_len, dtype=tf.int32) + keep_target = self.config.target_item_position in ('head', 'tail') if self.config.output_all_token_embeddings: seq_input = tf.cond( tf.constant(max_position) > cur_batch_max_seq_len, @@ -88,6 +93,29 @@ def call(self, inputs, training=None, **kwargs): ), lambda: tf.slice(seq_input, [0, 0, 0], [-1, max_position, -1]), ) + orig_mask = tf.cond( + tf.constant(max_position) > cur_batch_max_seq_len, + lambda: tf.pad(orig_mask, [[0, 0], + [0, max_position - cur_batch_max_seq_len]], + 'CONSTANT'), + lambda: tf.slice(orig_mask, [0, 0], [-1, max_position]), + ) + # 再根据 target 拼接方式，对 orig_mask 做相同的 concat/pad + if target is not None and keep_target: + # target 位置一般是有效的 1，或者你也可以选择 0 + target_mask = tf.ones([batch_size, 1], dtype=tf.int32) + if self.config.target_item_position == 'head': + orig_mask = tf.concat([target_mask, orig_mask], axis=1) + else: + orig_mask = tf.concat([orig_mask, target_mask], axis=1) + elif self.config.reserve_target_position: + target_mask = tf.ones([batch_size, 1], dtype=tf.int32) + if self.config.target_item_position == 'head': + orig_mask = tf.concat([target_mask, orig_mask], axis=1) + else: + orig_mask = tf.concat([orig_mask, target_mask], axis=1) + + seq_mask = orig_mask if seq_embed_size != self.config.hidden_size: seq_input = tf.layers.dense( @@ -99,7 +127,6 @@ def call(self, inputs, training=None, **kwargs): reuse=self.reuse, ) - keep_target = self.config.target_item_position in ('head', 'tail') if target is not None and keep_target: target_size = target.shape.as_list()[-1] assert seq_embed_size == target_size, ( @@ -126,4 +153,5 @@ def call(self, inputs, training=None, **kwargs): max_position += 1 with tf.control_dependencies([valid_len]): - return self.encode(seq_input, max_position) + return self.encode(seq_input, seq_mask, max_position, hidden_dropout, + attention_dropout, self.config.target_item_position) diff --git a/easy_rec/python/layers/multihead_cross_attention.py b/easy_rec/python/layers/multihead_cross_attention.py index 77697960c..0f6b6ea69 100644 --- a/easy_rec/python/layers/multihead_cross_attention.py +++ b/easy_rec/python/layers/multihead_cross_attention.py @@ -251,8 +251,8 @@ def transformer_encoder( num_attention_heads=12, intermediate_size=3072, intermediate_act_fn=gelu, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, initializer_range=0.02, reuse=None, name='transformer', @@ -378,8 +378,8 @@ def cross_attention_block( self_attention_mask=None, num_attention_heads=1, intermediate_size=512, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, initializer_range=0.02, name='', ): @@ -496,8 +496,8 @@ def cross_attention_tower( right_intermediate_size=0, left_input_mask=None, right_input_mask=None, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, initializer_range=0.02, name='', ): @@ -674,7 +674,7 @@ def embedding_postprocessor( reuse_position_embedding=None, initializer_range=0.02, max_position_embeddings=512, - dropout_prob=0.1, + dropout_prob=0.0, ): """Performs various post-processing on a word embedding tensor. diff --git a/easy_rec/python/loss/zero_inflated_lognormal.py b/easy_rec/python/loss/zero_inflated_lognormal.py index 76072e115..3ffb9a3b4 100644 --- a/easy_rec/python/loss/zero_inflated_lognormal.py +++ b/easy_rec/python/loss/zero_inflated_lognormal.py @@ -118,8 +118,8 @@ def zero_inflated_lognormal_loss( # add regular terms loc_penalty = mu_reg * tf.reduce_mean(tf.square(mu)) scale_penalty = sigma_reg * tf.reduce_mean(tf.square(logits[..., 2:])) - tf.summary.scalar('loss/%s_loc_penalty' % loss_name, loc_penalty) - tf.summary.scalar('loss/%s_scale_penalty' % loss_name, scale_penalty) + tf.summary.scalar('loss/%s_mu_penalty' % loss_name, loc_penalty) + tf.summary.scalar('loss/%s_sigma_penalty' % loss_name, scale_penalty) total_loss = class_weight * classification_loss + reg_weight * regression_loss return total_loss + loc_penalty + scale_penalty